def test_missing_dependency_dataframe(monkeypatch): api = SentinelAPI("mock_user", "mock_password") with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "pandas", None) api.to_dataframe({"test":"test"}) with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "geopandas", None) api.to_geodataframe({"test":"tst"})
def test_missing_dependency_dataframe(monkeypatch): api = SentinelAPI("mock_user", "mock_password") with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "pandas", None) api.to_dataframe({"test": "test"}) with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "geopandas", None) api.to_geodataframe({"test": "tst"})
def test_to_geopandas(products): gdf = SentinelAPI.to_geodataframe(products) assert type(gdf).__name__ == 'GeoDataFrame' print(gdf.unary_union.area) assert gdf.unary_union.area == pytest.approx(89.6, abs=0.1) assert len(gdf) == len(products) assert gdf.crs == {'init': 'epsg:4326'}
def downloadingData(aoi, collectionDate, plName, prLevel, clouds, username, password, directory): ''' Downloads the Sentinel2 - Data with the given parameters Parameters: aoi (str): The type and the coordinates of the area of interest collectionDate (datetime 64[ns]): The date of the data plName (str): The name of the platform prLevel (str): The name of the process clouds (tuple of ints): Min and max of cloudcoverpercentage username (str): The username of the Copernicus SciHub password (str): The password of the Copernicus SciHub directory (str): Pathlike string to the directory ''' api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus') '''Choosing the data with bounding box (footprint), date, platformname, processinglevel and cloudcoverpercentage''' products = api.query(aoi, date=collectionDate, platformname=plName, processinglevel=prLevel, cloudcoverpercentage=clouds) '''Filters the products and sorts by cloudcoverpercentage''' products_gdf = api.to_geodataframe(products) products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'], ascending=[True]) '''Downloads the choosen files from Scihub''' products_gdf_sorted.to_csv(os.path.join(directory, 'w')) api.download_all(products, directory, max_attempts=10, checksum=True)
def test_to_geopandas(products): gdf = SentinelAPI.to_geodataframe(products) assert type(gdf).__name__ == "GeoDataFrame" print(gdf.unary_union.area) assert gdf.unary_union.area == pytest.approx(89.6, abs=0.1) assert len(gdf) == len(products) assert gdf.crs == "EPSG:4326"
def get_image_products(self): """ Returns ------- products : List Contains collection of information of available images from query """ print('\n >>> CONDUCTING API QUERY') api = SentinelAPI(self.username, self.password, 'https://scihub.copernicus.eu/dhus') shape = gpd.read_file(self.aoi_dir) shape = shape.to_crs({'init': 'epsg:4326'}) footprint = None for i in shape['geometry']: footprint = i products = api.query(footprint, date=self.date_range, platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=(0, 10)) products = api.to_geodataframe(products) products = products.sort_values(['cloudcoverpercentage'], ascending=[True]) return products
def get_data(name, AREA, start_date='20190101', end_date='20190128', platformname='Sentinel-2', processinglevel='Level-2A'): #apiのクエリに次のが必要。中身はPOLYGON((..))っていうstr型 footprint_geojson = geojson_to_wkt( read_geojson('./location/' + str(name) + '.geojson')) # use sentinelAPI user = '' password = '' api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') products = api.query( footprint_geojson, date=(start_date, end_date), #取得希望期間の入力 platformname=platformname, processinglevel=processinglevel, ##2016年はL1C cloudcoverpercentage=(0, 100)) #被雲率(0%〜100%) print("この期間の画像の数は" + str(len(products)) + "枚です") #この後で雲の被覆率ごとにデータを並べ替えて、1個ダウンロードする。 products_gdf = api.to_geodataframe(products) products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'], ascending=[True]).head() #ファイルがまだなければデータダウンロード。作業ディレクトリにzipファイルがダウンロードされる for i in range(3): #3回までチャレンジ uuid = products_gdf_sorted.iloc[i]["uuid"] product_title = products_gdf_sorted.iloc[i][ "title"] #S2A_MSIL2A_20190101T01405... みたいな product_date = products_gdf_sorted.iloc[i]["summary"].split( ',')[0].split()[1][:10] print("この期間で1番被覆率が低いのは" + product_date + "日") if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True: print("新規にデータをダウンロードします") try: api.download(uuid) except: print("ダウンロード不可") else: break else: break if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True: #ダウンロードしたzipファイルを解凍 #str(product_title) + '.SAFE っていうフォルダが生成される file_name = str(product_title) + '.zip' with zipfile.ZipFile(file_name) as zf: zf.extractall() shutil.move(str(product_title) + '.zip', './data_geo/') shutil.move(str(product_title) + '.SAFE', './data_geo/') return product_title
def find_s2_uuid(s2_filename): """ Returns S2 uuid required for download via sentinelsat, based upon an input S2 file/scene name. I.e. S2A_MSIL1C_20180820T223011_N0206_R072_T60KWE_20180821T013410 Assumes esa hub creds stored as env variables. :param s2_file_name: Sentinel-2 scene name :return s2_uuid: download id """ copernicus_username = os.getenv("COPERNICUS_USERNAME") copernicus_pwd = os.getenv("COPERNICUS_PWD") print(f"ESA username: {copernicus_username}") esa_api = SentinelAPI(copernicus_username, copernicus_pwd) if s2_filename[-5:] == '.SAFE': res = esa_api.query(filename=s2_filename) res = esa_api.to_geodataframe(res) return res.uuid.values[0]
def searchOverlapping(geom, search_args): ''' use sentinel API to search for scenes that overlap with geom Inputs: ------- geom: shapely geometry. search_args : passed onto SentinelAPI.query, read the docs. Outputs: ------- Geodataframe with Sentinel Products overlapping with geom. ''' api = SentinelAPI(SENTINEL_USER, SENTINEL_PASS, 'https://scihub.copernicus.eu/dhus/') products = api.query(geom, **search_args) if len(products) == 0: return (None) return (api.to_geodataframe(products))
def check_login(username, password, latitude, longitude): ''' Checks the login and location details, SentinelAPI queries the Copernicus database ''' if username == None or password == None: print( '\n Enter Login Details for the Copernicus SciHub\n if not registered, go to:\n https://scihub.copernicus.eu\n' ) username = input('\n Username: '******'\n Password: '******'\n') if latitude == None or longitude == None: print('Latitude and Longitude in decimal degrees\n') try: latitude = float(input('\n Latitude: ')) longitude = float(input('\n Longitude: ')) except: print( '\n Latitude and Longitude are to be entered in decimal degrees\n Program Terminated\n' ) sys.exit() print('\n') try: api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus') footprint = f'POINT ({latitude} {longitude})' data_populate = api.query(footprint, date=('NOW-12MONTHS', 'NOW'), order_by='cloudcoverpercentage', platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=(0, 30)) data_database = api.to_geodataframe(data_populate) return api, data_database except: print('\n Incorrect Login Details.\n Program Terminated.') sys.exit()
class Downloader: def __init__(self, str_username, str_password, str_link): self.api = SentinelAPI(str_username, str_password, str_link) self.products = None def search_polygon(self, footprint: object, str_date_start: str, str_date_end: str, str_platform_name: str, percentage: object): print('searching') self.products = self.api.query(footprint, date=(str_date_start, str_date_end), platformname=str_platform_name, cloudcoverpercentage=(percentage[0], percentage[1])) size = self.api.get_products_size(self.products) print(f'found {size}GiB of data') # print(self.products) def download_zip(self, path): self.api.download_all(self.products, path, max_attempt, True) def download_products(self, path, download_file): if download_file: self.download_zip(path) print('downloaded') df_products = self.api.to_dataframe(self.products) return df_products def download_geoproduct(self, path, download_file): if download_file: self.download_zip(path) # print('download Geos') gdf_products = self.api.to_geodataframe(self.products) return gdf_products def download_json(self): return self.api.to_geojson(self.products) def download_one(self, key, path): self.api.download(key, path, True)
class GetImage(): def __init__(self): self.api = None self._connect_to_api() def _connect_to_api(self): user = '******' password = "******" self.api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') def _create_footprint(self, sf): footprint = None for i in sf['geometry']: footprint = i return footprint def _get_product(self, footprint, num_images, start_date=0, end_date=0, processing_level='Level-2A'): products = self.api.query(footprint, date=('20190601', '20190626'), platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=(0, 10)) products_gdf = self.api.to_geodataframe(products) best_products = products_gdf.sort_values( ['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]).head(num_images) print(best_products[['title', 'cloudcoverpercentage']]) return best_products def _download_image(self, products): save_path = "products/" extension = ".zip" origin_path = os.getcwd() os.makedirs(save_path, exist_ok=True) os.chdir(save_path) current_path = os.getcwd() self.api.download_all(products.index) for item in os.listdir(current_path): if item.endswith(extension): file_name = os.path.abspath(item) zip_ref = zipfile.ZipFile(file_name) zip_ref.extractall(current_path) zip_ref.close() os.remove(file_name) print(current_path) directories = [x for x in glob.glob(current_path + '/*/')] os.chdir(origin_path) return directories def get_image(self, shp_path, num_images): sf = gpd.read_file(shp_path) footprint = self._create_footprint(sf) products = self._get_product(footprint, num_images) list_images = self._download_image(products) print(list_images) return 0
import datetime from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt import psycopg2 api = SentinelAPI('MRDEVISH', 'Research1') today_date = datetime.date.today() footprint = geojson_to_wkt(read_geojson('search_polygon_10.geojson')) platformname = 'Sentinel-3' producttype='SL_2_LST___' products = api.query(footprint,platformname=platformname,producttype=producttype, date = (today_date-1,today_date)) products_pandas = api.to_geodataframe(products) Names = products_pandas['Product Name'] connection1 = psycopg2.connect(user="******", password="******", host="127.0.0.1", port="5432", database="db_products") cursor1 = connection1.cursor() query0 = """select "Product Name" from "Products"; """ cursor1.execute(query0) records = cursor1.fetchone() num=1 undownloaded = [] for i in Names: if i not in records: cursor1 = connection1.cursor() id_ = len(records)+num query = """INSERT INTO "Products" Values({},{},{},0);""".format(id_,i,today_date) cursor1.execute(query0) num=num+1 import pika
def lst_prod(shpExtent, start_time, end_time, outShp=None, platname="Sentinel-2", procLevel="Level-2A", max_cloud_cover=None): """ List Sentinel Products for one specific area platformname: * Sentinel-1 * Sentinel-2 * Sentinel-3 processinglevel: * Level-1A * Level-1B * Level-1C * Level-2A ... """ import os from sentinelsat import SentinelAPI, geojson_to_wkt from glass.g.rd.shp import shp_to_obj from glass.pys.oss import fprop from glass.g.wt.shp import df_to_shp from glass.cons.sentinel import con_datahub # Get Search Area if os.path.isfile(shpExtent): if fprop(shpExtent, 'ff') == '.json': boundary = geojson_to_wkt(shpExtent) else: boundary = shp_to_obj(shpExtent, output='array', fields=None, geom_as_wkt=True, srs_to=4326)[0]["GEOM"] else: # Assuming we have an WKT boundary = shpExtent # Create API instance user, password = con_datahub() api = SentinelAPI(user, password, URL_COPERNICUS) # Search for products products = api.query( boundary, date=(start_time, end_time), platformname=platname, cloudcoverpercentage=(0, 100 if not max_cloud_cover else max_cloud_cover), processinglevel=procLevel) df_prod = api.to_geodataframe(products) if not df_prod.shape[0]: return df_prod df_prod['ingestiondate'] = df_prod.ingestiondate.astype(str) df_prod['beginposition'] = df_prod.beginposition.astype(str) df_prod['endposition'] = df_prod.endposition.astype(str) # Export results to Shapefile if outShp: return df_to_shp(df_prod, outShp) else: return df_prod
def test_to_geopandas_empty(): gdf = SentinelAPI.to_geodataframe({}) assert type(gdf).__name__ == "GeoDataFrame" assert len(gdf) == 0
def test_missing_dependency_geodataframe(monkeypatch): with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "geopandas", None) SentinelAPI.to_geodataframe({"test": "tst"})
api = SentinelAPI('fabiolana', 'Albertone_2016') footprint = geojson_to_wkt(read_geojson('/media/sf_Downloads/__effis/ricerca.geojson')) products = api.query(footprint, # date=('20170801', date(2017, 12, 15)), date=('20151001', '20171205'), platformname='Sentinel-2', # producttype='SLC') #,orbitdirection='ASCENDING') #, cloudcoverpercentage=(0, 10)) # download all results from the search # api.download_all(products) # GeoJSON FeatureCollection containing footprints and metadata of the scenes api.to_geojson(products) # inserito in dataframe df_prods = pd.DataFrame(api.to_geojson(products)) # print df_prods # GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries print api.to_geodataframe(products) # Get basic information about the product: its title, file size, MD5 sum, date, footprint and # its download url # api.get_product_odata(<product_id>) # Get the product's full metadata available on the server # api.get_product_odata(<product_id>, full=True)
def main(args): """ Runs dataLayer processing scripts to turn raw dataLayer from (../raw) into cleaned dataLayer ready to be analyzed (saved in ../processed). """ ## Talk to Rune about how dataLayer is handle. If it should be part of the "big" project. ## set number_tiles:1764 config = TrainingConfig() config = update_config(args, config) logger = logging.getLogger(__name__) logger.info('making final dataLayer set from raw dataLayer') userTuple = [['pandagud', 'damp4ever'], ['pandagud2', 'damp4ever'], ['pandagud3', 'damp4ever'], ['au524478', 'Palantir1234']] current_user = random.choice(userTuple) api = SentinelAPI(current_user[0], current_user[1], 'https://scihub.copernicus.eu/dhus') # search by polygon, time, and SciHub query keywords path = r"C:\Users\panda\Downloads\LC80290292014132LGN00.geojson" footprint = geojson_to_wkt(read_geojson(path)) products = api.query(area=footprint, date=('20210101', '20210105'), platformname='Sentinel-2', order_by='+ingestiondate', limit=1) areas = api.to_geodataframe(products) geojson = api.to_geojson(products) api.download_all(products, into=r'C:\Users\panda\Sat_paper\Alfa') products = api.query(area=footprint, date=('20210401', '20210430'), producttype='GRD', platformname='Sentinel-1', sensoroperationalmode='IW', polarisationmode='VV VH', order_by='ingestiondate') firstproduct = next(iter(products)) online_product = '' for i in products: is_online = api.is_online(products.get(i).get('uuid')) if is_online: online_product = i break delete_list = [] for i in products: if i != online_product: delete_list.append(i) for i in delete_list: del products[i] ground_geojsons = read_geojson(path) products_geojsons = api.to_geojson(products) ground_polygon = ground_geojsons.get('features')[0].get('geometry').get( 'coordinates') ground_polygon = geometry.Polygon(ground_polygon[0][0]) import numpy as np titles = [] ids = [] for item in products_geojsons.get('features'): id = item.get('properties').get('id') item = item.get('properties').get('title') item = (item[17:25] + item[48:55]) titles.append(item) ids.append([item, id]) unique = list(set(titles)) union_list = [] for i, element in enumerate(unique): local_polygon = Polygon() for j in range(len(titles)): if titles[j] == element: item = products_geojsons.get('features')[j] item = item.get('geometry').get('coordinates') item = geometry.Polygon(item[0][0]) item = affinity.scale(item, xfact=1.01, yfact=1.01) polygons = [item, local_polygon] local_polygons = unary_union(polygons) local_polygon = item union_list.append([local_polygons, element]) for index, element in enumerate(union_list): wkt = element[0].wkt if ground_polygon.within(element[0]): found_id = element[1] break for i in ids: if found_id != i[0]: del products[i[1]] area_list = [] for index, item in enumerate(products_geojsons.get('features')): item = item.get('geometry').get('coordinates') item = geometry.Polygon(item[0][0]) local_intersection = item.intersection(ground_polygon) local_intersection = [local_intersection.area, index] area_list.append(local_intersection) area_list.sort(reverse=True) for index in range(len(area_list)): item = products_geojsons.get('features')[area_list[index][1]] id = item.get('properties').get('id') item = item.get('geometry').get('coordinates') item = geometry.Polygon(item[0][0]) if item.intersects(ground_polygon): local_intersection = ground_polygon.intersection(item) print(str(ground_polygon.area)) print(str(local_intersection.area)) # ground_polygon = ground_polygon.difference(local_intersection) ground_polygon = (ground_polygon.symmetric_difference( local_intersection)).difference(local_intersection) else: del products[id] import datetime from datetime import timedelta S2_geojson = read_geojson(path) start_S1_date = S2_geojson.get('features')[0].get('properties').get( 'ingestiondate') start_S1_date = start_S1_date.split('T')[0] start_S1_date = datetime.datetime.strptime(start_S1_date, '%Y-%m-%d').date() ## New end date for S1 end_S1_date = start_S1_date + timedelta(days=7) start_S1_date = start_S1_date - timedelta(days=7) start_S1_date_str = str(start_S1_date).replace('-', '') end_S1_date_str = str(end_S1_date).replace('-', '') ## COMBINE FOOTPRINT geom_in_geojson = [] geom_in_geojson.append( geojson.Feature(geometry=ground_polygon, properties={"MissingData": "Test"})) feature_collection = FeatureCollection(geom_in_geojson) pathToFile = r'C:\Users\panda\Sat_paper\missing.geojson' with open(pathToFile, 'w') as f: dump(feature_collection, f) print("Done")
def test_to_geopandas(products): gdf = SentinelAPI.to_geodataframe(products) assert type(gdf).__name__ == 'GeoDataFrame' assert abs(gdf.unary_union.area - 132.16) < 0.01 assert len(gdf) == len(products) assert gdf.crs == {'init': 'epsg:4326'}
def test_to_geopandas_empty(products): gdf = SentinelAPI.to_geodataframe({}) assert type(gdf).__name__ == 'GeoDataFrame' assert len(gdf) == 0
Coordinates_texts = [list(map(str, x)) for x in Coordinates_texts] for i in range(len(Coordinates_texts)): Coordinates_texts[i]=','.join(Coordinates_texts[i]) Coordinates_texts = ' '.join(Coordinates_texts) element.findall(('{0}Document/{0}Placemark/{0}Polygon/{0}outerBoundaryIs/{0}LinearRing/{0}coordinates'.format(namespace))).text = Coordinates_texts element.remove('{0}Document/{0}Placemark/{0}ExtendedData'.format(namespace)) a = gpd.read_file(kml) os.mkdir(os.path.join(os.getcwd(), 'ShapeFiles')) kml_name = os.path.basename(kml).split('.')[0] a.to_file(os.path.join(os.getcwd(), 'ShapeFiles', kml_name),driver='ESRI Shapefile') #insert your login to SentinelAPI user = '' password = '' api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') shapefiles_list = glob.glob(os.path.join(os.getcwd(), 'ShapeFiles')+ '*.shp') for shapefile in shapefiles_list: GPD_DF = gpd.read_file(shapefile) footprint = None for i in GPD_DF['geometry']: footprint = i products = api.query(footprint, #modify the date date = ('20190601', '20190626'), platformname = 'Sentinel-2', processinglevel = 'Level-2A', cloudcoverpercentage = (0,10) ) products_gdf = api.to_geodataframe(products) products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'], ascending=[True])
def test_to_geopandas(products): gdf = SentinelAPI.to_geodataframe(products) assert abs(gdf.unary_union.area - 132.16) < 0.01
class ApiSession(): """ ApiSession Class handles all connections with SentinelAPI. """ def __init__(self): self.key_file_path = '../ressources/apiKey.txt' self.export_directory = (os.path.normpath( os.path.join(os.getcwd(), os.pardir, 'downloadedImages'))) self.user, self.password = self.parsefile() self.api = SentinelAPI(self.user, self.password, query['url']) self.platform = 'Sentinel-2' def parsefile(self): """ Parses the apiKey.txt and returns a tuple containing the username and password for the SentinelAPI. """ text = '' try: text = open(self.key_file_path, 'r') except FileExistsError as exception: print('Api key file not found, must be in ressources/apiKey.txt') print('Raised Error: {}'.format(exception)) info = [] for line in text: info.append(line.strip().split(',')[1]) return (info[0], info[1]) def query(self, footprint): """ Queries the SentinelAPI and returns a geojson containing data candidates. """ return self.api.query(footprint, date=(query['begin'], query['end']), platformname=query['platform'], processinglevel=query['processing']) def to_geo_df(self, product): """ Returns GeoDataFrame """ return self.api.to_geodataframe(product) def download(self, link, directory): """ Dowloads Data to directory using link provided. """ self.api.download(link, directory_path=directory) def query_to_dataframe(self, footprint, output_path, contains=True): """ Saves the queried geopandas to a csv file so that it could be used in the future. contains: if true will only save the links that fully contain the footprint """ catalog = self.query(footprint) catalog = self.to_geo_df(catalog) desired_columns = [ 'summary', 'vegetationpercentage', 'notvegetatedpercentage', 'waterpercentage', 'unclassifiedpercentage', 'snowicepercentage', 'cloudcoverpercentage', 'geometry', 'size' ] filtered_catalog = catalog[desired_columns] if contains: contains = [ footprint.within(geometry) for geometry in catalog['geometry'] ] filtered_catalog = filtered_catalog[contains] output = filtered_catalog.to_csv() output_file = open(output_path, 'w') output_file.write(output) output_file.close()
from tensorflow import keras import numpy as np import matplotlib.pyplot as plt import IPython.display as display from PIL import Image import os import pathlib import rasterio from rasterio import plot from pyrsgis import raster user = '******' password = '******' # connect to the API api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') # search by polygon, time, and Hub query keywords footprint = geojson_to_wkt( read_geojson('/home/russell/XCity/gsonData/test.geojson')) products = api.query(footprint, date=('20190225', '20190227'), platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=(0, 20)) # download all results from the search api.download_all(products) # GeoJSON FeatureCollection containing footprints and metadata of the scenes api.to_geojson(products) # GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries api.to_geodataframe(products) print("finished")
(59.51138530046753, 24.825137916849023), (59.459087606762346, 24.907535377786523), (59.4147455486766, 24.929508034036523), (59.39832075950073, 24.844363991067773), (59.37664183245853, 24.814151588724023), (59.35249898189222, 24.75304013852871), (59.32798867805195, 24.573825660989648)] # Copernicus Hub likes coordinates in lng,lat format return Polygon([(y, x) for x, y in tln_points]) username = "******" password = "******" hub = SentinelAPI(username, password, "https://scihub.copernicus.eu/dhus") data_products = hub.query( get_tallinn_polygon(), # which area interests you date=("20200101", "20200420"), cloudcoverpercentage=(0, 10), # we don't want clouds platformname="Sentinel-2", processinglevel="Level-2A" # more processed, ready to use data ) data_products = hub.to_geodataframe(data_products) # we want to avoid downloading overlapping images, so selecting by this keyword data_products = data_products[data_products["title"].str.contains("T35VLF")] print(data_products.shape)
# convert to Pandas DataFrame products_df = api.to_dataframe(products) # sort for most recent and lowest cloud cover products_df_sorted = products_df.sort_values(['ingestiondate', 'cloudcoverpercentage'], ascending=[True, True]) test = products_df_sorted.head(1) test['cloudcoverpercentage'] test['ingestiondate'] set(products_df['processinglevel']) # GeoJSON FeatureCollection containing footprints and metadata of the scenes geojson_products = api.to_geojson(products) # GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries geodata_products = api.to_geodataframe(products) # plot product time vs cloudcover # data = [go.Scatter(x=products_df_sorted.ingestiondate, y=products_df_sorted[['cloudcoverpercentage']])] # py.plotly.iplot(data, filename = 'time-series-simple') # download api.download_all(test.index) # unzip # todo: needs loop to run through the data sets directory_to_extract_to = "unzip" zip = zipfile.ZipFile(str(test.title[0]) + '.zip') zip.extractall(directory_to_extract_to) zip.close()
class DownloadImages(object): ''' This class is composed by a set of mehtods that download the Sentinel-2 images of a given Polygon and updates a diretory with the donnwloaded files. Parameters --------- username: str personal username in Copernicus / Sentinel API password: str personal password in Copernicus / Sentinel API ''' def __init__(self, username, password): self.username = username self.password = password self.api = SentinelAPI(self.username, self.password, 'https://scihub.copernicus.eu/dhus') def get_products(self, polygon, initial_date, final_date, cloudcover=(0, 10)): ''' Overview: returns geodataframe of query products from SentinelAPI. Inputs polygon: Shape polygon of interest for api query request. inital_date: str string in the format 'YMD' of the initial request date. final_date - str string in the format 'YMD' of the final request date. cloudcover: tuple, default=(0,10) minimum cloud coverage range for api request. Output: geodataframe of query products. ''' products = self.api.query(polygon, date=(initial_date, final_date), platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=cloudcover) products = self.api.to_geodataframe(products) return products def download_folders(self, polygon, regional_poly, products, database_path, threshold=0.9): ''' Overview: downloads folders of interest that are not in the current database Inputs polygon: Shape polygon of interest for area intersection calculation. regional_poly: bool to determine how intersection should be calculated, if region or not. products: geodataframe dataframe with all products returned from AI query request. database_path: str path to the database, where folders are to be downloaded. threshold: float, default=0.9 minimum area of intersection required with polygon to donwload folder. Output returns geodataframe with products that were downloaded. ''' delete = [] for idx, row in products.iterrows(): intersect_area = polygon.intersection(row.geometry).area if regional_poly: area = intersect_area / row.geometry.area else: area = intersect_area / polygon.area if area >= threshold: if row.filename not in os.listdir(database_path): #print('downloading: ', row.filename) self.api.download(idx, directory_path=database_path) filepath = os.path.join(database_path, row.title + '.zip') with zipfile.ZipFile(filepath) as zip_ref: zip_ref.extractall(path=database_path) else: delete.append(idx) else: delete.append(idx) products = products.drop(delete) return products def update_downloaded(self, csv_path, downloaded_products): ''' Overview: updates directory with downloaded folders. Inputs csv_path: str path to csv file, filepath. downloaded_products: geodataframe gdf of all new products that were downloaded. Output N/A. ''' try: df = pd.read_csv(csv_path) df.append(downloaded_products) df.to_csv(csv_path) except: downloaded_products.to_csv(csv_path) def full_pipe(self, csv_path, polygon, intial_date, final_date, database_path, threshold=0.9, cloudcover=(0, 10)): ''' Overview: runs entire Download Images Pipeline, downloading and updating diretory with new products of interest. Inputs csv_path: str path to csv file, filepath. polygon: Shape polygon of interest for api query request and area threshold calculation to determine which folders to download. inital_date: str string in the format 'YMD' of the initial request date. final_date - str string in the format 'YMD' of the final request date. database_path: str path to the database, where folders are to be downloaded. threshold: float, default=0.9 minimum area of intersection required with polygon to donwload folder. cloudcover: tuple, default=(0,10) minimum cloud coverage range for api request. Output N/A. ''' products = self.get_products(polygon, intial_date, final_date, cloudcover) products = self.download_folders(polygon, products, database_path, threshold) self.update_downloaded(csv_path, products)