def test_small_query(): api = SentinelAPI(**_api_kwargs) api.query(**_small_query) assert api._last_query == ( '(beginPosition:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z]) ' 'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")') assert api._last_status_code == 200
def test_scihub_unresponsive(): timeout_connect = 6 timeout_read = 6.6 timeout = (timeout_connect, timeout_read) api = SentinelAPI("mock_user", "mock_password", timeout=timeout) with requests_mock.mock() as rqst: rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ConnectTimeout) with pytest.raises(requests.exceptions.ConnectTimeout): api.query(**_small_query) with pytest.raises(requests.exceptions.ConnectTimeout): api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ConnectTimeout): api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ConnectTimeout): api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b']) with requests_mock.mock() as rqst: rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ReadTimeout) with pytest.raises(requests.exceptions.ReadTimeout): api.query(**_small_query) with pytest.raises(requests.exceptions.ReadTimeout): api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ReadTimeout): api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ReadTimeout): api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
def test_SentinelAPI_wrong_credentials(): api = SentinelAPI( "wrong_user", "wrong_password" ) with pytest.raises(SentinelAPIError) as excinfo: api.query(**_small_query) assert excinfo.value.response.status_code == 401
def test_SentinelAPI_connection(): api = SentinelAPI(**_api_auth) api.query(**_small_query) assert api._last_query == ( 'beginPosition:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z] ' 'footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"') assert api._last_response.status_code == 200
def test_get_products_invalid_json(): api = SentinelAPI("mock_user", "mock_password") with requests_mock.mock() as rqst: rqst.post( 'https://scihub.copernicus.eu/apihub/search?format=json', text="{Invalid JSON response", status_code=200 ) with pytest.raises(SentinelAPIError) as excinfo: api.query( area=geojson_to_wkt(read_geojson(FIXTURES_DIR + "/map.geojson")), date=("20151219", "20151228"), platformname="Sentinel-2" ) assert excinfo.value.msg == "Invalid API response."
def test_date_arithmetic(): api = SentinelAPI(**_api_kwargs) products = api.query('ENVELOPE(0, 10, 10, 0)', ('2016-12-01T00:00:00Z-1DAY', '2016-12-01T00:00:00Z+1DAY-1HOUR')) assert api._last_response.status_code == 200 assert len(products) > 0
def test_large_query(): api = SentinelAPI(**_api_kwargs) full_products = list(api.query(**_large_query)) assert api._last_query == ( 'beginPosition:[2015-12-01T00:00:00Z TO 2015-12-31T00:00:00Z] ' 'footprint:"Intersects(POLYGON((0 0,0 10,10 10,10 0,0 0)))"') assert api._last_response.status_code == 200 assert len(full_products) > api.page_size result = list(api.query(limit=150, **_large_query)) assert result == full_products[:150] result = list(api.query(limit=20, offset=90, **_large_query)) assert result == full_products[90:110] result = list(api.query(limit=20, offset=len(full_products) - 10, **_large_query)) assert result == full_products[-10:]
def products(): """A fixture for tests that need some non-specific set of products as input.""" api = SentinelAPI(**_api_auth) products = api.query( geojson_to_wkt(read_geojson('tests/map.geojson')), "20151219", "20151228" ) return products
def test_large_query(): api = SentinelAPI(**_api_kwargs) products = api.query(**_large_query) assert api._last_query == ( '(beginPosition:[2015-12-01T00:00:00Z TO 2015-12-31T00:00:00Z]) ' 'AND (footprint:"Intersects(POLYGON((0 0,0 10,10 10,10 0,0 0)))")') assert api._last_status_code == 200 assert len(products) > api.page_size
def products(api_kwargs, vcr, test_wkt): """A fixture for tests that need some non-specific set of products as input.""" with vcr.use_cassette('products_fixture', decode_compressed_response=False): api = SentinelAPI(**api_kwargs) products = api.query( test_wkt, ("20151219", "20151228") ) assert len(products) > 20 return products
def test_SentinelAPI_wrong_credentials(): api = SentinelAPI( "wrong_user", "wrong_password" ) with pytest.raises(SentinelAPIError) as excinfo: api.query(**_small_query) assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b']) assert excinfo.value.response.status_code == 401
def test_area_relation(): api = SentinelAPI(**_api_auth) params = dict( area="POLYGON((10.83 53.04,11.64 53.04,11.64 52.65,10.83 52.65,10.83 53.04))", date=("20151219", "20151226") ) result = api.query(**params) n_intersects = len(result) assert n_intersects > 10 result = api.query(area_relation="contains", **params) n_contains = len(result) assert 0 < n_contains < n_intersects result = api.query(area_relation="IsWithin", **params) n_iswithin = len(result) assert n_iswithin == 0 # Check that unsupported relations raise an error with pytest.raises(ValueError) as excinfo: api.query(area_relation="disjoint", **params)
def test_get_products_size(products): assert SentinelAPI.get_products_size(products) == 90.94 # load a new very small query api = SentinelAPI(**_api_auth) with my_vcr.use_cassette('test_get_products_size'): products = api.query( raw="S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E") assert len(products) > 0 # Rounded to zero assert SentinelAPI.get_products_size(products) == 0
def test_order_by(): api = SentinelAPI(**_api_auth) products = api.query( geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')), ("20151219", "20151228"), platformname="Sentinel-2", cloudcoverpercentage=(0, 10), order_by="cloudcoverpercentage, -beginposition" ) assert len(products) == 3 vals = [x["cloudcoverpercentage"] for x in products.values()] assert sorted(vals) == vals
def test_s2_cloudcover(): api = SentinelAPI(**_api_auth) products = api.query( geojson_to_wkt(read_geojson('tests/map.geojson')), "20151219", "20151228", platformname="Sentinel-2", cloudcoverpercentage="[0 TO 10]" ) assert len(products) == 3 product_ids = list(products) assert product_ids[0] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef" assert product_ids[1] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e" assert product_ids[2] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_footprints_s1(): api = SentinelAPI(**_api_auth) products = api.query( geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')), (datetime(2014, 10, 10), datetime(2014, 12, 31)), producttype="GRD" ) footprints = api.to_geojson(products) for footprint in footprints['features']: assert not footprint['geometry'].errors() with open(FIXTURES_DIR + '/expected_search_footprints_s1.geojson') as geojson_file: expected_footprints = geojson.loads(geojson_file.read()) # to compare unordered lists (JSON objects) they need to be sorted or changed to sets assert set(footprints) == set(expected_footprints)
def test_too_long_query(): api = SentinelAPI(**_api_kwargs) # Test whether our limit calculation is reasonably correct and # that a relevant error message is provided def create_query(n): return api.format_query(date=("NOW", "NOW"), raw=" abc_:*.+*~!," * n) # Expect no error q = create_query(170) assert 0.99 < SentinelAPI.check_query_length(q) < 1.0 with pytest.raises(SentinelAPIError) as excinfo: api.query(raw=q) assert "Invalid query string" in excinfo.value.msg # Expect HTTP status 500 Internal Server Error q = create_query(171) assert 1.0 <= SentinelAPI.check_query_length(q) < 1.01 with pytest.raises(SentinelAPIError) as excinfo: api.query(raw=q) assert excinfo.value.response.status_code == 500 assert ("Request Entity Too Large" in excinfo.value.msg or "Request-URI Too Long" in excinfo.value.msg)
def test_footprints_s1(): api = SentinelAPI(**_api_auth) products = api.query( geojson_to_wkt(read_geojson('tests/map.geojson')), datetime(2014, 10, 10), datetime(2014, 12, 31), producttype="GRD" ) footprints = api.to_geojson(products) for footprint in footprints['features']: validation = geojson.is_valid(footprint['geometry']) assert validation['valid'] == 'yes', validation['message'] with open('tests/expected_search_footprints_s1.geojson') as geojson_file: expected_footprints = geojson.loads(geojson_file.read()) # to compare unordered lists (JSON objects) they need to be sorted or changed to sets assert set(footprints) == set(expected_footprints)
def test_s2_cloudcover(): api = SentinelAPI(**_api_auth) products = api.query( geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')), ("20151219", "20151228"), platformname="Sentinel-2", cloudcoverpercentage=(0, 10) ) assert len(products) == 3 product_ids = list(products) assert product_ids[0] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef" assert product_ids[1] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e" assert product_ids[2] == "0848f6b8-5730-4759-850e-fc9945d42296" # For order-by test vals = [x["cloudcoverpercentage"] for x in products.values()] assert sorted(vals) != vals
#!/usr/bin/env python import time import datetime from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt start_date = '20160101' # hardcoded value current_date = datetime.date.today().strftime("%Y%m%d") #current_date = datetime.date.today().strftime("%Y%m%d") print "loading!" api = SentinelAPI('mpc', 'q11g33h99', 'https://scihub.copernicus.eu/dhus') footprint = geojson_to_wkt(read_geojson('map.geojson')) products = api.query(footprint, date=(start_date, current_date), platformname='Sentinel-2', producttype='S2MSI1C', cloudcoverpercentage=(0, 15)) api.download_all(products) print "DONE!!!"
def products(): """A fixture for tests that need some non-specific set of products as input.""" api = SentinelAPI(**_api_auth) products = api.query(geojson_to_wkt(read_geojson('tests/map.geojson')), "20151219", "20151228") return products
# loop through features within the same file for j, geometry in enumerate(read_geojson(path)["features"]): # converts to well-known text footprint = geojson_to_wkt(geometry) # determine if searching by dates or by seasons if dates is not None and seasons is None: # find products within dates and suitable cloud cover products = api.query( footprint, date=('{}{}{}'.format(dates['year'][0], dates['month'][0], dates['day'][0]), '{}{}{}'.format(dates['year'][1], dates['month'][1], dates['day'][1])), platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=(0, cloud_cover_percentage)) # print error message if there are no images which fit the requirements if len(list(products)) == 0: print( "There are no Sentinel-2 images for this AOI taken within the specified dates" ) break # convert from dictionary to Pandas dataframe products_df = api.to_dataframe(products)
#==================================================================== # Retrive latest sentinel 2 #==================================================================== #https://www.evernote.com/Home.action#n=e77ce355-1b1e-4a89-896b-4036f905dfea&ses=1&sh=5&sds=5&x=sentinel& #This CLI works: sentinelsat -u jfiddes -p sT0kkang -g extent.json -s 20151201 -e 20151207 --sentinel 2 --cloud 90 -d #This API doesnt work yet (input posiitions from getExtent.py: from geojson import Polygon gj = Polygon([[(float(lonW), float(latN)), (float(lonW), float(latS)) , (float(lonE),float(latS)), (float(lonE), float(latN)), (float(lonW), float(latN))]]) {"coordinates": [[[lonW, latN], [lonW, latN], [lonE,latS], [lonE, latN] ]], "type": "Polygon"} import geojson gj.is_valid from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt from datetime import date api = SentinelAPI('jfiddes', 'sT0kkang', 'https://scihub.copernicus.eu/dhus') footprint = gj products = api.query(footprint, beginposition = '[20160101 TO 20160103]', platformname = 'Sentinel-2', cloudcoverpercentage = '[0 TO 30]')
def sent1_query(user, passwd, geojsonfile, start_date, end_date, output_folder=None, api=True): """ A convenience function that wraps sentinelsat query and download Notes ----------- I have found the sentinesat sometimes fails to download the second image, so I have written some code to avoid this - choose api = False for this Parameters ----------- user : string username for esa hub passwd : string password for hub geojsonfile : string AOI polygon of interest start_date : string date of beginning of search end_date : string date of end of search output_folder : string where you intend to download the imagery """ #TODO: Check if SentinelAPI will use TokenAuth instead of hard-coded cred strings api = SentinelAPI(user, passwd) if oldsat is True: footprint = get_coordinates(geojsonfile) else: footprint = geojson_to_wkt(read_geojson(geojsonfile)) products = api.query(footprint, ((start_date, end_date)), platformname="Sentinel-1", producttype="GRD", polarisationmode="VV, VH") products_df = api.to_dataframe(products) if api is True and output_folder != None: api.download_all(directory_path=output_folder) else: prods = np.arange(len(products)) #TODO: investigate flaky sentinelAPI # the api was proving flaky whereas the cmd line always works hence this # is alternate the download option if output_folder != None: # procList = [] for prod in prods: #os.chdir(output_folder) sceneID = products[prod]['id'] cmd = [ 'sentinel', 'download', '-p', output_folder, user, passwd, sceneID ] print(sceneID + ' downloading') subprocess.call(cmd) return products_df, products
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt from datetime import date api = SentinelAPI(user='******', password='******', api_url='https://s5phub.copernicus.eu/dhus') footprint = geojson_to_wkt( read_geojson( '~/Dropbox/daniel/sentinel_5_colombia/maps/colombia_area.geojson')) products = api.query(area=footprint, date="[20191231 TO NOW]", platformname='Sentinel-5p') # downloading products api.download_all( products, directory_path= "~/Dropbox/daniel/sentinel_5_colombia/products/202001_20200313")
class download_sentinel(Dataset_general): def __init__(self, username, password): super().__init__() self.username = username self.password = password def sentinel_2(self,download_type='ROI_polygon', roi_polygon=None, startdate=None, enddate=None, cloudcover_max=5): params = {'download_type':download_type,'roi_polygon': roi_polygon, 'startdate': startdate, 'enddate': enddate,'platformname': 'Sentinel-2'} return self.fetch_datasets(**params) def sentinel_1(self,download_type='ROI_polygon', roi_polygon=None, startdate=None, enddate=None): params = {'download_type':download_type,'roi_polygon': roi_polygon, 'startdate': startdate, 'enddate': enddate,'platformname': 'Sentinel-1'} return self.fetch_datasets(**params) def fetch_datasets(self, download_type='ROI_polygon', roi_polygon=None, startdate=None, enddate=None, cloudcover_max=5, platformname='Sentinel-2'): """ :param download_type: :param username: :param password: :param roi_polygon: :param startdate: :param enddate: :param cloudcover_max: :param platformname: :return: """ if startdate: self.startdate = startdate if enddate: self.enddate = enddate if roi_polygon: self.roi_polygon = roi_polygon self.api = SentinelAPI(self.username, self.password, 'https://scihub.copernicus.eu/dhus') product_id = None if download_type == 'full': if product_id: self.api.download(product_id) else: print('product id required') if download_type == 'ROI_polygon': if roi_polygon.split('.')[-1] == 'geojson': file_obj = open(self.roi_polygon, "r") json_data = file_obj.read() file_obj.close() json_data = json_data.encode().decode('utf-8-sig') # Remove utf-8 data if any present in the file json_data = json.loads(json_data) footprint = geojson_to_wkt(json_data) if platformname == 'Sentinel-2': self.products = self.api.query(footprint, date=(self.startdate, self.enddate), platformname=platformname, cloudcoverpercentage=(0, cloudcover_max)) self.list_products = list(self.products.items()) elif platformname == 'Sentinel-1': self.products = self.api.query(footprint, date=(self.startdate, self.enddate), platformname=platformname) self.list_products = list(self.products.items()) print(len(self.list_products), ' products found') def download_files(self, list_product_ids,directory_path='.',unzip=True): for product_id in list_product_ids: self.api.download(product_id, directory_path=directory_path) if unzip: print('extracting files') file_names =glob(os.path.join(directory_path)+'/S*.zip') for filename in file_names: with zipfile.ZipFile(filename, 'r') as zip_ref: zip_ref.extractall(directory_path) """ add function to display product AOI Polygon"""
def run(self): print("Running " + str(self.data_end) + str(self.data_start)) api = SentinelAPI('biancasantana', '988245535', 'https://scihub.copernicus.eu/dhus', show_progressbars=True) footprint = geojson_to_wkt(read_geojson('Ufrj.geojson')) p = 'Sentinel-2' products = api.query(footprint, date=(str(self.data_start), str(data_end)), platformname=p, cloudcoverpercentage='[0 TO 100]') print(type(products)) products_df = api.to_dataframe(products) ids = products_df.index #correct abaixo ru = str(ids) io = (ru.replace("Index([", "")) reti_col = (io.replace("],", "")) ids_pasta = reti_col.replace("dtype='object')", "") b = "''" for i in range(0, len(b)): pro = ids_pasta.replace(b[i], "") b = "," for i in range(0, len(b)): pio = pro.replace(b[i], "") prin = pio.split() with open('sentinel_ids_download.csv', mode='w', encoding='utf-8', newline='') as csv_file: fieldnames = ["id"] writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer.writeheader() for idi in prin: writer.writerow({"id": idi}) #correct acima products_df = api.to_dataframe(products) title = (products_df.set_index('title')) ids_txt = (title.index) ru = str(ids_txt) int = (ru.replace("Index([", "")) ret_para = (int.replace("dtype='object', name='title'", "")) reti_col = (ret_para.replace("],", "")) ids_pasta = reti_col.replace(")", "") b = "''" for i in range(0, len(b)): pro = ids_pasta.replace(b[i], "") print(pro) b = "," for i in range(0, len(b)): pio = pro.replace(b[i], "") prin = pio.split() with open('sentinel_certo.csv', mode='w', encoding='utf-8', newline='') as csv_file: fieldnames = ["id"] writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer.writeheader() for idi in prin: writer.writerow({"id": idi}) #correct acim return products
args = parser.parse_args() api = SentinelAPI(args.user, args.password, 'https://scihub.copernicus.eu/dhus') shapeFile = gpd.read_file(args.pathShapeFile) footprint = None for i in shapeFile['geometry']: footprint = i """ En el caso de no funcionar con shp file utilizar un gejson de la misma region """ #footprint = geojson_to_wkt(read_geojson(pathGeojson)) products = api.query(footprint, date = ('20170702','20190920'), platformname = 'Sentinel-2', processinglevel = 'Level-2A', cloudcoverpercentage = (0,10) ) products_gdf = api.to_geodataframe(products) products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'], ascending=[True]) shapeFile_wgt = shapeFile.to_crs(epsg=4326) number_download = 0 max_downloads = 4 dict_percentage = {} for i in range(len(products_gdf_sorted)): actual_polygon = products_gdf_sorted[i:i+1]
[-122.74628, 49.005211], [-122.74628, 49.362603], [-123.293536, 49.362603]] m = Polygon([AREA]) object_name = "Vancouver" with open(object_name + ".geojson", 'w') as f: json.dump(m, f) footprint_geojson = geojson_to_wkt(read_geojson(object_name + ".geojson")) user = input("USER NAME: ") password = getpass("PASSWORD: "******"https://scihub.copernicus.eu/dhus") m = folium.Map([(AREA[0][1] + AREA[len(AREA) - 1][1]) / 2, (AREA[0][0] + AREA[len(AREA) - 1][0]) / 2], zoom_start=10) folium.GeoJson(object_name + '.geojson').add_to(m) m.save(outfile="datamap.html") # os.system('open datamap.html') products = api.query(footprint_geojson, date=('20201201', '20201221'), platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=(0, 100)) products_gdf = api.to_geodataframe(products) products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'], ascending=[True]) products_gdf_sorted
# connect to the API from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt from datetime import date user = QInputDialog().getText(None, "User", "User please:") password = QInputDialog().getText(None, "Password", "Password please:") api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') point = QgsPointXY(-3.70256, 40.4165) geom = QgsGeometry.fromPointXY(point).asWkt() products = api.query(geom, platformname = 'Sentinel-2', cloudcoverpercentage = (0, 25),producttype='S2MSI1C',beginPosition='[2020-01-20T00:00:00.000Z TO 2020-02-01T23:59:59.999Z]') json = api.to_geojson(products) for i in json["features"]: summary = i['properties']['summary'] print(summary) uuid = i['properties']['uuid'] product = api.query(uuid=uuid) api.download_all(product, directory_path='C:\cursos')
def test_SentinelAPI_wrong_credentials(): api = SentinelAPI("wrong_user", "wrong_password") with pytest.raises(SentinelAPIError) as excinfo: api.query(**_small_query) assert excinfo.value.response.status_code == 401
class DownloadImages(object): ''' This class is composed by a set of mehtods that download the Sentinel-2 images of a given Polygon and updates a diretory with the donnwloaded files. Parameters --------- username: str personal username in Copernicus / Sentinel API password: str personal password in Copernicus / Sentinel API ''' def __init__(self, username, password): self.username = username self.password = password self.api = SentinelAPI(self.username, self.password, 'https://scihub.copernicus.eu/dhus') def get_products(self, polygon, initial_date, final_date, cloudcover=(0, 10)): ''' Overview: returns geodataframe of query products from SentinelAPI. Inputs polygon: Shape polygon of interest for api query request. inital_date: str string in the format 'YMD' of the initial request date. final_date - str string in the format 'YMD' of the final request date. cloudcover: tuple, default=(0,10) minimum cloud coverage range for api request. Output: geodataframe of query products. ''' products = self.api.query(polygon, date=(initial_date, final_date), platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=cloudcover) products = self.api.to_geodataframe(products) return products def download_folders(self, polygon, regional_poly, products, database_path, threshold=0.9): ''' Overview: downloads folders of interest that are not in the current database Inputs polygon: Shape polygon of interest for area intersection calculation. regional_poly: bool to determine how intersection should be calculated, if region or not. products: geodataframe dataframe with all products returned from AI query request. database_path: str path to the database, where folders are to be downloaded. threshold: float, default=0.9 minimum area of intersection required with polygon to donwload folder. Output returns geodataframe with products that were downloaded. ''' delete = [] for idx, row in products.iterrows(): intersect_area = polygon.intersection(row.geometry).area if regional_poly: area = intersect_area / row.geometry.area else: area = intersect_area / polygon.area if area >= threshold: if row.filename not in os.listdir(database_path): #print('downloading: ', row.filename) self.api.download(idx, directory_path=database_path) filepath = os.path.join(database_path, row.title + '.zip') with zipfile.ZipFile(filepath) as zip_ref: zip_ref.extractall(path=database_path) else: delete.append(idx) else: delete.append(idx) products = products.drop(delete) return products def update_downloaded(self, csv_path, downloaded_products): ''' Overview: updates directory with downloaded folders. Inputs csv_path: str path to csv file, filepath. downloaded_products: geodataframe gdf of all new products that were downloaded. Output N/A. ''' try: df = pd.read_csv(csv_path) df.append(downloaded_products) df.to_csv(csv_path) except: downloaded_products.to_csv(csv_path) def full_pipe(self, csv_path, polygon, intial_date, final_date, database_path, threshold=0.9, cloudcover=(0, 10)): ''' Overview: runs entire Download Images Pipeline, downloading and updating diretory with new products of interest. Inputs csv_path: str path to csv file, filepath. polygon: Shape polygon of interest for api query request and area threshold calculation to determine which folders to download. inital_date: str string in the format 'YMD' of the initial request date. final_date - str string in the format 'YMD' of the final request date. database_path: str path to the database, where folders are to be downloaded. threshold: float, default=0.9 minimum area of intersection required with polygon to donwload folder. cloudcover: tuple, default=(0,10) minimum cloud coverage range for api request. Output N/A. ''' products = self.get_products(polygon, intial_date, final_date, cloudcover) products = self.download_folders(polygon, products, database_path, threshold) self.update_downloaded(csv_path, products)
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt # ------------------------------------------- # necessary information: user = '******' password = '******' # YYYYMMDD start_date = '20150101' end_date = '20180207' # map.geojson with boundary coordinates # just generate and save as "map.geojson" using: --- http://geojson.io --- geojson_path = 'directory\\to\\the\\file\\map.geojson' # where to save the data save_path = 'directory\\to\\the\\save_folder' # ------------------------------------------- # connect to the API / SentinelHub api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus', show_progressbars=True) footprint = geojson_to_wkt(read_geojson(geojson_path)) products = api.query(footprint,date=(start_date, end_date), platformname='Sentinel-2', producttype='S2MSI2Ap') print 'Number of images: {}'.format(len(products)) api.download_all(products, save_path)
# connect to the API from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt from datetime import date api = SentinelAPI('kongstad', 'Delphini1', 'https://scihub.copernicus.eu/dhus') # search by polygon, time, and SciHub query keywords footprint = geojson_to_wkt(read_geojson('geojson/scoresbysund_small.geojson')) products = api.query(footprint, date=('20170826', date(2017, 8, 30)), platformname='Sentinel-2', cloudcoverpercentage=(0, 5)) api.to_geojson(products) # download all results from the search #api.download_all(products,'sentinel2images/')
# Set up API api = SentinelAPI(user='******', password='******', api_url='https://s5phub.copernicus.eu/dhus') # Define area of interest in WKT format AOI = '' # Date startdate = '20200101' enddate = '20200430' # Download list of Sentinel S5-P NO2 products in region of interest products = api.query(AOI, date=(startdate,enddate), platformname='Sentinel-5', producttype='L2__NO2___', processingmode='Offline', # 'Near real time' or 'Offline' ) # Convert to pandas dataframe for ease of use products_df = api.to_dataframe(products) # Convert AOI to shapely file AOIshape = wkt.loads(AOI) # Create empty list of overlaping geometries differences = [] # Check which images don't have complete overlap with AOI for image in range(len(products_df)):
def sent2_query(user, passwd, geojsonfile, start_date, end_date, cloud='100', output_folder=None, api=True): """ A convenience function that wraps sentinelsat query & download Notes ----------- I have found the sentinesat sometimes fails to download the second image, so I have written some code to avoid this - choose api = False for this Parameters ----------- user : string username for esa hub passwd : string password for hub geojsonfile : string AOI polygon of interest start_date : string date of beginning of search end_date : string date of end of search output_folder : string where you intend to download the imagery cloud : string (optional) include a cloud filter in the search """ ##set up your copernicus username and password details, and copernicus download site... BE CAREFUL if you share this script with others though! api = SentinelAPI(user, passwd) # NOWT WRONG WITH API - # TODO Maybe improve check of library so it doesn't use a global # if oldsat is True: # footprint = get_coordinates(geojsonfile) # else: footprint = geojson_to_wkt(read_geojson(geojsonfile)) products = api.query(footprint, ((start_date, end_date)), platformname="Sentinel-2", cloudcoverpercentage="[0 TO " + cloud + "]") #,producttype="GRD") products_df = api.to_dataframe(products) if api is True and output_folder != None: api.download_all(directory_path=output_folder) else: prods = np.arange(len(products)) # the api was proving flaky whereas the cmd line always works hence this # is alternate the download option if output_folder != None: # procList = [] for prod in prods: #os.chdir(output_folder) sceneID = products[prod]['id'] cmd = [ 'sentinel', 'download', '-p', output_folder, user, passwd, sceneID ] print(sceneID + ' downloading') subprocess.call(cmd) #[p.wait() for p in procList] return products_df, products
api = SentinelAPI('ahui0911', '19940911', 'https://scihub.copernicus.eu/dhus') footprint = geojson_to_wkt( read_geojson( r'F:\snappy_InSAR_code\GeoJOSN\stockholm_DSC_dw_scene.geojson')) # products = api.query(footprint, date=('20170501', '20171031'), platformname = 'Sentinel-1', # producttype='SLC', relativeorbitnumber = 102, orbitdirection="DESCENDING") # Launch Date -> S1A: 2014-04-03, S1B: 2016-04-25 ### DSC rorb = 22 products = api.query(footprint, date=('20150501', '20151031'), platformname='Sentinel-1', producttype='SLC', relativeorbitnumber=22, orbitdirection="DESCENDING", order_by='+beginposition', filename='S1A*') ### ASC rorb = 102 # products = api.query(footprint, date=('20170501', '20171031'), platformname = 'Sentinel-1', # producttype='SLC', relativeorbitnumber = 102, orbitdirection="ASCENDING", order_by='+beginposition') print("Total Number of Searched Products:" + str(len(products.keys()))) # api.download_all(products, savePath) if not os.path.exists(savePath): os.mkdir(savePath) ### If a product doesn't exist, then download it.
def sent2_amazon(user, passwd, geojsonfile, start_date, end_date, output_folder, tile=None, cloud='100'): """ Query the ESA catalogue then download S2 from AWS with correct renaming of stuff Uses joblib to parallelise multiple files from aws Way quicker than ESA-based download Notes: ------------------------ Credit to sentinelsat for the query aspect of this function, and sentinelhub for the AWS aspect. Parameters ---------- user : string username for esa hub passwd : string password for hub geojsonfile : string AOI polygon of interest start_date : string date of beginning of search end_date : string date of end of search output_folder : string where you intend to download the imagery tile : string S2 tile cloud : string (optional) include a cloud filter in the search """ # Examples of sentinehub usage: #download_safe_format('S2A_OPER_PRD_MSIL1C_PDMC_20160121T043931_R069_V20160103T171947_20160103T171947') #download_safe_format('S2A_MSIL1C_20170414T003551_N0204_R016_T54HVH_20170414T003551') #download_safe_format(tile=('T38TML','2015-12-19'), entire_product=True) #entire prod really mean whole tile in old format! Avoid! #download_safe_format(tile=('T54HVH','2017-04-14')) # Use sentinel sat to query api = SentinelAPI(user, passwd) # if oldsat is True: # footprint = get_coordinates(geojsonfile) # else: footprint = geojson_to_wkt(read_geojson(geojsonfile)) products = api.query(footprint, ((start_date, end_date)), platformname="Sentinel-2", cloudcoverpercentage="[0 TO " + cloud + "]") #,producttype="GRD") products_df = api.to_dataframe(products) # If using an aoi shape this is the option to follow at present until I # write a native function if tile is None: Parallel(n_jobs=-1, verbose=2)( delayed(download_safe_format)(i, folder=output_folder) for i in products_df.identifier) # If the tile id is known then use this - likely handy for oldfmt else: # A kludge for now until I spend more than two mins writing this func dateList = [] for prod in products_df['ingestiondate']: date1 = prod.strftime('%Y-%m-%d') dateList.append(date1) Parallel(n_jobs=-1, verbose=2)( delayed(download_safe_format)(tile=(tile, i), folder=output_folder) for i in dateList) return products_df, products
def test_invalid_query(): api = SentinelAPI(**_api_auth) with pytest.raises(SentinelAPIError): api.query(raw="xxx:yyy")
def _handler(self, request, response): response.update_status("start fetching resource", 10) init_process_logger('log.txt') response.outputs['output_log'].file = 'log.txt' # products = [inpt.data for inpt in request.inputs['indices']] indice = request.inputs['indices'][0].data bbox = [] # order xmin ymin xmax ymax bboxStr = request.inputs['BBox'][0].data bboxStr = bboxStr.split(',') bbox.append(float(bboxStr[0])) bbox.append(float(bboxStr[2])) bbox.append(float(bboxStr[1])) bbox.append(float(bboxStr[3])) if 'end' in request.inputs: end = request.inputs['end'][0].data end = dt.combine(end, time(23, 59, 59)) else: end = dt.now() if 'start' in request.inputs: start = request.inputs['start'][0].data start = dt.combine(start, time(0, 0, 0)) else: start = end - timedelta(days=30) if (start > end): start = dt.now() - timedelta(days=30) end = dt.now() LOGGER.exception( "periode end befor periode start, period is set to the last 30 days from now" ) username = request.inputs['username'][0].data password = request.inputs['password'][0].data cloud_cover = request.inputs['cloud_cover'][0].data api = SentinelAPI(username, password) geom = { "type": "Polygon", "coordinates": [[[bbox[0], bbox[1]], [bbox[2], bbox[1]], [bbox[2], bbox[3]], [bbox[0], bbox[3]], [bbox[0], bbox[1]]]] } footprint = geojson_to_wkt(geom) response.update_status("start searching tiles acording query", 15) products = api.query( footprint, date=(start, end), platformname='Sentinel-2', cloudcoverpercentage=(0, cloud_cover), # producttype='SLC', # orbitdirection='ASCENDING', ) LOGGER.debug('%s products found' % len(products.keys())) DIR_cache = cache_path() DIR_EO = join(DIR_cache, 'scihub.copernicus') if not exists(DIR_EO): makedirs(DIR_EO) # api.download_all(products) # try: # with open(filepathes, 'w') as fp: # fp.write('############################################\n') # fp.write('### Following files are fetched ###\n') # fp.write('############################################\n') # fp.write('\n') resources = [] for key in products.keys(): try: filename = products[key]['filename'] # form = products[key]['format'] ID = str(products[key]['identifier']) file_zip = join(DIR_EO, '%s.zip' % (ID)) DIR_tile = join(DIR_EO, '%s' % (filename)) response.update_status("fetch file %s" % ID, 20) LOGGER.debug('path: %s' % DIR_tile) if exists(file_zip): LOGGER.debug('file %s.zip already fetched' % ID) else: try: api.download(key, directory_path=DIR_EO) response.update_status( "***%s sucessfully fetched" % ID, 20) LOGGER.debug('Tile %s fetched' % ID) LOGGER.debug('Files %s fetched ' % ID) except: LOGGER.exception('failed to extract file %s' % filename) if exists(DIR_tile): LOGGER.debug('file %s already unzipped' % filename) else: try: # zipfile = join(DIR_EO, '%szip' % (filename)).strip(form) zip_ref = zipfile.ZipFile(file_zip, 'r') zip_ref.extractall(DIR_EO) zip_ref.close() LOGGER.debug('Tile %s unzipped' % ID) except: LOGGER.exception('failed to extract %s ' % file_zip) resources.append(DIR_tile) except: LOGGER.exception('failed to fetch %s' % key) size = float(products[key]['size'].split(' ')[0]) producttype = products[key]['producttype'] beginposition = str(products[key]['beginposition']) imgs = [] tiles = [] for resource in resources: try: response.update_status("Calculating %s indices " % (indice), 40) if indice == 'NDVI': LOGGER.debug('Calculate NDVI for %s', resource) tile = eodata.get_ndvi(resource) LOGGER.debug('resources BAI calculated') if indice == 'BAI': LOGGER.debug('Calculate BAI for %s', resource) tile = eodata.get_bai(resource) LOGGER.debug('resources BAI calculated') tiles.append(tile) except: LOGGER.exception('failed to calculate indice for %s ' % resource) for tile in tiles: try: LOGGER.debug("Plot tile %s" % tile) img = eodata.plot_band(tile, file_extension='PNG', colorscheem=indice) imgs.append(img) except: LOGGER.exception("Failed de plot tile %s " % tile) from flyingpigeon.utils import archive tarf = archive(imgs) response.outputs['output_archive'].file = tarf i = next((i for i, x in enumerate(imgs) if x), None) if i is None: i = "dummy.png" response.outputs['output_plot'].file = imgs[i] # from flyingpigeon import visualisation as vs # # images = vs.concat_images(imgs, orientation='v') response.update_status("done", 100) return response
class ApiSession(): """ ApiSession Class handles all connections with SentinelAPI. """ def __init__(self): self.key_file_path = '../ressources/apiKey.txt' self.export_directory = (os.path.normpath( os.path.join(os.getcwd(), os.pardir, 'downloadedImages'))) self.user, self.password = self.parsefile() self.api = SentinelAPI(self.user, self.password, query['url']) self.platform = 'Sentinel-2' def parsefile(self): """ Parses the apiKey.txt and returns a tuple containing the username and password for the SentinelAPI. """ text = '' try: text = open(self.key_file_path, 'r') except FileExistsError as exception: print('Api key file not found, must be in ressources/apiKey.txt') print('Raised Error: {}'.format(exception)) info = [] for line in text: info.append(line.strip().split(',')[1]) return (info[0], info[1]) def query(self, footprint): """ Queries the SentinelAPI and returns a geojson containing data candidates. """ return self.api.query(footprint, date=(query['begin'], query['end']), platformname=query['platform'], processinglevel=query['processing']) def to_geo_df(self, product): """ Returns GeoDataFrame """ return self.api.to_geodataframe(product) def download(self, link, directory): """ Dowloads Data to directory using link provided. """ self.api.download(link, directory_path=directory) def query_to_dataframe(self, footprint, output_path, contains=True): """ Saves the queried geopandas to a csv file so that it could be used in the future. contains: if true will only save the links that fully contain the footprint """ catalog = self.query(footprint) catalog = self.to_geo_df(catalog) desired_columns = [ 'summary', 'vegetationpercentage', 'notvegetatedpercentage', 'waterpercentage', 'unclassifiedpercentage', 'snowicepercentage', 'cloudcoverpercentage', 'geometry', 'size' ] filtered_catalog = catalog[desired_columns] if contains: contains = [ footprint.within(geometry) for geometry in catalog['geometry'] ] filtered_catalog = filtered_catalog[contains] output = filtered_catalog.to_csv() output_file = open(output_path, 'w') output_file.write(output) output_file.close()
import zipfile from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt import cfg import glob api = SentinelAPI(cfg.copernicus["user"], cfg.copernicus["password"], 'https://scihub.copernicus.eu/dhus') outdir = '/download_storage/S2_L2A_MSI_ARD/BoyacaImages/' footprint = geojson_to_wkt(read_geojson('/origin/S2_L2A_MSI_ARD/Search Polygons/BoyacaCentral.geojson')) products = api.query(footprint, date=('20200919', '20201109'), platformname='Sentinel-2', producttype= 'S2MSI2A', cloudcoverpercentage=(0, 90)) """ date (tuple of (str or datetime) or str, optional) -- A time interval filter based on the Sensing Start Time of the products. Expects a tuple of (start, end), e.g. (“NOW-1DAY”, “NOW”). The timestamps can be either a Python datetime or a string in one of the following formats: yyyyMMdd yyyy-MM-ddThh:mm:ss.SSSZ (ISO-8601) yyyy-MM-ddThh:mm:ssZ NOW NOW-<n>DAY(S) (or HOUR(S), MONTH(S), etc.) NOW+<n>DAY(S) yyyy-MM-ddThh:mm:ssZ-<n>DAY(S) NOW/DAY (or HOUR, MONTH etc.) - rounds the value to the given unit Alternatively, an already fully formatted string such as “[NOW-1DAY TO NOW]” can be used as well.
def test_invalid_query(): api = SentinelAPI(**_api_auth) with pytest.raises(SentinelAPIError) as excinfo: api.query(raw="xxx:yyy")
import os from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt mypath = r'C:\Users\mccoy\PycharmProjects\untitled\gz_2010_us_outline_500k.json' # products = api.query(footprint, # producttype='SLC', # orbitdirection='ASCENDING') # api.download_all(products) testAPI = SentinelAPI(user='******', password='******', api_url='https://s5phub.copernicus.eu/dhus') footprint = geojson_to_wkt(read_geojson(mypath)) #products = testAPI.query(area = footprint, date = "[NOW-20DAYS TO NOW]", platformname='Sentinel-5p') #Pollution map for the US products = testAPI.query(area = footprint, date = "[NOW-1DAYS TO NOW]",producttype='L2__NO2___') # download all results from the search mypath = testAPI.download_all(products) downloadedfile = mypath[0][next(iter(mypath[0]))]['path'] dir_path = os.path.dirname(os.path.realpath(__file__)) downloadedfile_full = dir_path + downloadedfile # GeoJSON FeatureCollection containing footprints and metadata of the scenes testAPI.to_geojson(products) # GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries api.to_geodataframe(products) #Get all data for the whole world #products = testAPI.query(date = "[NOW-1DAYS TO NOW]")
def query_sentinel_data(cfg, save_json=True): workpath = Path(os.getcwd()) # api = SentinelAPI('puzhao', 'kth10044ESA!', 'https://scihub.copernicus.eu/dhus') # user, password = "******", "19940911" api = SentinelAPI(cfg.user, cfg.password, 'https://scihub.copernicus.eu/dhus') now = datetime.now().strftime("%Y-%m-%dT%H%M%S") today = datetime.today().strftime("%Y-%m-%d") if cfg.start_date is None: cfg.start_date = (datetime.today() + timedelta(-1)).strftime("%Y-%m-%d") if cfg.end_date is None: cfg.end_date = (datetime.today() + timedelta(1)).strftime("%Y-%m-%d") print("now: ", now) cfg.update({ "roi_url": cfg.roi_url, 'placename': "British Columbia", "query_by": "roi", # 'place' has problem "query_date": today, "start_date": cfg.start_date, "end_date": cfg.end_date, "platformname": cfg.platformname, # Sentinel-2 "producttype": cfg.producttype, # S2MSI1C, S2MSI2A # 'relativeorbitnumber': 84, # "orbitdirection": "ASCENDING", "download_flag": cfg.download_flag, "download_one": True, # download one by one "download_all": True, # download all once }) pprint(cfg) Sat_Abb_Dict = {'Sentinel-1': 'S1', 'Sentinel-2': 'S2', 'Sentinel-3': 'S3'} SAT = Sat_Abb_Dict[cfg.platformname] datafolder = Path(cfg.datafolder) savePath = datafolder / "data" / f"{SAT}_{cfg.producttype}" if not os.path.exists(savePath): os.makedirs(savePath) cfg.update({"sat_folder": str(savePath)}) cfg.download_all = False if cfg.download_one else True cfg.download_all = cfg.download_all and cfg.download_flag cfg.download_one = cfg.download_one and cfg.download_flag if cfg.query_by == "roi": footprint = geojson_to_wkt(read_geojson(str(workpath / cfg.roi_url))) roi_name = os.path.split(cfg.roi_url)[-1].split(".")[0] if cfg.query_by == "place": footprint = placename_to_wkt(cfg.placename) roi_name = cfg.placename.replace(" ", "_") # print(BC) ### DSC rorb = 22 if cfg.platformname == "Sentinel-1": cfg.checkProperty = "system:index" cfg.check_eeImgCol = "COPERNICUS/S1_GRD" products = api.query( footprint, date=(cfg.start_date.replace("-", ""), cfg.end_date.replace("-", "")), platformname=cfg.platformname, producttype=cfg.producttype, order_by='+beginposition', ) else: # S2, S3 ... cfg.checkProperty = "PRODUCT_ID" cfg.check_eeImgCol = "COPERNICUS/S2" if 'S2MSI1C' == cfg.producttype else "COPERNICUS/S2_SR" products = api.query( footprint, date=(cfg.start_date.replace("-", ""), cfg.end_date.replace("-", "")), platformname=cfg.platformname, producttype=cfg.producttype, order_by='+beginposition', cloudcoverpercentage=(0, cfg.cloudcoverpercentage), # for S2 only ) # print(products['0c05435b-0cd3-45a0-93f4-8c317eb1d558']) print("\n\n===========> Sentinel Auto-Query <============") products_df = api.to_dataframe(products) # print(products_df.keys()) # print(products_df.index) # pprint(products_df[['sensoroperationalmode', 'orbitdirection', 'relativeorbitnumber']]) products_dict = products_df.transpose().to_dict() products_list = products_df.index.tolist() if len(products_list) > 0: example_dict = products_dict[products_list[0]] property_list = [ key for key in example_dict.keys() if is_jsonable(example_dict[key]) ] # pprint(products_dict.keys()) # select property for saving to json orbit_dict = {'ASCENDING': 'ASC', 'DESCENDING': 'DSC'} products_QueryInfo = edict() checkImgCol = ee.ImageCollection(f"{cfg.check_eeImgCol}") if SAT == "S1": sentinel_asset = ee.ImageCollection("users/omegazhangpzh/Sentinel1") if SAT == "S2": sentinel_asset = ee.ImageCollection("users/omegazhangpzh/Sentinel2") for product_id in products_dict.keys(): title = products_dict[product_id]['title'] filtered_size = ee.Number(checkImgCol.filter(ee.Filter.eq(cfg.checkProperty, title)).size())\ .add(sentinel_asset.filter(ee.Filter.eq(cfg.checkProperty, title)).size()).getInfo() flag = filtered_size > 0 print(title, flag) # flag = False if not flag: # if this product is not available in GEE # print(title) # print(title, flag.getInfo()) products_QueryInfo[title] = { key: products_dict[product_id][key] for key in property_list } # products_QueryInfo[title]['product_id'] = product_id orbit_direction = products_dict[product_id]['orbitdirection'] orbit_num = products_dict[product_id]['relativeorbitnumber'] products_QueryInfo[title]['orbit_key'] = orbit_dict[ orbit_direction] + "_" + str(orbit_num) QueryInfo = edict() QueryInfo["products"] = products_QueryInfo QueryInfo["results"] = edict() QueryInfo["results"]['total_number'] = len(products_QueryInfo.keys()) QueryInfo["results"]['products_list'] = sorted( list(products_QueryInfo.keys())) QueryInfo["results"]['orbKey_list'] = list( set([ products_QueryInfo[product]['orbit_key'] for product in list(products_QueryInfo.keys()) ])) QueryInfo["cfg"] = cfg # roi_name = os.path.split(cfg.roi_url)[-1].split(".")[0] jsonPath = datafolder / "outputs" / roi_name if not os.path.exists(str(jsonPath)): os.makedirs(jsonPath) if save_json: """ save as json """ json_url = jsonPath / f"{SAT}_{cfg.producttype}_{now}.json" print("\njson_url: " + str(json_url)) with open(str(json_url), 'w') as fp: json.dump(edict(QueryInfo), fp, ensure_ascii=False, indent=4) """ save as geojson """ import geojson with open(jsonPath / f"S1_{cfg.producttype}_{now}.geojson", 'w') as fp: geojson.dump(api.to_geojson(products), fp, indent=4) print() # print(footprint) print("now: ", now) print("Total Number of Searched Products:" + str(len(QueryInfo["results"]['products_list']))) pprint(QueryInfo["results"]['products_list']) return QueryInfo
m = folium.Map([(AREA[0][1] + AREA[len(AREA) - 1][1]) / 2, (AREA[0][0] + AREA[len(AREA) - 1][0]) / 2], zoom_start=10) folium.GeoJson(str(object_name) + '.geojson').add_to(m) m #光学画像であるSentinel-2の画像を取得するため、既に指定している「場所」の情報以外である以下4つ: #・対象とする衛星 #・期間 #・データの処理レベル #・被雲率 products = api.query( footprint_geojson, date=('20200601', '20200701'), #取得希望期間の入力 platformname='Sentinel-2', processinglevel='Level-2A', cloudcoverpercentage=(0, 100)) #被雲率(0%〜100%) len(products) products_gdf = api.to_geodataframe(products) products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'], ascending=[True]) products_gdf_sorted products_gdf_sorted.head() uuid = products_gdf_sorted.iloc[0]["uuid"] product_title = products_gdf_sorted.iloc[0]["title"] api.download(uuid)
class SentinelDownloader(object): def __init__(self, user, password, api_url='https://scihub.copernicus.eu/apihub'): self._apiname = api_url self._user = user self._password = password # init logger root = logging.getLogger() root.addHandler(logging.StreamHandler(sys.stderr)) if self._apiname == 'https://scihub.copernicus.eu/apihub': try: from sentinelsat import SentinelAPI except ImportError as e: gs.fatal( _("Module requires sentinelsat library: {}").format(e)) # connect SciHub via API self._api = SentinelAPI(self._user, self._password, api_url=self._apiname) elif self._apiname == 'USGS_EE': try: import landsatxplore.api from landsatxplore.errors import EarthExplorerError except ImportError as e: gs.fatal( _("Module requires landsatxplore library: {}").format(e)) api_login = False while api_login is False: # avoid login conflict in possible parallel execution try: self._api = landsatxplore.api.API(self._user, self._password) api_login = True except EarthExplorerError as e: time.sleep(1) self._products_df_sorted = None def filter(self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None): args = {} if clouds: args['cloudcoverpercentage'] = (0, int(clouds)) if relativeorbitnumber: args['relativeorbitnumber'] = relativeorbitnumber if producttype.startswith('S2') and int(relativeorbitnumber) > 143: gs.warning("This relative orbit number is out of range") elif int(relativeorbitnumber) > 175: gs.warning("This relative orbit number is out of range") if producttype: args['producttype'] = producttype if producttype.startswith('S2'): args['platformname'] = 'Sentinel-2' else: args['platformname'] = 'Sentinel-1' if not start: start = 'NOW-60DAYS' else: start = start.replace('-', '') if not end: end = 'NOW' else: end = end.replace('-', '') if query: redefined = [ value for value in args.keys() if value in query.keys() ] if redefined: gs.warning( "Query overrides already defined options ({})".format( ','.join(redefined))) args.update(query) gs.verbose( "Query: area={} area_relation={} date=({}, {}) args={}".format( area, area_relation, start, end, args)) products = self._api.query(area=area, area_relation=area_relation, date=(start, end), **args) products_df = self._api.to_dataframe(products) if len(products_df) < 1: gs.message(_('No product found')) return # sort and limit to first sorted product if sortby: self._products_df_sorted = products_df.sort_values( sortby, ascending=[asc] * len(sortby)) else: self._products_df_sorted = products_df if limit: self._products_df_sorted = self._products_df_sorted.head( int(limit)) gs.message( _('{} Sentinel product(s) found').format( len(self._products_df_sorted))) def list(self): if self._products_df_sorted is None: return id_kw = ('uuid', 'entity_id') identifier_kw = ('identifier', 'display_id') cloud_kw = ('cloudcoverpercentage', 'cloud_cover') time_kw = ('beginposition', 'acquisition_date') kw_idx = 1 if self._apiname == 'USGS_EE' else 0 for idx in range(len(self._products_df_sorted[id_kw[kw_idx]])): if cloud_kw[kw_idx] in self._products_df_sorted: ccp = '{0:2.0f}%'.format( float(self._products_df_sorted[cloud_kw[kw_idx]][idx])) else: ccp = 'cloudcover_NA' print_str = '{0} {1}'.format( self._products_df_sorted[id_kw[kw_idx]][idx], self._products_df_sorted[identifier_kw[kw_idx]][idx]) if kw_idx == 1: time_string = self._products_df_sorted[time_kw[kw_idx]][idx] else: time_string = self._products_df_sorted[ time_kw[kw_idx]][idx].strftime("%Y-%m-%dT%H:%M:%SZ") print_str += ' {0} {1}'.format(time_string, ccp) if kw_idx == 0: print_str += ' {0}'.format( self._products_df_sorted['producttype'][idx]) print(print_str) def download(self, output, sleep=False, maxretry=False): if self._products_df_sorted is None: return if not os.path.exists(output): os.makedirs(output) gs.message(_('Downloading data into <{}>...').format(output)) if self._apiname == 'USGS_EE': from landsatxplore.earthexplorer import EarthExplorer #from landsatxplore.exceptions import EarthExplorerError from landsatxplore.errors import EarthExplorerError from zipfile import ZipFile ee_login = False while ee_login is False: # avoid login conflict in possible parallel execution try: ee = EarthExplorer(self._user, self._password) ee_login = True except EarthExplorerError as e: time.sleep(1) for idx in range(len(self._products_df_sorted['entity_id'])): scene = self._products_df_sorted['entity_id'][idx] identifier = self._products_df_sorted['display_id'][idx] zip_file = os.path.join(output, '{}.zip'.format(identifier)) gs.message('Downloading {}...'.format(identifier)) try: ee.download(identifier=identifier, output_dir=output, timeout=600) except EarthExplorerError as e: gs.fatal(_(e)) ee.logout() # extract .zip to get "usual" .SAFE with ZipFile(zip_file, 'r') as zip: safe_name = zip.namelist()[0].split('/')[0] outpath = os.path.join(output, safe_name) zip.extractall(path=output) gs.message(_('Downloaded to <{}>').format(outpath)) try: os.remove(zip_file) except Exception as e: gs.warning( _('Unable to remove {0}:{1}').format(zip_file, e)) else: for idx in range(len(self._products_df_sorted['uuid'])): gs.message('{} -> {}.SAFE'.format( self._products_df_sorted['uuid'][idx], os.path.join(output, self._products_df_sorted['identifier'][idx]))) # download out = self._api.download(self._products_df_sorted['uuid'][idx], output) if sleep: x = 1 online = out['Online'] while not online: # sleep is in minutes so multiply by 60 time.sleep(int(sleep) * 60) out = self._api.download( self._products_df_sorted['uuid'][idx], output) x += 1 if x > maxretry: online = True def save_footprints(self, map_name): if self._products_df_sorted is None: return if self._apiname == 'USGS_EE': gs.fatal( _('USGS Earth Explorer does not support footprint download.')) try: from osgeo import ogr, osr except ImportError as e: gs.fatal( _("Option <footprints> requires GDAL library: {}").format(e)) gs.message(_("Writing footprints into <{}>...").format(map_name)) driver = ogr.GetDriverByName("GPKG") tmp_name = gs.tempfile() + '.gpkg' data_source = driver.CreateDataSource(tmp_name) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) # features can be polygons or multi-polygons layer = data_source.CreateLayer(str(map_name), srs, ogr.wkbMultiPolygon) # attributes attrs = OrderedDict([("uuid", ogr.OFTString), ("ingestiondate", ogr.OFTString), ("cloudcoverpercentage", ogr.OFTInteger), ("producttype", ogr.OFTString), ("identifier", ogr.OFTString)]) # Sentinel-1 data does not have cloudcoverpercentage prod_types = [type for type in self._products_df_sorted["producttype"]] s1_types = ["SLC", "GRD"] if any(type in prod_types for type in s1_types): del attrs["cloudcoverpercentage"] for key in attrs.keys(): field = ogr.FieldDefn(key, attrs[key]) layer.CreateField(field) # features for idx in range(len(self._products_df_sorted['uuid'])): wkt = self._products_df_sorted['footprint'][idx] feature = ogr.Feature(layer.GetLayerDefn()) newgeom = ogr.CreateGeometryFromWkt(wkt) # convert polygons to multi-polygons newgeomtype = ogr.GT_Flatten(newgeom.GetGeometryType()) if newgeomtype == ogr.wkbPolygon: multigeom = ogr.Geometry(ogr.wkbMultiPolygon) multigeom.AddGeometryDirectly(newgeom) feature.SetGeometry(multigeom) else: feature.SetGeometry(newgeom) for key in attrs.keys(): if key == 'ingestiondate': value = self._products_df_sorted[key][idx].strftime( "%Y-%m-%dT%H:%M:%SZ") else: value = self._products_df_sorted[key][idx] feature.SetField(key, value) layer.CreateFeature(feature) feature = None data_source = None # coordinates of footprints are in WKT -> fp precision issues # -> snap gs.run_command('v.import', input=tmp_name, output=map_name, layer=map_name, snap=1e-10, quiet=True) def get_products_from_uuid_usgs(self, uuid_list): scenes = [] for uuid in uuid_list: metadata = self._api.metadata(uuid, 'SENTINEL_2A') scenes.append(metadata) scenes_df = pandas.DataFrame.from_dict(scenes) self._products_df_sorted = scenes_df gs.message( _('{} Sentinel product(s) found').format( len(self._products_df_sorted))) def set_uuid(self, uuid_list): """Set products by uuid. TODO: Find better implementation :param uuid: uuid to download """ if self._apiname == 'USGS_EE': self.get_products_from_uuid_usgs(uuid_list) else: from sentinelsat.sentinel import SentinelAPIError self._products_df_sorted = {'uuid': []} for uuid in uuid_list: try: odata = self._api.get_product_odata(uuid, full=True) except SentinelAPIError as e: gs.error('{0}. UUID {1} skipped'.format(e, uuid)) continue for k, v in odata.items(): if k == 'id': k = 'uuid' elif k == 'Sensing start': k = 'beginposition' elif k == 'Product type': k = 'producttype' elif k == 'Cloud cover percentage': k = 'cloudcoverpercentage' elif k == 'Identifier': k = 'identifier' elif k == 'Ingestion Date': k = 'ingestiondate' elif k == 'footprint': pass else: continue if k not in self._products_df_sorted: self._products_df_sorted[k] = [] self._products_df_sorted[k].append(v) def filter_USGS(self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None): if area_relation != 'Intersects': gs.fatal( _('USGS Earth Explorer only supports area_relation' ' "Intersects"')) if relativeorbitnumber: gs.fatal( _('USGS Earth Explorer does not support "relativeorbitnumber"' ' option.')) if producttype and producttype != 'S2MSI1C': gs.fatal( _('USGS Earth Explorer only supports producttype S2MSI1C')) if query: if not any( key in query for key in ['identifier', 'filename', 'usgs_identifier']): gs.fatal( _('USGS Earth Explorer only supports query options' ' "filename", "identifier" or "usgs_identifier".')) if 'usgs_identifier' in query: # get entityId from usgs identifier and directly save results usgs_id = query['usgs_identifier'] check_s2l1c_identifier(usgs_id, source='usgs') # entity_id = self._api.lookup('SENTINEL_2A', [usgs_id], # inverse=True) entity_id = self._api.get_entity_id([usgs_id], 'SENTINEL_2A') self.get_products_from_uuid_usgs(entity_id) return else: if "filename" in query: esa_id = query['filename'].replace('.SAFE', '') else: esa_id = query['identifier'] check_s2l1c_identifier(esa_id, source='esa') esa_prod_id = esa_id.split('_')[-1] utm_tile = esa_id.split('_')[-2] acq_date = esa_id.split('_')[2].split('T')[0] acq_date_string = '{0}-{1}-{2}'.format(acq_date[:4], acq_date[4:6], acq_date[6:]) start_date = end_date = acq_date_string # build the USGS style S2-identifier if utm_tile.startswith('T'): utm_tile_base = utm_tile[1:] bbox = get_bbox_from_S2_UTMtile(utm_tile_base) else: # get coordinate pairs from wkt string str_1 = 'POLYGON((' str_2 = '))' coords = area[area.find(str_1) + len(str_1):area.rfind(str_2)].split(',') # add one space to first pair for consistency coords[0] = ' ' + coords[0] lons = [float(pair.split(' ')[1]) for pair in coords] lats = [float(pair.split(' ')[2]) for pair in coords] bbox = (min(lons), min(lats), max(lons), max(lats)) start_date = start end_date = end usgs_args = { 'dataset': 'SENTINEL_2A', 'bbox': bbox, 'start_date': start_date, 'end_date': end_date } if clouds: usgs_args['max_cloud_cover'] = clouds if limit: usgs_args['max_results'] = limit scenes = self._api.search(**usgs_args) self._api.logout() if query: # check if the UTM-Tile is correct, remove otherwise for scene in scenes: if scene['display_id'].split('_')[1] != utm_tile: scenes.remove(scene) # remove redundant scene if len(scenes) == 2: for scene in scenes: prod_id = scene['display_id'].split('_')[-1] if prod_id != esa_prod_id: scenes.remove(scene) if len(scenes) < 1: gs.message(_('No product found')) return scenes_df = pandas.DataFrame.from_dict(scenes) if sortby: # replace sortby keywords with USGS keywords for idx, keyword in enumerate(sortby): if keyword == 'cloudcoverpercentage': sortby[idx] = 'cloud_cover' # turn cloudcover to float to make it sortable scenes_df['cloud_cover'] = pandas.to_numeric( scenes_df['cloud_cover']) elif keyword == 'ingestiondate': sortby[idx] = 'acquisition_date' # what does sorting by footprint mean elif keyword == 'footprint': sortby[idx] = 'display_id' self._products_df_sorted = scenes_df.sort_values(sortby, ascending=[asc] * len(sortby), ignore_index=True) else: self._products_df_sorted = scenes_df gs.message( _('{} Sentinel product(s) found').format( len(self._products_df_sorted)))