def test_missing_dependency_dataframe(monkeypatch):
api = SentinelAPI("mock_user", "mock_password")
with pytest.raises(ImportError):
monkeypatch.setitem(sys.modules, "pandas", None)
api.to_dataframe({"test":"test"})
with pytest.raises(ImportError):
monkeypatch.setitem(sys.modules, "geopandas", None)
api.to_geodataframe({"test":"tst"})
def test_missing_dependency_dataframe(monkeypatch):
api = SentinelAPI("mock_user", "mock_password")
with pytest.raises(ImportError):
monkeypatch.setitem(sys.modules, "pandas", None)
api.to_dataframe({"test": "test"})
with pytest.raises(ImportError):
monkeypatch.setitem(sys.modules, "geopandas", None)
api.to_geodataframe({"test": "tst"})
def test_to_geopandas(products):
gdf = SentinelAPI.to_geodataframe(products)
assert type(gdf).__name__ == 'GeoDataFrame'
print(gdf.unary_union.area)
assert gdf.unary_union.area == pytest.approx(89.6, abs=0.1)
assert len(gdf) == len(products)
assert gdf.crs == {'init': 'epsg:4326'}
def downloadingData(aoi, collectionDate, plName, prLevel, clouds, username,
password, directory):
'''
Downloads the Sentinel2 - Data with the given parameters
Parameters:
aoi (str): The type and the coordinates of the area of interest
collectionDate (datetime 64[ns]): The date of the data
plName (str): The name of the platform
prLevel (str): The name of the process
clouds (tuple of ints): Min and max of cloudcoverpercentage
username (str): The username of the Copernicus SciHub
password (str): The password of the Copernicus SciHub
directory (str): Pathlike string to the directory
'''
api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
'''Choosing the data with bounding box (footprint), date, platformname, processinglevel and cloudcoverpercentage'''
products = api.query(aoi,
date=collectionDate,
platformname=plName,
processinglevel=prLevel,
cloudcoverpercentage=clouds)
'''Filters the products and sorts by cloudcoverpercentage'''
products_gdf = api.to_geodataframe(products)
products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'],
ascending=[True])
'''Downloads the choosen files from Scihub'''
products_gdf_sorted.to_csv(os.path.join(directory, 'w'))
api.download_all(products, directory, max_attempts=10, checksum=True)
def test_to_geopandas(products):
gdf = SentinelAPI.to_geodataframe(products)
assert type(gdf).__name__ == "GeoDataFrame"
print(gdf.unary_union.area)
assert gdf.unary_union.area == pytest.approx(89.6, abs=0.1)
assert len(gdf) == len(products)
assert gdf.crs == "EPSG:4326"
def get_image_products(self):
"""
Returns
-------
products : List
Contains collection of information of available images from query
"""
print('\n >>> CONDUCTING API QUERY')
api = SentinelAPI(self.username, self.password,
'https://scihub.copernicus.eu/dhus')
shape = gpd.read_file(self.aoi_dir)
shape = shape.to_crs({'init': 'epsg:4326'})
footprint = None
for i in shape['geometry']:
footprint = i
products = api.query(footprint,
date=self.date_range,
platformname='Sentinel-2',
processinglevel='Level-2A',
cloudcoverpercentage=(0, 10))
products = api.to_geodataframe(products)
products = products.sort_values(['cloudcoverpercentage'],
ascending=[True])
return products
def get_data(name,
AREA,
start_date='20190101',
end_date='20190128',
platformname='Sentinel-2',
processinglevel='Level-2A'):
#apiのクエリに次のが必要。中身はPOLYGON((..))っていうstr型
footprint_geojson = geojson_to_wkt(
read_geojson('./location/' + str(name) + '.geojson'))
# use sentinelAPI
user = ''
password = ''
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus')
products = api.query(
footprint_geojson,
date=(start_date, end_date), #取得希望期間の入力
platformname=platformname,
processinglevel=processinglevel, ##2016年はL1C
cloudcoverpercentage=(0, 100)) #被雲率（0％〜100％）
print("この期間の画像の数は" + str(len(products)) + "枚です")
#この後で雲の被覆率ごとにデータを並べ替えて、1個ダウンロードする。
products_gdf = api.to_geodataframe(products)
products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'],
ascending=[True]).head()
#ファイルがまだなければデータダウンロード。作業ディレクトリにzipファイルがダウンロードされる
for i in range(3): #3回までチャレンジ
uuid = products_gdf_sorted.iloc[i]["uuid"]
product_title = products_gdf_sorted.iloc[i][
"title"] #S2A_MSIL2A_20190101T01405... みたいな
product_date = products_gdf_sorted.iloc[i]["summary"].split(
',')[0].split()[1][:10]
print("この期間で1番被覆率が低いのは" + product_date + "日")
if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True:
print("新規にデータをダウンロードします")
try:
api.download(uuid)
except:
print("ダウンロード不可")
else:
break
else:
break
if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True:
#ダウンロードしたzipファイルを解凍
#str(product_title) + '.SAFE っていうフォルダが生成される
file_name = str(product_title) + '.zip'
with zipfile.ZipFile(file_name) as zf:
zf.extractall()
shutil.move(str(product_title) + '.zip', './data_geo/')
shutil.move(str(product_title) + '.SAFE', './data_geo/')
return product_title
def find_s2_uuid(s2_filename):
"""
Returns S2 uuid required for download via sentinelsat, based upon an input S2 file/scene name.
I.e. S2A_MSIL1C_20180820T223011_N0206_R072_T60KWE_20180821T013410
Assumes esa hub creds stored as env variables.
:param s2_file_name: Sentinel-2 scene name
:return s2_uuid: download id
"""
copernicus_username = os.getenv("COPERNICUS_USERNAME")
copernicus_pwd = os.getenv("COPERNICUS_PWD")
print(f"ESA username: {copernicus_username}")
esa_api = SentinelAPI(copernicus_username, copernicus_pwd)
if s2_filename[-5:] == '.SAFE':
res = esa_api.query(filename=s2_filename)
res = esa_api.to_geodataframe(res)
return res.uuid.values[0]
def searchOverlapping(geom, search_args):
'''
use sentinel API to search for scenes that overlap with geom
Inputs:
-------
geom: shapely geometry.
search_args : passed onto SentinelAPI.query, read the docs.
Outputs:
-------
Geodataframe with Sentinel Products overlapping with geom.
'''
api = SentinelAPI(SENTINEL_USER, SENTINEL_PASS,
'https://scihub.copernicus.eu/dhus/')
products = api.query(geom, **search_args)
if len(products) == 0:
return (None)
return (api.to_geodataframe(products))
def check_login(username, password, latitude, longitude):
''' Checks the login and location details, SentinelAPI queries the Copernicus database '''
if username == None or password == None:
print(
'\n Enter Login Details for the Copernicus SciHub\n if not registered, go to:\n https://scihub.copernicus.eu\n'
)
username = input('\n Username: '******'\n Password: '******'\n')
if latitude == None or longitude == None:
print('Latitude and Longitude in decimal degrees\n')
try:
latitude = float(input('\n Latitude: '))
longitude = float(input('\n Longitude: '))
except:
print(
'\n Latitude and Longitude are to be entered in decimal degrees\n Program Terminated\n'
)
sys.exit()
print('\n')
try:
api = SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus')
footprint = f'POINT ({latitude} {longitude})'
data_populate = api.query(footprint,
date=('NOW-12MONTHS', 'NOW'),
order_by='cloudcoverpercentage',
platformname='Sentinel-2',
processinglevel='Level-2A',
cloudcoverpercentage=(0, 30))
data_database = api.to_geodataframe(data_populate)
return api, data_database
except:
print('\n Incorrect Login Details.\n Program Terminated.')
sys.exit()
class Downloader:
def __init__(self, str_username, str_password, str_link):
self.api = SentinelAPI(str_username, str_password, str_link)
self.products = None
def search_polygon(self, footprint: object, str_date_start: str,
str_date_end: str, str_platform_name: str, percentage: object):
print('searching')
self.products = self.api.query(footprint,
date=(str_date_start, str_date_end),
platformname=str_platform_name,
cloudcoverpercentage=(percentage[0], percentage[1]))
size = self.api.get_products_size(self.products)
print(f'found {size}GiB of data')
# print(self.products)
def download_zip(self, path):
self.api.download_all(self.products, path, max_attempt, True)
def download_products(self, path, download_file):
if download_file:
self.download_zip(path)
print('downloaded')
df_products = self.api.to_dataframe(self.products)
return df_products
def download_geoproduct(self, path, download_file):
if download_file:
self.download_zip(path)
# print('download Geos')
gdf_products = self.api.to_geodataframe(self.products)
return gdf_products
def download_json(self):
return self.api.to_geojson(self.products)
def download_one(self, key, path):
self.api.download(key, path, True)
class GetImage():
def __init__(self):
self.api = None
self._connect_to_api()
def _connect_to_api(self):
user = '******'
password = "******"
self.api = SentinelAPI(user, password,
'https://scihub.copernicus.eu/dhus')
def _create_footprint(self, sf):
footprint = None
for i in sf['geometry']:
footprint = i
return footprint
def _get_product(self,
footprint,
num_images,
start_date=0,
end_date=0,
processing_level='Level-2A'):
products = self.api.query(footprint,
date=('20190601', '20190626'),
platformname='Sentinel-2',
processinglevel='Level-2A',
cloudcoverpercentage=(0, 10))
products_gdf = self.api.to_geodataframe(products)
best_products = products_gdf.sort_values(
['cloudcoverpercentage', 'ingestiondate'],
ascending=[True, True]).head(num_images)
print(best_products[['title', 'cloudcoverpercentage']])
return best_products
def _download_image(self, products):
save_path = "products/"
extension = ".zip"
origin_path = os.getcwd()
os.makedirs(save_path, exist_ok=True)
os.chdir(save_path)
current_path = os.getcwd()
self.api.download_all(products.index)
for item in os.listdir(current_path):
if item.endswith(extension):
file_name = os.path.abspath(item)
zip_ref = zipfile.ZipFile(file_name)
zip_ref.extractall(current_path)
zip_ref.close()
os.remove(file_name)
print(current_path)
directories = [x for x in glob.glob(current_path + '/*/')]
os.chdir(origin_path)
return directories
def get_image(self, shp_path, num_images):
sf = gpd.read_file(shp_path)
footprint = self._create_footprint(sf)
products = self._get_product(footprint, num_images)
list_images = self._download_image(products)
print(list_images)
return 0
import datetime
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt
import psycopg2
api = SentinelAPI('MRDEVISH', 'Research1')
today_date = datetime.date.today()
footprint = geojson_to_wkt(read_geojson('search_polygon_10.geojson'))
platformname = 'Sentinel-3'
producttype='SL_2_LST___'
products = api.query(footprint,platformname=platformname,producttype=producttype, date = (today_date-1,today_date))
products_pandas = api.to_geodataframe(products)
Names = products_pandas['Product Name']
connection1 = psycopg2.connect(user="******",
password="******",
host="127.0.0.1",
port="5432",
database="db_products")
cursor1 = connection1.cursor()
query0 = """select "Product Name" from "Products"; """
cursor1.execute(query0)
records = cursor1.fetchone()
num=1
undownloaded = []
for i in Names:
if i not in records:
cursor1 = connection1.cursor()
id_ = len(records)+num
query = """INSERT INTO "Products" Values({},{},{},0);""".format(id_,i,today_date)
cursor1.execute(query0)
num=num+1
import pika
def lst_prod(shpExtent,
start_time,
end_time,
outShp=None,
platname="Sentinel-2",
procLevel="Level-2A",
max_cloud_cover=None):
"""
List Sentinel Products for one specific area
platformname:
* Sentinel-1
* Sentinel-2
* Sentinel-3
processinglevel:
* Level-1A
* Level-1B
* Level-1C
* Level-2A
...
"""
import os
from sentinelsat import SentinelAPI, geojson_to_wkt
from glass.g.rd.shp import shp_to_obj
from glass.pys.oss import fprop
from glass.g.wt.shp import df_to_shp
from glass.cons.sentinel import con_datahub
# Get Search Area
if os.path.isfile(shpExtent):
if fprop(shpExtent, 'ff') == '.json':
boundary = geojson_to_wkt(shpExtent)
else:
boundary = shp_to_obj(shpExtent,
output='array',
fields=None,
geom_as_wkt=True,
srs_to=4326)[0]["GEOM"]
else:
# Assuming we have an WKT
boundary = shpExtent
# Create API instance
user, password = con_datahub()
api = SentinelAPI(user, password, URL_COPERNICUS)
# Search for products
products = api.query(
boundary,
date=(start_time, end_time),
platformname=platname,
cloudcoverpercentage=(0,
100 if not max_cloud_cover else max_cloud_cover),
processinglevel=procLevel)
df_prod = api.to_geodataframe(products)
if not df_prod.shape[0]:
return df_prod
df_prod['ingestiondate'] = df_prod.ingestiondate.astype(str)
df_prod['beginposition'] = df_prod.beginposition.astype(str)
df_prod['endposition'] = df_prod.endposition.astype(str)
# Export results to Shapefile
if outShp:
return df_to_shp(df_prod, outShp)
else:
return df_prod
def test_to_geopandas_empty():
gdf = SentinelAPI.to_geodataframe({})
assert type(gdf).__name__ == "GeoDataFrame"
assert len(gdf) == 0
def test_missing_dependency_geodataframe(monkeypatch):
with pytest.raises(ImportError):
monkeypatch.setitem(sys.modules, "geopandas", None)
SentinelAPI.to_geodataframe({"test": "tst"})
api = SentinelAPI('fabiolana', 'Albertone_2016')
footprint = geojson_to_wkt(read_geojson('/media/sf_Downloads/__effis/ricerca.geojson'))
products = api.query(footprint,
# date=('20170801', date(2017, 12, 15)),
date=('20151001', '20171205'),
platformname='Sentinel-2',
# producttype='SLC')
#,orbitdirection='ASCENDING') #,
cloudcoverpercentage=(0, 10))
# download all results from the search
# api.download_all(products)
# GeoJSON FeatureCollection containing footprints and metadata of the scenes
api.to_geojson(products)
# inserito in dataframe
df_prods = pd.DataFrame(api.to_geojson(products))
# print df_prods
# GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries
print api.to_geodataframe(products)
# Get basic information about the product: its title, file size, MD5 sum, date, footprint and
# its download url
# api.get_product_odata(<product_id>)
# Get the product's full metadata available on the server
# api.get_product_odata(<product_id>, full=True)
def main(args):
""" Runs dataLayer processing scripts to turn raw dataLayer from (../raw) into
cleaned dataLayer ready to be analyzed (saved in ../processed).
"""
## Talk to Rune about how dataLayer is handle. If it should be part of the "big" project.
## set number_tiles:1764
config = TrainingConfig()
config = update_config(args, config)
logger = logging.getLogger(__name__)
logger.info('making final dataLayer set from raw dataLayer')
userTuple = [['pandagud', 'damp4ever'], ['pandagud2', 'damp4ever'],
['pandagud3', 'damp4ever'], ['au524478', 'Palantir1234']]
current_user = random.choice(userTuple)
api = SentinelAPI(current_user[0], current_user[1],
'https://scihub.copernicus.eu/dhus')
# search by polygon, time, and SciHub query keywords
path = r"C:\Users\panda\Downloads\LC80290292014132LGN00.geojson"
footprint = geojson_to_wkt(read_geojson(path))
products = api.query(area=footprint,
date=('20210101', '20210105'),
platformname='Sentinel-2',
order_by='+ingestiondate',
limit=1)
areas = api.to_geodataframe(products)
geojson = api.to_geojson(products)
api.download_all(products, into=r'C:\Users\panda\Sat_paper\Alfa')
products = api.query(area=footprint,
date=('20210401', '20210430'),
producttype='GRD',
platformname='Sentinel-1',
sensoroperationalmode='IW',
polarisationmode='VV VH',
order_by='ingestiondate')
firstproduct = next(iter(products))
online_product = ''
for i in products:
is_online = api.is_online(products.get(i).get('uuid'))
if is_online:
online_product = i
break
delete_list = []
for i in products:
if i != online_product:
delete_list.append(i)
for i in delete_list:
del products[i]
ground_geojsons = read_geojson(path)
products_geojsons = api.to_geojson(products)
ground_polygon = ground_geojsons.get('features')[0].get('geometry').get(
'coordinates')
ground_polygon = geometry.Polygon(ground_polygon[0][0])
import numpy as np
titles = []
ids = []
for item in products_geojsons.get('features'):
id = item.get('properties').get('id')
item = item.get('properties').get('title')
item = (item[17:25] + item[48:55])
titles.append(item)
ids.append([item, id])
unique = list(set(titles))
union_list = []
for i, element in enumerate(unique):
local_polygon = Polygon()
for j in range(len(titles)):
if titles[j] == element:
item = products_geojsons.get('features')[j]
item = item.get('geometry').get('coordinates')
item = geometry.Polygon(item[0][0])
item = affinity.scale(item, xfact=1.01, yfact=1.01)
polygons = [item, local_polygon]
local_polygons = unary_union(polygons)
local_polygon = item
union_list.append([local_polygons, element])
for index, element in enumerate(union_list):
wkt = element[0].wkt
if ground_polygon.within(element[0]):
found_id = element[1]
break
for i in ids:
if found_id != i[0]:
del products[i[1]]
area_list = []
for index, item in enumerate(products_geojsons.get('features')):
item = item.get('geometry').get('coordinates')
item = geometry.Polygon(item[0][0])
local_intersection = item.intersection(ground_polygon)
local_intersection = [local_intersection.area, index]
area_list.append(local_intersection)
area_list.sort(reverse=True)
for index in range(len(area_list)):
item = products_geojsons.get('features')[area_list[index][1]]
id = item.get('properties').get('id')
item = item.get('geometry').get('coordinates')
item = geometry.Polygon(item[0][0])
if item.intersects(ground_polygon):
local_intersection = ground_polygon.intersection(item)
print(str(ground_polygon.area))
print(str(local_intersection.area))
# ground_polygon = ground_polygon.difference(local_intersection)
ground_polygon = (ground_polygon.symmetric_difference(
local_intersection)).difference(local_intersection)
else:
del products[id]
import datetime
from datetime import timedelta
S2_geojson = read_geojson(path)
start_S1_date = S2_geojson.get('features')[0].get('properties').get(
'ingestiondate')
start_S1_date = start_S1_date.split('T')[0]
start_S1_date = datetime.datetime.strptime(start_S1_date,
'%Y-%m-%d').date()
## New end date for S1
end_S1_date = start_S1_date + timedelta(days=7)
start_S1_date = start_S1_date - timedelta(days=7)
start_S1_date_str = str(start_S1_date).replace('-', '')
end_S1_date_str = str(end_S1_date).replace('-', '')
## COMBINE FOOTPRINT
geom_in_geojson = []
geom_in_geojson.append(
geojson.Feature(geometry=ground_polygon,
properties={"MissingData": "Test"}))
feature_collection = FeatureCollection(geom_in_geojson)
pathToFile = r'C:\Users\panda\Sat_paper\missing.geojson'
with open(pathToFile, 'w') as f:
dump(feature_collection, f)
print("Done")
def test_to_geopandas(products):
gdf = SentinelAPI.to_geodataframe(products)
assert type(gdf).__name__ == 'GeoDataFrame'
assert abs(gdf.unary_union.area - 132.16) < 0.01
assert len(gdf) == len(products)
assert gdf.crs == {'init': 'epsg:4326'}
def test_to_geopandas_empty(products):
gdf = SentinelAPI.to_geodataframe({})
assert type(gdf).__name__ == 'GeoDataFrame'
assert len(gdf) == 0
Coordinates_texts = [list(map(str, x)) for x in Coordinates_texts]
for i in range(len(Coordinates_texts)):
Coordinates_texts[i]=','.join(Coordinates_texts[i])
Coordinates_texts = ' '.join(Coordinates_texts)
element.findall(('{0}Document/{0}Placemark/{0}Polygon/{0}outerBoundaryIs/{0}LinearRing/{0}coordinates'.format(namespace))).text = Coordinates_texts
element.remove('{0}Document/{0}Placemark/{0}ExtendedData'.format(namespace))
a = gpd.read_file(kml)
os.mkdir(os.path.join(os.getcwd(), 'ShapeFiles'))
kml_name = os.path.basename(kml).split('.')[0]
a.to_file(os.path.join(os.getcwd(), 'ShapeFiles', kml_name),driver='ESRI Shapefile')
#insert your login to SentinelAPI
user = ''
password = ''
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus')
shapefiles_list = glob.glob(os.path.join(os.getcwd(), 'ShapeFiles')+ '*.shp')
for shapefile in shapefiles_list:
GPD_DF = gpd.read_file(shapefile)
footprint = None
for i in GPD_DF['geometry']:
footprint = i
products = api.query(footprint,
#modify the date
date = ('20190601', '20190626'),
platformname = 'Sentinel-2',
processinglevel = 'Level-2A',
cloudcoverpercentage = (0,10)
)
products_gdf = api.to_geodataframe(products)
products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'], ascending=[True])
def test_to_geopandas(products):
gdf = SentinelAPI.to_geodataframe(products)
assert abs(gdf.unary_union.area - 132.16) < 0.01
def test_to_geopandas(products):
gdf = SentinelAPI.to_geodataframe(products)
assert abs(gdf.unary_union.area - 132.16) < 0.01
class ApiSession():
"""
ApiSession Class handles all connections with SentinelAPI.
"""
def __init__(self):
self.key_file_path = '../ressources/apiKey.txt'
self.export_directory = (os.path.normpath(
os.path.join(os.getcwd(), os.pardir, 'downloadedImages')))
self.user, self.password = self.parsefile()
self.api = SentinelAPI(self.user, self.password, query['url'])
self.platform = 'Sentinel-2'
def parsefile(self):
"""
Parses the apiKey.txt and returns a tuple containing the username
and password for the SentinelAPI.
"""
text = ''
try:
text = open(self.key_file_path, 'r')
except FileExistsError as exception:
print('Api key file not found, must be in ressources/apiKey.txt')
print('Raised Error: {}'.format(exception))
info = []
for line in text:
info.append(line.strip().split(',')[1])
return (info[0], info[1])
def query(self, footprint):
"""
Queries the SentinelAPI and returns a geojson containing data
candidates.
"""
return self.api.query(footprint,
date=(query['begin'], query['end']),
platformname=query['platform'],
processinglevel=query['processing'])
def to_geo_df(self, product):
"""
Returns GeoDataFrame
"""
return self.api.to_geodataframe(product)
def download(self, link, directory):
"""
Dowloads Data to directory using link provided.
"""
self.api.download(link, directory_path=directory)
def query_to_dataframe(self, footprint, output_path, contains=True):
"""
Saves the queried geopandas to a csv file so that it could be
used in the future.
contains: if true will only save the links that fully contain the
footprint
"""
catalog = self.query(footprint)
catalog = self.to_geo_df(catalog)
desired_columns = [
'summary', 'vegetationpercentage', 'notvegetatedpercentage',
'waterpercentage', 'unclassifiedpercentage', 'snowicepercentage',
'cloudcoverpercentage', 'geometry', 'size'
]
filtered_catalog = catalog[desired_columns]
if contains:
contains = [
footprint.within(geometry) for geometry in catalog['geometry']
]
filtered_catalog = filtered_catalog[contains]
output = filtered_catalog.to_csv()
output_file = open(output_path, 'w')
output_file.write(output)
output_file.close()
from tensorflow import keras
import numpy as np
import matplotlib.pyplot as plt
import IPython.display as display
from PIL import Image
import os
import pathlib
import rasterio
from rasterio import plot
from pyrsgis import raster
user = '******'
password = '******'
# connect to the API
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus')
# search by polygon, time, and Hub query keywords
footprint = geojson_to_wkt(
read_geojson('/home/russell/XCity/gsonData/test.geojson'))
products = api.query(footprint,
date=('20190225', '20190227'),
platformname='Sentinel-2',
processinglevel='Level-2A',
cloudcoverpercentage=(0, 20))
# download all results from the search
api.download_all(products)
# GeoJSON FeatureCollection containing footprints and metadata of the scenes
api.to_geojson(products)
# GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries
api.to_geodataframe(products)
print("finished")
def test_to_geopandas_empty(products):
gdf = SentinelAPI.to_geodataframe({})
assert type(gdf).__name__ == 'GeoDataFrame'
assert len(gdf) == 0
(59.51138530046753, 24.825137916849023),
(59.459087606762346, 24.907535377786523),
(59.4147455486766, 24.929508034036523),
(59.39832075950073, 24.844363991067773),
(59.37664183245853, 24.814151588724023),
(59.35249898189222, 24.75304013852871),
(59.32798867805195, 24.573825660989648)]
# Copernicus Hub likes coordinates in lng,lat format
return Polygon([(y, x) for x, y in tln_points])
username = "******"
password = "******"
hub = SentinelAPI(username, password, "https://scihub.copernicus.eu/dhus")
data_products = hub.query(
get_tallinn_polygon(), # which area interests you
date=("20200101", "20200420"),
cloudcoverpercentage=(0, 10), # we don't want clouds
platformname="Sentinel-2",
processinglevel="Level-2A" # more processed, ready to use data
)
data_products = hub.to_geodataframe(data_products)
# we want to avoid downloading overlapping images, so selecting by this keyword
data_products = data_products[data_products["title"].str.contains("T35VLF")]
print(data_products.shape)
# convert to Pandas DataFrame products_df = api.to_dataframe(products) # sort for most recent and lowest cloud cover products_df_sorted = products_df.sort_values(['ingestiondate', 'cloudcoverpercentage'], ascending=[True, True]) test = products_df_sorted.head(1) test['cloudcoverpercentage'] test['ingestiondate'] set(products_df['processinglevel']) # GeoJSON FeatureCollection containing footprints and metadata of the scenes geojson_products = api.to_geojson(products) # GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries geodata_products = api.to_geodataframe(products) # plot product time vs cloudcover # data = [go.Scatter(x=products_df_sorted.ingestiondate, y=products_df_sorted[['cloudcoverpercentage']])] # py.plotly.iplot(data, filename = 'time-series-simple') # download api.download_all(test.index) # unzip # todo: needs loop to run through the data sets directory_to_extract_to = "unzip" zip = zipfile.ZipFile(str(test.title[0]) + '.zip') zip.extractall(directory_to_extract_to) zip.close()
class DownloadImages(object):
'''
This class is composed by a set of mehtods that download the Sentinel-2 images of
a given Polygon and updates a diretory with the donnwloaded files.
Parameters
---------
username: str
personal username in Copernicus / Sentinel API
password: str
personal password in Copernicus / Sentinel API
'''
def __init__(self, username, password):
self.username = username
self.password = password
self.api = SentinelAPI(self.username, self.password,
'https://scihub.copernicus.eu/dhus')
def get_products(self,
polygon,
initial_date,
final_date,
cloudcover=(0, 10)):
'''
Overview: returns geodataframe of query products from SentinelAPI.
Inputs
polygon: Shape
polygon of interest for api query request.
inital_date: str
string in the format 'YMD' of the initial request date.
final_date - str
string in the format 'YMD' of the final request date.
cloudcover: tuple, default=(0,10)
minimum cloud coverage range for api request.
Output:
geodataframe of query products.
'''
products = self.api.query(polygon,
date=(initial_date, final_date),
platformname='Sentinel-2',
processinglevel='Level-2A',
cloudcoverpercentage=cloudcover)
products = self.api.to_geodataframe(products)
return products
def download_folders(self,
polygon,
regional_poly,
products,
database_path,
threshold=0.9):
'''
Overview: downloads folders of interest that are not in the current database
Inputs
polygon: Shape
polygon of interest for area intersection calculation.
regional_poly:
bool to determine how intersection should be calculated, if region or not.
products: geodataframe
dataframe with all products returned from AI query request.
database_path: str
path to the database, where folders are to be downloaded.
threshold: float, default=0.9
minimum area of intersection required with polygon to donwload folder.
Output
returns geodataframe with products that were downloaded.
'''
delete = []
for idx, row in products.iterrows():
intersect_area = polygon.intersection(row.geometry).area
if regional_poly:
area = intersect_area / row.geometry.area
else:
area = intersect_area / polygon.area
if area >= threshold:
if row.filename not in os.listdir(database_path):
#print('downloading: ', row.filename)
self.api.download(idx, directory_path=database_path)
filepath = os.path.join(database_path, row.title + '.zip')
with zipfile.ZipFile(filepath) as zip_ref:
zip_ref.extractall(path=database_path)
else:
delete.append(idx)
else:
delete.append(idx)
products = products.drop(delete)
return products
def update_downloaded(self, csv_path, downloaded_products):
'''
Overview: updates directory with downloaded folders.
Inputs
csv_path: str
path to csv file, filepath.
downloaded_products: geodataframe
gdf of all new products that were downloaded.
Output
N/A.
'''
try:
df = pd.read_csv(csv_path)
df.append(downloaded_products)
df.to_csv(csv_path)
except:
downloaded_products.to_csv(csv_path)
def full_pipe(self,
csv_path,
polygon,
intial_date,
final_date,
database_path,
threshold=0.9,
cloudcover=(0, 10)):
'''
Overview: runs entire Download Images Pipeline, downloading and updating diretory with new products of interest.
Inputs
csv_path: str
path to csv file, filepath.
polygon: Shape
polygon of interest for api query request and area threshold calculation to determine which folders to download.
inital_date: str
string in the format 'YMD' of the initial request date.
final_date - str
string in the format 'YMD' of the final request date.
database_path: str
path to the database, where folders are to be downloaded.
threshold: float, default=0.9
minimum area of intersection required with polygon to donwload folder.
cloudcover: tuple, default=(0,10)
minimum cloud coverage range for api request.
Output
N/A.
'''
products = self.get_products(polygon, intial_date, final_date,
cloudcover)
products = self.download_folders(polygon, products, database_path,
threshold)
self.update_downloaded(csv_path, products)
