def test_get_coordinates(fixture_path):
wkt = ('POLYGON((-66.2695 -8.0592,-66.2695 0.7031,'
'-57.3047 0.7031,-57.3047 -8.0592,-66.2695 -8.0592))')
assert geojson_to_wkt(read_geojson(fixture_path('map.geojson'))) == wkt
assert geojson_to_wkt(read_geojson(fixture_path('map_z.geojson'))) == wkt
assert geojson_to_wkt(read_geojson(
fixture_path('map_nested.geojson'))) == wkt
def test_get_coordinates():
wkt = (
'POLYGON ((-66.2695312 -8.0592296, -66.2695312 0.7031074, ' +
'-57.3046875 0.7031074, -57.3046875 -8.0592296, -66.2695312 -8.0592296))'
)
assert geojson_to_wkt(read_geojson('tests/map.geojson')) == wkt
assert geojson_to_wkt(read_geojson('tests/map_z.geojson')) == wkt
def test_get_coordinates():
wkt = ('POLYGON((-66.2695 -8.0592,-66.2695 0.7031,'
'-57.3047 0.7031,-57.3047 -8.0592,-66.2695 -8.0592))')
assert geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')) == wkt
assert geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map_z.geojson')) == wkt
assert geojson_to_wkt(read_geojson(FIXTURES_DIR +
'/map_nested.geojson')) == wkt
def get_product_identifiers(date):
date = date.date()
footprint = geojson_to_wkt(
read_geojson(os.path.join(DIRNAME, 'norway_sweden.json')))
scenes = query(
'Sentinel3',
start_date=date,
end_date=date + datetime.timedelta(days=1),
geometry=footprint,
)
selected_scenes = []
for scene in scenes.values():
id = scene['properties']['title']
#Only S3A SLSTR level 1
if 'S3A_SL_1_RBT___' not in id:
continue
poly1 = shapely.wkt.loads(footprint)
poly2 = shapely.wkt.loads(geojson_to_wkt(scene))
intersection = poly1.intersection(poly2)
#Require some overlap
if intersection.area / poly1.area < .10:
continue
hour_of_day = int(id.split('_')[7].split('T')[1][:2])
#Limit to morning passes
if not (7 <= hour_of_day <= 12):
continue
selected_scenes.append(scene)
return selected_scenes
def search_download_sen2_data(user, password, area_polygon, datum, cloudcover,
downloadpath):
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt
import zipfile
import os
import geopandas as gpd
# connect to API
api = SentinelAPI(user,
password,
api_url="https://scihub.copernicus.eu/apihub/")
#### Test
outlines_geojson = gpd.read_file("outlines.shp")
# Avoid Fiona Bug https://github.com/Toblerity/Fiona/issues/438
try:
os.remove("outline.geojson")
except OSError:
pass
outlines_geojson.to_file("outline.geojson", driver="GeoJSON")
area_polygon = "outline.geojson"
##### End test
# Search for products matching query
products = api.query(area=geojson_to_wkt(read_geojson(area_polygon)),
date=datum,
platformname="Sentinel-2",
producttype="S2MSI1C",
cloudcoverpercentage=cloudcover)
# count number of products matching query
print(
"Tiles found:",
api.count(area=geojson_to_wkt(read_geojson(area_polygon)),
date=datum,
platformname="Sentinel-2",
producttype="S2MSI1C",
cloudcoverpercentage=cloudcover), ", Total size: ",
api.get_products_size(products), "GB. Now downloading those tiles")
# downloading all products
download_zip = api.download_all(products, directory_path=downloadpath)
# Unzip files, delete
for key in download_zip[0].keys():
with zipfile.ZipFile(download_zip[0][key]['path']) as zip_file:
for member in zip_file.namelist():
filename = os.path.basename(member)
if not filename:
continue
source = zip_file.open(member)
source.close()
os.remove(download_zip[0][key]['path'])
def next(self):
if len(self.queue) > 0:
return self.api.download(self.queue.popleft())
else:
#get 2 more
dataSample = self.data[self.index]
urlData = extractURLData(dataSample["aws_s3"][0])
# search by polygon, time, and SciHub query keywords
footprint = geojson_to_wkt(toFeatureCollection(mgrs.toLatLon(str.encode(urlData["mgrs"]))))
products = self.api.query(footprint,
date=(urlData["date"]["from"], urlData["date"]["to"]),
platformname='Sentinel-2',
cloudcoverpercentage=(0, 30))
#get first 2 keys which are the product IDs you need to download
products = list(products)[:2]
print('PRODUCTS ',products)
if len(products) > 0:
self.updateIndex(self.index)
if len(products) > 1:
# if found 2 products, add one to the queue
self.queue.append(products[0])
imageData = self.api.download(products[1])
return products[1]
else:
imageData = self.api.download(products[0])
return products[0]
else:
# if didn't find any products, updateIndex and go to next data
self.updateIndex(self.index)
self.next()
def get_product_identifiers(date):
date = date.date()
footprint = geojson_to_wkt(
read_geojson(os.path.join(DIRNAME, 'norway_sweden.json')))
scenes = api.query(footprint,
date=(date, date + datetime.timedelta(days=1)),
raw='Sentinel-3')
selected_scenes = []
for scene in scenes.values():
id = scene['identifier']
#Only S3A SLSTR level 1
if 'S3A_SL_1_RBT___' not in id:
continue
poly1 = shapely.wkt.loads(footprint)
poly2 = shapely.wkt.loads(scene['footprint'])
intersection = poly1.intersection(poly2)
#Require some overlap
if intersection.area < 10:
continue
hour_of_day = int(id.split('_')[7].split('T')[1][:2])
#Limit to morning passes
if not (7 <= hour_of_day <= 11):
continue
selected_scenes.append(id)
return selected_scenes
def sentinel_query_task(parameters, task_id=None):
def validate_name(string: str):
return string.replace('_', '-')
def order_by(order_by='ingestiondate', ascending=False):
prefix_order_by = '+' if ascending else '-'
order_by = prefix_order_by + order_by
return order_by
task = DownloadTask.objects.get(pk=task_id)
platformname = parameters['products']
longitude_min, longitude_max = parameters['longitude']
latitude_min, latitude_max = parameters['latitude']
extent = Polygon([[(longitude_min, latitude_max),
(longitude_max, latitude_max),
(longitude_max, latitude_min),
(longitude_min, latitude_min),
(longitude_min, latitude_max)]])
extent = geojson_to_wkt(extent)
order_by = order_by('ingestiondate', ascending=False)
api = SentinelAPI(user, password, api_url)
products = api.query(extent,
date=(parameters['time']),
platformname='SENTINEL-1',
producttype='GRD',
orbitdirection='DESCENDING')
products_df = api.to_dataframe(products)
task.execution_start = datetime.now()
task.update_status('WAIT', "Download Sentinel Query. ")
return products_df
def products():
"""A fixture for tests that need some non-specific set of products as input."""
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')),
("20151219", "20151228"))
return products
def geojsonLocation(geoJson_name, geoJon_location):
footprintLocation = (geoJon_location + geoJson_name + ".geojson"
) # geojson file location
footprint = geojson_to_wkt(read_geojson(footprintLocation))
return footprintLocation, footprint
def raw_products():
"""A fixture for tests that need some non-specific set of products in the form of a raw response as input."""
api = SentinelAPI(**_api_auth)
raw_products = api._load_query(
api.format_query(geojson_to_wkt(read_geojson('tests/map.geojson')),
"20151219", "20151228"))
return raw_products
def query(self):
def order_by(order_by='ingestiondate', ascending=False):
prefix_order_by = '+' if ascending else '-'
order_by = prefix_order_by + order_by
return order_by
longitude_min, longitude_max = self.parameters['longitude']
latitude_min, latitude_max = self.parameters['latitude']
# extent = Polygon([[(latitude_min, longitude_min),
# (latitude_max, longitude_min),
# (latitude_max, longitude_max),
# (latitude_min, longitude_max),
# (latitude_min, longitude_min)]])
extent = Polygon([[(longitude_min, latitude_min),
(longitude_max, latitude_min),
(longitude_max, latitude_max),
(longitude_min, latitude_max),
(longitude_min, latitude_min)]])
order_by = order_by('ingestiondate', ascending=False)
extent = geojson_to_wkt(extent)
products = self.api.query(extent,
date=(self.parameters['time']),
area_relation='Intersects',
cloudcoverpercentage=(0, 10),
platformname='Sentinel-2',
order_by=order_by) #producttype='S2MSI2A'
self.products = products
products_df = self.api.to_dataframe(products)
if products_df.empty:
raise Exception("No products were found")
self.products_df = products_df
return self.products_df
def getscenes_test_dataset():
logger = logging.getLogger('root')
api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
logging.info(api.api_url)
# download single scene by known product id
#api.download(<product_id>)
t0 = datetime.now() - timedelta(months=8)
tf = datetime.now()
# search by polygon, time, and SciHub query keywords
footprint = geojson_to_wkt(
read_geojson(home['parameters'] + '/madalena.geojson'))
products_s1a = api.query(footprint,
date=(date(t0.year, t0.month,
t0.day), date(tf.year, tf.month,
tf.day)),
producttype="GRD",
platformname='Sentinel-1')
for item in products_s1a:
logging.info(products_s1a[item]['title'])
tests1In = home['scratch'] + "/test_dataset/s1a_scenes/in"
# download all results from the search
api.download_all(products_s1a, directory_path=tests1In)
def query(self, platformname, min_lon, max_lon, min_lat, max_lat,
start_time, end_time):
def order_by(order_by='ingestiondate', ascending=False):
prefix_order_by = '+' if ascending else '-'
order_by = prefix_order_by + order_by
return order_by
min_lon, max_lon, min_lat, max_lat = self.validate_coord(
min_lon), self.validate_coord(max_lon), self.validate_coord(
min_lat), self.validate_coord(max_lat)
extent = Polygon([[(min_lon, max_lat), (max_lon, max_lat),
(max_lon, min_lat), (min_lon, min_lat),
(min_lon, max_lat)]])
extent = geojson_to_wkt(extent)
start_time, end_time = self.validate_date(
start_time), self.validate_date(end_time)
order_by = order_by('ingestiondate', ascending=False)
products = self.api.query(extent,
date=(start_time, end_time),
area_relation='Intersects',
cloudcoverpercentage=(0, 10),
platformname=platformname,
order_by=order_by) #producttype='S2MSI2A'
self.products = products
products_df = self.api.to_dataframe(products)
self.products_df = products_df
return self.products_df
def fetch_sentinel_data(self, start_date, end_date):
api = SentinelAPI('mpc', 'q11g33h99',
'https://scihub.copernicus.eu/dhus')
footprint = geojson_to_wkt(geojson.loads(self.geo_json_data))
print(start_date, end_date)
print footprint
products = api.query(footprint,
date=(start_date, end_date),
platformname='Sentinel-1',
producttype='GRD') #, orbitdirection='ASCENDING'
for i, (product_id, props) in enumerate(products.items()):
file_name = props['title'] + self.ending
if any(file_name in s for s in self.processed_grd_data):
print file_name
outputfilename = file_name[17:32] + "_" + file_name[
0:16] + '.tif'
filepath_out = self.output_folder + 'geotifs/'
command = "LD_LIBRARY_PATH=. ~/snap/bin/gpt sentinel_scripts/autoGraph.xml -Pinputfile=\"" + self.preprocessed_path + file_name + "\"" + " -Poutputfile=\"" + filepath_out + outputfilename + "\""
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
shell=True)
(output, err) = p.communicate()
def get_scenes():
logger = logging.getLogger('root')
api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
logging.info(api.api_url)
# download single scene by known product id
#api.download(<product_id>)
tf = datetime.now()
# tf=datetime(2018,1,10)
t0 = tf - timedelta(days=7)
# get_s1_orbits(t0,tf)
# search by polygon, time, and SciHub query keywords
footprint = geojson_to_wkt(
read_geojson(home['parameters'] + '/extent_' + location['region'] +
'.geojson'))
products_s1a = api.query(footprint,
date=(date(t0.year, t0.month,
t0.day), date(tf.year, tf.month,
tf.day)),
producttype="GRD",
platformname='Sentinel-1')
for item in products_s1a:
logging.info(products_s1a[item]['title'])
# download all results from the search
# already downloaded files are skipped
api.download_all(products_s1a, directory_path=sarIn)
def sent2_query(user, passwd, geojsonfile, start_date, end_date, cloud=50):
"""
Fetches a list of Sentienl-2 products
Parameters
-----------
user : string
Username for ESA hub. Register at https://scihub.copernicus.eu/dhus/#/home
passwd : string
password for the ESA Open Access hub
geojsonfile : string
Path to a geometry file containing a polygon of the outline of the area you wish to download.
Can be a geojson (.json/.geojson) or a shapefile (.shp)
See www.geojson.io for a tool to build these.
start_date : string
Date of beginning of search in the format YYYY-MM-DDThh:mm:ssZ (ISO standard)
end_date : string
Date of end of search in the format yyyy-mm-ddThh:mm:ssZ
See https://www.w3.org/TR/NOTE-datetime, or use cehck_for_s2_data_by_date
cloud : int, optional
The maximum cloud clover percentage (as calculated by Copernicus) to download. Defaults to 50%
Returns
-------
products : dict
A dictionary of Sentinel-2 granule products that are touched by your AOI polygon, keyed by product ID.
Returns both level 1 and level 2 data.
Notes
-----
If you get a 'request too long' error, it is likely that your polygon is too complex. The following functions
download by granule; there is no need to have a precise polygon at this stage.
"""
# Originally by Ciaran Robb
api = SentinelAPI(user, passwd)
geom = ogr.Open(geojsonfile)
with TemporaryDirectory() as td:
if not _is_4326(geom):
reproj_geom_path = os.path.join(td, "temp.shp")
reproject_vector(geojsonfile, os.path.join(td, "temp.shp"), 4326)
geojsonfile = reproj_geom_path
if geojsonfile.endswith("json"):
footprint = geojson_to_wkt(read_geojson(geojsonfile))
elif geojsonfile.endswith("shp"):
footprint = shapefile_to_wkt(geojsonfile)
else:
raise InvalidGeometryFormatException("Please provide a .json, .geojson or a .shp as geometry.")
log.info("Sending Sentinel-2 query:\nfootprint: {}\nstart_date: {}\nend_date: {}\n cloud_cover: {} ".format(
footprint, start_date, end_date, cloud))
products = api.query(footprint,
date=(start_date, end_date), platformname="Sentinel-2",
cloudcoverpercentage="[0 TO {}]".format(cloud))
return products
def seed_db():
db.session.add(User(email="*****@*****.**"))
ecobici_manager = EcobiciManager.instance()
station_list = ecobici_manager.get_station_list()
for station in station_list.get('stations', []):
geo = 'POINT({} {})'.format(
station.get("location").get("lon"),
station.get("location").get("lat"))
db.session.add(
Station(id=station.get("id"),
name=station.get("name"),
address=station.get("address"),
geo=geo,
zip_code=station.get("zipCode"),
station_type=station.get("stationType")))
shapes = {'cdmx': 'roi_extent_latlon.json'}
for name, filename in shapes.items():
source = os.path.join('tejidos/static', filename)
footprint = geojson_to_wkt(read_geojson(source))
db.session.add(Shape(name=name, geo=footprint))
db.session.commit()
def create_sentinel_xml(self):
f = open("sentinel_scripts/autoGraph.xml", 'w')
parsing_xml = (open(
self.template_path + "/snap/output_1.xml",
'r').read()).format(polygon_string=str(
geojson_to_wkt(geojson.loads(self.geo_json_data))))
f.write(parsing_xml)
def product_list(coordinates,start_date,end_date) :
polygon_geojson = Polygon([coordinates])
tmp_file = tempfile.mkstemp(suffix='.geojson')
with open(tmp_file[1], 'w') as outfile:
geojson.dump(polygon_geojson, outfile)
footprint = geojson_to_wkt(read_geojson(tmp_file[1]))
products = api.query(footprint,date=("%sT00:00:00.000Z"%(start_date),"%sT23:59:59.999Z"%(end_date)),producttype='S2MSI1C')
return products
def products():
"""A fixture for tests that need some non-specific set of products as input."""
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson('tests/map.geojson')),
"20151219", "20151228"
)
return products
def raw_products():
"""A fixture for tests that need some non-specific set of products in the form of a raw response as input."""
api = SentinelAPI(**_api_auth)
raw_products = api._load_query(api.format_query(
geojson_to_wkt(read_geojson('tests/map.geojson')),
"20151219", "20151228")
)
return raw_products
def products():
"""A fixture for tests that need some non-specific set of products as input."""
api = SentinelAPI(**_api_auth)
with my_vcr.use_cassette('products_fixture',
decode_compressed_response=False):
products = api.query(
geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')),
("20151219", "20151228"))
return products
def get_data(name,
AREA,
start_date='20190101',
end_date='20190128',
platformname='Sentinel-2',
processinglevel='Level-2A'):
#apiのクエリに次のが必要。中身はPOLYGON((..))っていうstr型
footprint_geojson = geojson_to_wkt(
read_geojson('./location/' + str(name) + '.geojson'))
# use sentinelAPI
user = ''
password = ''
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus')
products = api.query(
footprint_geojson,
date=(start_date, end_date), #取得希望期間の入力
platformname=platformname,
processinglevel=processinglevel, ##2016年はL1C
cloudcoverpercentage=(0, 100)) #被雲率（0％〜100％）
print("この期間の画像の数は" + str(len(products)) + "枚です")
#この後で雲の被覆率ごとにデータを並べ替えて、1個ダウンロードする。
products_gdf = api.to_geodataframe(products)
products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'],
ascending=[True]).head()
#ファイルがまだなければデータダウンロード。作業ディレクトリにzipファイルがダウンロードされる
for i in range(3): #3回までチャレンジ
uuid = products_gdf_sorted.iloc[i]["uuid"]
product_title = products_gdf_sorted.iloc[i][
"title"] #S2A_MSIL2A_20190101T01405... みたいな
product_date = products_gdf_sorted.iloc[i]["summary"].split(
',')[0].split()[1][:10]
print("この期間で1番被覆率が低いのは" + product_date + "日")
if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True:
print("新規にデータをダウンロードします")
try:
api.download(uuid)
except:
print("ダウンロード不可")
else:
break
else:
break
if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True:
#ダウンロードしたzipファイルを解凍
#str(product_title) + '.SAFE っていうフォルダが生成される
file_name = str(product_title) + '.zip'
with zipfile.ZipFile(file_name) as zf:
zf.extractall()
shutil.move(str(product_title) + '.zip', './data_geo/')
shutil.move(str(product_title) + '.SAFE', './data_geo/')
return product_title
def get_footprint(api, path, data, delta):
footprint = geojson_to_wkt(read_geojson(path))
products = api.query(footprint,
date=((date(data[0], data[1], data[2]) -
timedelta(delta)),
date(data[0], data[1], data[2])),
platformname='Sentinel-2',
producttype='S2MSI2A*')
# cloudcoverpercentage=(0, 30))
return products
def test_order_by():
api = SentinelAPI(**_api_auth)
products = api.query(geojson_to_wkt(
read_geojson(FIXTURES_DIR + '/map.geojson')), ("20151219", "20151228"),
platformname="Sentinel-2",
cloudcoverpercentage=(0, 10),
order_by="cloudcoverpercentage, -beginposition")
assert len(products) == 3
vals = [x["cloudcoverpercentage"] for x in products.values()]
assert sorted(vals) == vals
def fetch_datasets(self, download_type='ROI_polygon', roi_polygon=None, startdate=None, enddate=None, cloudcover_max=5,
platformname='Sentinel-2'):
"""
:param download_type:
:param username:
:param password:
:param roi_polygon:
:param startdate:
:param enddate:
:param cloudcover_max:
:param platformname:
:return:
"""
if startdate:
self.startdate = startdate
if enddate:
self.enddate = enddate
if roi_polygon:
self.roi_polygon = roi_polygon
self.api = SentinelAPI(self.username, self.password, 'https://scihub.copernicus.eu/dhus')
product_id = None
if download_type == 'full':
if product_id:
self.api.download(product_id)
else:
print('product id required')
if download_type == 'ROI_polygon':
if roi_polygon.split('.')[-1] == 'geojson':
file_obj = open(self.roi_polygon, "r")
json_data = file_obj.read()
file_obj.close()
json_data = json_data.encode().decode('utf-8-sig') # Remove utf-8 data if any present in the file
json_data = json.loads(json_data)
footprint = geojson_to_wkt(json_data)
if platformname == 'Sentinel-2':
self.products = self.api.query(footprint,
date=(self.startdate, self.enddate),
platformname=platformname,
cloudcoverpercentage=(0, cloudcover_max))
self.list_products = list(self.products.items())
elif platformname == 'Sentinel-1':
self.products = self.api.query(footprint,
date=(self.startdate, self.enddate),
platformname=platformname)
self.list_products = list(self.products.items())
print(len(self.list_products), ' products found')
def geojson_handler(request):
if request.is_ajax() and request.method == 'POST':
polygon_data = json.loads(
request.FILES.get('geojson_data').file.read().decode('UTF-8'))
coordinates = polygon_data.get(
'features')[0]['geometry']['coordinates']
if coordinates:
user_data.footprint = geojson_to_wkt(dict(polygon_data))
print(user_data.footprint)
return HttpResponse('OK')
return HttpResponse("Couldn't load data")
def _downloader(self, datelist, sensor, continent, obs_folder, obs_lst,
uid, psw):
local_obs_date = self._available_dec(obs_lst)
api = SentinelAPI(uid, psw)
if continent == 'Africa':
footprint = geojson_to_wkt(
read_geojson(r'L:/HSL/poi/AOI_Africa.geojson'))
elif continent == 'West_Asia':
footprint = geojson_to_wkt(
read_geojson(r'L:/HSL/poi/AOI_West_Asia.geojson'))
elif continent == 'Europe':
footprint = geojson_to_wkt(
read_geojson(r'L:/HSL/poi/AOI_Europe.geojson'))
products = api.query(footprint,
filename=f'S3{sensor}_*',
producttype='SY_2_V10___',
date=(datelist[0], datelist[-1]))
products_df = api.to_dataframe(products)
if products_df.size != 0:
products_df_sorted = products_df.sort_values(['ingestiondate'],
ascending=[True])
products_df_sorted = products_df_sorted.head(24)
download_list = products_df_sorted[
~products_df_sorted['beginposition'].
isin(local_obs_date)].drop_duplicates(subset=['beginposition'],
keep='last')
if download_list.size != 0:
downloaded = api.download_all(download_list.index,
directory_path=obs_folder,
n_concurrent_dl=2)
if len(downloaded[2]) != 0:
print('Info ! Some dataset were not downloaded')
self._unzipper(obs_folder)
else:
pass
def test_order_by():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')),
("20151219", "20151228"),
platformname="Sentinel-2",
cloudcoverpercentage=(0, 10),
order_by="cloudcoverpercentage, -beginposition"
)
assert len(products) == 3
vals = [x["cloudcoverpercentage"] for x in products.values()]
assert sorted(vals) == vals
def test_get_products_invalid_json():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.post('https://scihub.copernicus.eu/apihub/search?format=json',
text="{Invalid JSON response",
status_code=200)
with pytest.raises(SentinelAPIError) as excinfo:
api.query(area=geojson_to_wkt(
read_geojson(FIXTURES_DIR + "/map.geojson")),
date=("20151219", "20151228"),
platformname="Sentinel-2")
assert excinfo.value.msg == "Invalid API response."
def main():
if len(sys.argv) < 2:
print(" Usage: python3 get_past_scenes.py [year] [month]")
return 1
api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
logging.info(api.api_url)
t0 = datetime(int(sys.argv[1]), int(sys.argv[2]), 1, 0, 0, 0)
tf = t0 + timedelta(days=12)
# search by polygon, time, and SciHub query keywords
footprint = geojson_to_wkt(
read_geojson(home['parameters'] + '/extent_' + location['region'] +
'.geojson'))
products_s1a = api.query(footprint,
date=(date(t0.year, t0.month,
t0.day), date(tf.year, tf.month,
tf.day)),
producttype="GRD",
platformname='Sentinel-1')
unavailable = []
for uuid in products_s1a:
product_info = api.get_product_odata(uuid)
if any(product_info['title'] in s for s in os.listdir(sarIn)):
logging.info('Skipping ' + uuid + '. Already exists in ' + sarIn)
continue
logging.info('Is ' + uuid + ' online?')
logging.info(product_info['Online'])
if not product_info['Online']:
logging.info('Requesting unavailable uuids')
api.download(uuid)
unavailable = unavailable + [uuid]
else:
logging.info('Downloading available uuids')
api.download(uuid, directory_path=sarIn)
logging.info(
'Sleeping 30 minutes (the API does not allow intensive requests)')
time.sleep(30 * 60)
while len(unavailable) > 0:
for uuid in unavailable:
product_info = api.get_product_odata(uuid)
if product_info['Online']:
logging.info(uuid + ' is available! Downloading:')
api.download(uuid, directory_path=sarIn)
unavailable.remove(uuid)
time.sleep(600)
return 0
def __init__(self, geojson_path='sample-polygone.geojson', max_threads=4):
# private
self.__lock__ = Lock()
self.__dbconn__ = DBClient()
self.__api__ = self.__get_sentinel_api__()
self.__geojson_path__ = geojson_path
self.__footprint__ = geojson_to_wkt(read_geojson(geojson_path))
self.__threadpool__ = ThreadPool(max_threads)
self.__img_path__ = self.__get_env_var__('img_path')
self.__tiff_path__ = self.__get_env_var__('tiff_path')
# public
self.max_threads = max_threads
self.__fetch_measurements__()
def test_s2_cloudcover():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson('tests/map.geojson')),
"20151219", "20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]"
)
assert len(products) == 3
product_ids = list(products)
assert product_ids[0] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef"
assert product_ids[1] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert product_ids[2] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_get_products_invalid_json():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.post(
'https://scihub.copernicus.eu/apihub/search?format=json',
text="{Invalid JSON response", status_code=200
)
with pytest.raises(SentinelAPIError) as excinfo:
api.query(
area=geojson_to_wkt(read_geojson(FIXTURES_DIR + "/map.geojson")),
date=("20151219", "20151228"),
platformname="Sentinel-2"
)
assert excinfo.value.msg == "Invalid API response."
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')),
(datetime(2014, 10, 10), datetime(2014, 12, 31)), producttype="GRD"
)
footprints = api.to_geojson(products)
for footprint in footprints['features']:
assert not footprint['geometry'].errors()
with open(FIXTURES_DIR + '/expected_search_footprints_s1.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson('tests/map.geojson')),
datetime(2014, 10, 10), datetime(2014, 12, 31), producttype="GRD"
)
footprints = api.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s1.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_s2_cloudcover():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')),
("20151219", "20151228"),
platformname="Sentinel-2",
cloudcoverpercentage=(0, 10)
)
assert len(products) == 3
product_ids = list(products)
assert product_ids[0] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef"
assert product_ids[1] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert product_ids[2] == "0848f6b8-5730-4759-850e-fc9945d42296"
# For order-by test
vals = [x["cloudcoverpercentage"] for x in products.values()]
assert sorted(vals) != vals
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt
# -------------------------------------------
# necessary information:
user = '******'
password = '******'
# YYYYMMDD
start_date = '20150101'
end_date = '20180207'
# map.geojson with boundary coordinates
# just generate and save as "map.geojson" using: --- http://geojson.io ---
geojson_path = 'directory\\to\\the\\file\\map.geojson'
# where to save the data
save_path = 'directory\\to\\the\\save_folder'
# -------------------------------------------
# connect to the API / SentinelHub
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus', show_progressbars=True)
footprint = geojson_to_wkt(read_geojson(geojson_path))
products = api.query(footprint,date=(start_date, end_date), platformname='Sentinel-2', producttype='S2MSI2Ap')
print 'Number of images: {}'.format(len(products))
api.download_all(products, save_path)
def test_boundaries_latitude_more(fixture_path):
with pytest.raises(ValueError):
geojson_to_wkt(read_geojson(fixture_path('map_boundaries_lat.geojson')))
def test_boundaries_longitude_less(fixture_path):
with pytest.raises(ValueError):
geojson_to_wkt(read_geojson(fixture_path('map_boundaries_lon.geojson')))
def test_get_coordinates(fixture_path):
wkt = ('POLYGON((-66.2695 -8.0592,-66.2695 0.7031,'
'-57.3047 0.7031,-57.3047 -8.0592,-66.2695 -8.0592))')
assert geojson_to_wkt(read_geojson(fixture_path('map.geojson'))) == wkt
assert geojson_to_wkt(read_geojson(fixture_path('map_z.geojson'))) == wkt
assert geojson_to_wkt(read_geojson(fixture_path('map_nested.geojson'))) == wkt
def test_wkt(geojson_path):
return geojson_to_wkt(read_geojson(geojson_path))
def test_get_coordinates():
wkt = ('POLYGON((-66.2695 -8.0592,-66.2695 0.7031,'
'-57.3047 0.7031,-57.3047 -8.0592,-66.2695 -8.0592))')
assert geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')) == wkt
assert geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map_z.geojson')) == wkt
def test_get_coordinates():
wkt = ('POLYGON ((-66.2695312 -8.0592296, -66.2695312 0.7031074, ' +
'-57.3046875 0.7031074, -57.3046875 -8.0592296, -66.2695312 -8.0592296))')
assert geojson_to_wkt(read_geojson('tests/map.geojson')) == wkt
assert geojson_to_wkt(read_geojson('tests/map_z.geojson')) == wkt
