def query_images(username,
password,
start_date,
end_date,
aoi,
platform,
max_cloud_cover,
query_status=None):
global images
password = base64.b64decode(password)
if max_cloud_cover is not None or max_cloud_cover != '':
cloud_cover_percentage = '[0 TO ' + max_cloud_cover + ']'
try:
# Try to connect to ApiHub, else use SciHub
try:
api = SentinelAPI(username, password,
'https://scihub.copernicus.eu/apihub/')
except:
api = SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus/')
# Search images by polygon, start and end date, and SciHub query keywords
if platform == None or platform == '' and max_cloud_cover == None or max_cloud_cover == '':
images = api.query(get_coordinates(aoi), start_date, end_date)
elif platform != None and max_cloud_cover != None or max_cloud_cover != '':
if platform == 'Sentinel-1':
images = api.query(get_coordinates(aoi),
start_date,
end_date,
platformname=platform)
else:
images = api.query(get_coordinates(aoi),
start_date,
end_date,
platformname=platform,
cloudcoverpercentage=cloud_cover_percentage)
elif platform == None or platform == '' and max_cloud_cover != None or max_cloud_cover != '':
images = api.query(get_coordinates(aoi),
start_date,
end_date,
cloudcoverpercentage=cloud_cover_percentage)
else:
images = api.query(get_coordinates(aoi),
start_date,
end_date,
platformname=platform)
if query_status is not None:
if len(images) == 1:
query_status.set(str(len(images)) + " image found.")
else:
query_status.set(str(len(images)) + " images found.")
except SentinelAPIError as apiError:
error_message = apiError.msg.replace('\n', ' ').replace('# ', '')
if query_status is not None:
query_status.set(error_message)
return api, images
def search(
user, password, tile, geojson, start, end, download, md5,
sentinel1, sentinel2, cloud, footprints, path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if cloud:
search_kwargs.update(
{"platformname": "Sentinel-2",
"cloudcoverpercentage": "[0 TO %s]" % cloud})
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
if tile:
api.query(point = get_coordinates(tile = tile), initial_date = start, end_date = end, **search_kwargs)
elif geojson:
api.query(area = get_coordinates(geojson_file = geojson), initial_date = start, end_date = end, **search_kwargs)
else:
raise ValueError("Either a --geojson or --tile arguments must be given.")
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
result = api.download_all(path, checksum=md5)
if md5 is True:
corrupt_scenes = [(path, info["id"]) for path, info in result.items() if info is not None]
if len(corrupt_scenes) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for corrupt_tuple in corrupt_scenes:
outfile.write("%s : %s\n" % corrupt_tuple)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
print('---')
print(
'%s scenes found with a total size of %.2f GB' %
(len(api.get_products()), api.get_products_size()))
def downloadSentinel(user, pw, aoi, start, stop):
# For image before November 16th, 2015
curl = pycurl.Curl()
curl.setopt(pycurl.CAINFO, certifi.where())
curl.setopt(pycurl.URL, 'https://scihub.copernicus.eu/dhus')
curl.perform()
# For image before November 16th, 2015
api = SentinelAPI(user, pw, 'https://scihub.copernicus.eu/dhus')
AOI = KMLtoGeoJason.kml2geojson(aoi)
api.query(get_coordinates(AOI), start, stop, producttype='GRD')
# footprint generation of all found images:
a=api.get_footprints()
name = AOI[:-8]+"_S1footprint.geojson"
foot = open(name, "w")
foot.write(dumps(a, indent=2) + "\n")
foot.close()
##
## with open(name) as f:
## contents = f.read()
## display(contents)
# selected image download and unzip:
imageId = raw_input("Insert Sentinel-1 image id: ")
output_img = 'C:\Users\ithaca\Documents\Magda\Tool_MIE\SENTINEL-1_TOOL\Immagini_grandi'
s1 = api.download(imageId, output_img)
path = os.path.dirname(s1)
with zipfile.ZipFile(s1, "r") as z:
z.extractall(path)
def test_to_geopandas():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
gdf = api.to_geodataframe(products)
def test_to_pandas():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
df = api.to_dataframe(products)
assert 'S2A_OPER_PRD_MSIL1C_PDMC_20151228T112701_R110_V20151227T142229_20151227T142229' in df.index
def test_get_products_size():
api = SentinelAPI(
environ.get('SENTINEL_USER'),
environ.get('SENTINEL_PASSWORD')
)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228", platformname="Sentinel-2"
)
assert api.get_products_size() == 63.58
def test_footprints():
api = SentinelAPI(
environ.get('SENTINEL_USER'),
environ.get('SENTINEL_PASSWORD')
)
api.query(get_coordinates('tests/map.geojson'), datetime(2014, 10, 10), datetime(2014, 12, 31), producttype="GRD")
expected_footprints = geojson.loads(open('tests/expected_search_footprints.geojson', 'r').read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_to_dict():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
dictionary = api.to_dict(products)
# check the type
assert isinstance(dictionary, dict)
# check if dictionary has id key
assert 'S2A_OPER_PRD_MSIL1C_PDMC_20151228T112701_R110_V20151227T142229_20151227T142229' in dictionary
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
api.query(
get_coordinates('tests/map.geojson'),
datetime(2014, 10, 10), datetime(2014, 12, 31), producttype="GRD"
)
with open('tests/expected_search_footprints_s1.geojson', 'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_s2_cloudcover():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]")
assert len(products) == 3
assert products[0]["id"] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef"
assert products[1]["id"] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert products[2]["id"] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_footprints_s2():
api = SentinelAPI(**_api_auth)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228", platformname="Sentinel-2"
)
with open('tests/expected_search_footprints_s2.geojson', 'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_footprints_s2():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
with open('tests/expected_search_footprints_s2.geojson',
'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_s2_cloudcover():
api = SentinelAPI(**_api_auth)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]"
)
assert len(api.get_products()) == 3
assert api.get_products()[0]["id"] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef"
assert api.get_products()[1]["id"] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert api.get_products()[2]["id"] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_footprints_s1():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query(get_coordinates('tests/map.geojson'),
datetime(2014, 10, 10),
datetime(2014, 12, 31),
producttype="GRD")
with open('tests/expected_search_footprints_s1.geojson',
'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_footprints_s2():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
with open('tests/expected_search_footprints_s2.geojson',
'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.to_geojson(products)) == set(expected_footprints)
def test_get_products_invalid_json():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.post('https://scihub.copernicus.eu/apihub/search?format=json',
text="{Invalid JSON response",
status_code=200)
with pytest.raises(SentinelAPIError) as excinfo:
api.query(area=get_coordinates("tests/map.geojson"),
initial_date="20151219",
end_date="20151228",
platformname="Sentinel-2")
assert excinfo.value.msg == "API response not valid. JSON decoding failed."
def test_get_products_size():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
assert api.get_products_size() == 63.58
api.query_raw(
"S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E")
assert len(api.get_products()) > 0
# Rounded to zero
assert api.get_products_size() == 0
def test_s2_cloudcover():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]")
assert len(api.get_products()) == 3
assert api.get_products(
)[0]["id"] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef"
assert api.get_products(
)[1]["id"] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert api.get_products(
)[2]["id"] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_s2_cloudcover():
api = SentinelAPI(
environ.get('SENTINEL_USER'),
environ.get('SENTINEL_PASSWORD')
)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]"
)
assert len(api.get_products()) == 2
assert api.get_products()[0]["id"] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert api.get_products()[1]["id"] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_get_products_size():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
assert api.get_products_size(products) == 63.58
# reset products
# load new very small query
products = api.load_query(
"S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E")
assert len(products) > 0
# Rounded to zero
assert api.get_products_size(products) == 0
def search(user, password, geojson, start, end, download, footprints, path, query):
"""Search for Sentinel-1 products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password)
if query is not None:
query = dict([i.split('=') for i in query.split(',')])
api.query(get_coordinates(geojson), start, end, **query)
else:
api.query(get_coordinates(geojson), start, end)
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
api.download_all(path)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
def test_get_products_size():
api = SentinelAPI(**_api_auth)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228", platformname="Sentinel-2"
)
assert api.get_products_size() == 63.58
# reset products
api.products = []
# load new very small query
api.load_query("S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E")
assert len(api.get_products()) > 0
# Rounded to zero
assert api.get_products_size() == 0
def test_get_products_invalid_json():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.get(
'https://scihub.copernicus.eu/apihub/search?format=json&rows=15000&q=(beginPosition:[2015-12-19T00:00:00Z TO 2015-12-28T00:00:00Z]) AND (footprint:"Intersects(POLYGON((-66.2695312 -8.0592296,-66.2695312 0.7031074,-57.3046875 0.7031074,-57.3046875 -8.0592296,-66.2695312 -8.0592296)))") AND (platformname:Sentinel-2)',
text="Invalid JSON response", status_code=200
)
api.query(
area=get_coordinates("tests/map.geojson"),
initial_date="20151219",
end_date="20151228",
platformname="Sentinel-2"
)
with pytest.raises(ValueError) as val_err:
api.get_products()
assert val_err.value.message == "API response not valid. JSON decoding failed."
def test_get_products_invalid_json():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.post(
'https://scihub.copernicus.eu/apihub/search?format=json',
text="{Invalid JSON response", status_code=200
)
with pytest.raises(SentinelAPIError) as excinfo:
api.query(
area=get_coordinates("tests/map.geojson"),
initial_date="20151219",
end_date="20151228",
platformname="Sentinel-2"
)
api.get_products()
assert excinfo.value.msg == "API response not valid. JSON decoding failed."
def all_in_one(user,
password,
area,
api_url='https://scihub.copernicus.eu/apihub/',
path='./',
download='no',
initial_date=None,
end_date=datetime.now(),
**keywords):
### login information Copernicus Open Access Hub (https://scihub.copernicus.eu/dhus/#/home)
api = MySentinelAPI(user, password, api_url)
### read area information
try:
geojson = get_coordinates(area)
except IOError:
area = area.split(',')
area = api.get_area(float(area[0]), float(area[1]), float(area[2]),
float(area[3]))
geojson = area
### Sentinel data search
products = api.query(geojson, initial_date, end_date, **keywords)
# products_df = api.to_dataframe(products)
print(api._last_query)
print(
'%s product results for your query. The products need %s Gb disk space'
) % (len(products), api.get_products_size(products))
print([i['title'] for i in products if 'title' in i])
### download all query products
if download == 'yes':
result = api.download_all(products,
directory_path=path,
max_attempts=10,
checksum=True,
check_existing=True,
show_progress=False)
# print('Downloaded files:')
# print(result.viewkeys())
return result
# print(products_df.index.values)
# return products_df
# print([i['title'] for i in products if 'title' in i])
return products
def search(
user, password, geojson, start, end, download, md5,
sentinel1, sentinel2, cloud, footprints, path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if cloud:
search_kwargs.update(
{"platformname": "Sentinel-2",
"cloudcoverpercentage": "[0 TO %s]" % cloud})
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
api.query(get_coordinates(geojson), start, end, **search_kwargs)
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
result = api.download_all(path, checksum=md5)
if md5 is True:
corrupt_scenes = [(path, info["id"]) for path, info in result.items() if info is not None]
if len(corrupt_scenes) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for corrupt_tuple in corrupt_scenes:
outfile.write("%s : %s\n" % corrupt_tuple)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
print('---')
print(
'%s scenes found with a total size of %.2f GB' %
(len(api.get_products()), api.get_products_size()))
from sentinelsat.sentinel import SentinelAPI, get_coordinates, format_date
from datetime import date, datetime, timedelta
import logging
import json
logging.basicConfig(format='%(message)s', level='DEBUG')
# connect to the API
api = SentinelAPI('s3guest', 's3guest', 'https://scihub.copernicus.eu/s3')
# download single scene by known product id
#api.download(<product_id>)
# search by polygon, time, and SciHub query keywords
#products = api.query(,'20151219', date(2015, 12, 29), platformname = 'Sentinel-2', cloudcoverpercentage = '[0 TO 30]'})
products = api.query(get_coordinates('map.geojson'),
initial_date=datetime(2017, 2, 24),
end_date=datetime(2017, 2, 25))
#api.get_products_size(products)
print(api.get_products_size(products))
# download all results from the search
#api.download_all(products)
# GeoJSON FeatureCollection containing footprints and metadata of the scenes
fp = api.get_footprints(products)
with open('footprints.json', 'w') as outfile:
#json.dump({'numbers':n, 'strings':s, 'x':x, 'y':y}, outfile, indent=4)
json.dump(fp, outfile, indent=4)
for possible values and probably more parameters please visit
https://scihub.copernicus.eu/twiki/do/view/SciHubUserGuide/3FullTextSearch#Search_Keywords
"""
print(datetime.now())
### login information Copernicus Open Access Hub (https://scihub.copernicus.eu/dhus/#/home)
api = SentinelAPI('user',
'password',
api_url='https://scihub.copernicus.eu/apihub/')
### Sentinel data search
# products = api.query(area=get_coordinates('/media/tweiss/Work/coordinates_wallerfing.geojson'), initial_date='20151219', end_date=datetime(2015, 12, 24), platformname='Sentinel-1', producttype='GRD')
# products = api.query(get_area(48.68, 48.70, 12.89 ,12.999), '20151219', datetime(2015, 12, 29), platformname='Sentinel-1')
products = api.query(area=get_coordinates(
'/media/tweiss/Work/python_code/MULTIPLY/sentinel_download/coordinates_test_sites/coordinates_test_sites'
),
initial_date='20170628',
end_date='20170730',
platformname='Sentinel-1',
producttype='SLC')
print(api._last_query)
print('%s product results for your query. The products need %s Gb disk space'
) % (len(products), api.get_products_size(products))
### convert to Pandas DataFrame
products_df = api.to_dataframe(products)
print(products_df.index.values)
### download all query products
path = '/media/nas_data/Thomas/Wallerfing/Sentinel_1_data'
def test_get_coordinates():
coords = ('-66.2695312 -8.0592296,-66.2695312 0.7031074,' +
'-57.3046875 0.7031074,-57.3046875 -8.0592296,-66.2695312 -8.0592296')
assert get_coordinates('tests/map.geojson') == coords
def test_get_coordinates():
coords = (
'-66.2695312 -8.0592296,-66.2695312 0.7031074,' +
'-57.3046875 0.7031074,-57.3046875 -8.0592296,-66.2695312 -8.0592296')
assert get_coordinates('tests/map.geojson') == coords
