def test_small_query():
api = SentinelAPI(**_api_kwargs)
api.query(**_small_query)
assert api.last_query == (
'(beginPosition:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z]) '
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")')
assert api.last_status_code == 200
def test_small_query():
api = SentinelAPI(**_api_kwargs)
api.query(**_small_query)
assert api.last_query == (
'(beginPosition:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z]) '
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")')
assert api.last_status_code == 200
def downloadSentinel(user, pw, aoi, start, stop):
# For image before November 16th, 2015
curl = pycurl.Curl()
curl.setopt(pycurl.CAINFO, certifi.where())
curl.setopt(pycurl.URL, 'https://scihub.copernicus.eu/dhus')
curl.perform()
# For image before November 16th, 2015
api = SentinelAPI(user, pw, 'https://scihub.copernicus.eu/dhus')
AOI = KMLtoGeoJason.kml2geojson(aoi)
api.query(get_coordinates(AOI), start, stop, producttype='GRD')
# footprint generation of all found images:
a=api.get_footprints()
name = AOI[:-8]+"_S1footprint.geojson"
foot = open(name, "w")
foot.write(dumps(a, indent=2) + "\n")
foot.close()
##
## with open(name) as f:
## contents = f.read()
## display(contents)
# selected image download and unzip:
imageId = raw_input("Insert Sentinel-1 image id: ")
output_img = 'C:\Users\ithaca\Documents\Magda\Tool_MIE\SENTINEL-1_TOOL\Immagini_grandi'
s1 = api.download(imageId, output_img)
path = os.path.dirname(s1)
with zipfile.ZipFile(s1, "r") as z:
z.extractall(path)
def test_large_query():
api = SentinelAPI(**_api_kwargs)
api.query(**_large_query)
assert api.last_query == (
'(beginPosition:[2015-01-01T00:00:00Z TO 2015-12-31T00:00:00Z]) '
'AND (footprint:"Intersects(POLYGON((0 0,0 10,10 10,10 0,0 0)))")')
assert api.last_status_code == 200
assert len(api.products) > api.max_rows
def test_SentinelAPI_connection():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query('0 0,1 1,0 1,0 0', datetime(2015, 1, 1), datetime(2015, 1, 2))
assert api.url == 'https://scihub.copernicus.eu/apihub/search?format=json&rows=15000'
assert api.last_query == '(beginPosition:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z]) ' + \
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")'
assert api.content.status_code == 200
def test_large_query():
api = SentinelAPI(**_api_kwargs)
api.query(**_large_query)
assert api.last_query == (
'(beginPosition:[2015-01-01T00:00:00Z TO 2015-12-31T00:00:00Z]) '
'AND (footprint:"Intersects(POLYGON((0 0,0 10,10 10,10 0,0 0)))")')
assert api.last_status_code == 200
assert len(api.products) > api.page_size
def test_SentinelAPI_wrong_credentials():
api = SentinelAPI(
"wrong_user",
"wrong_password"
)
with pytest.raises(SentinelAPIError) as excinfo:
api.query(**_small_query)
assert excinfo.value.http_status == 401
def query_and_create(self, start=None, end=None):
'''Query the last scene available on period filtered by start time
and end dates. By default start is current time and end is 7 days ago.'''
end = end or datetime.utcnow()
start = start or datetime.utcnow() - timedelta(days=7)
coords = ','.join([('%f %f' % coord) for coord in self.geom.coords[0]])
scenes_created = []
try:
api = SentinelAPI(settings.SENTINEL_USER, settings.SENTINEL_PASSWORD, settings.SENTINEL_API_URL)
print('sentinel initialized on %s, with %s - %s' %
(settings.SENTINEL_API_URL, settings.SENTINEL_USER, settings.SENTINEL_PASSWORD))
except AttributeError:
api = SentinelAPI(settings.SENTINEL_USER, settings.SENTINEL_PASSWORD)
print('sentinel initialized on %s, with %s - %s' %
(settings.SENTINEL_API_URL, ettings.SENTINEL_USER, settings.SENTINEL_PASSWORD))
print('sentinelsat query -s %s -e %s coords %s q %s' % (start, end, coords, self.query))
if self.query:
query = dict([i.split('=') for i in self.query.split(',')])
api.query(coords, start, end, **query)
else:
api.query(coords, start, end)
features = api.get_footprints()['features']
print('%s features found' % len(features))
for feature in features:
product_id = feature['properties']['product_id']
try:
Scene.objects.get(product=product_id)
print('Scene of product %s already exists' % product_id)
except Scene.DoesNotExist:
print('Creating scene with data: %s' % feature)
scene = Scene.objects.create(
product=product_id,
identifier=feature['properties']['identifier'],
date=datetime.strptime(
feature['properties']['date_beginposition'],
'%Y-%m-%dT%H:%M:%S.%fZ'
),
polarisation=feature['properties']['polarisationmode'],
orbit_direction=feature['properties']['orbitdirection'],
sensor_mode=feature['properties']['sensoroperationalmode'],
product_type=feature['properties']['producttype'],
sat=feature['properties']['platformname'],
geom=Polygon(feature['geometry']['coordinates'][0]))
print('Scene of product %s created' % product_id)
scenes_created.append(scene)
return scenes_created
def test_get_products_size():
api = SentinelAPI(
environ.get('SENTINEL_USER'),
environ.get('SENTINEL_PASSWORD')
)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228", platformname="Sentinel-2"
)
assert api.get_products_size() == 63.58
def test_SentinelAPI_wrong_credentials():
api = SentinelAPI("wrong_user", "wrong_password")
with pytest.raises(SentinelAPIError) as excinfo:
api.query('0 0,1 1,0 1,0 0', datetime(2015, 1, 1),
datetime(2015, 1, 2))
assert excinfo.value.http_status == 401
with pytest.raises(SentinelAPIError):
api.get_products_size()
api.get_products()
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
api.query(
get_coordinates('tests/map.geojson'),
datetime(2014, 10, 10), datetime(2014, 12, 31), producttype="GRD"
)
with open('tests/expected_search_footprints_s1.geojson', 'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_footprints_s2():
api = SentinelAPI(**_api_auth)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228", platformname="Sentinel-2"
)
with open('tests/expected_search_footprints_s2.geojson', 'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_SentinelAPI_connection():
api = SentinelAPI(**_api_auth)
api.query(**_small_query)
assert api.url.startswith(
'https://scihub.copernicus.eu/apihub/search?format=json&rows={rows}'.
format(rows=api.max_rows))
assert api.last_query == (
'(beginPosition:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z]) '
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")')
assert api.last_status_code == 200
def test_SentinelAPI_connection():
api = SentinelAPI(
environ.get('SENTINEL_USER'),
environ.get('SENTINEL_PASSWORD')
)
api.query('0 0,1 1,0 1,0 0', datetime(2015, 1, 1), datetime(2015, 1, 2))
assert api.url == 'https://scihub.copernicus.eu/apihub/search?format=json&rows=15000' + \
'&q=(beginPosition:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z]) ' + \
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")'
assert api.content.status_code == 200
def test_SentinelAPI_wrong_credentials():
api = SentinelAPI(
"wrong_user",
"wrong_password"
)
api.query('0 0,1 1,0 1,0 0', datetime(2015, 1, 1), datetime(2015, 1, 2))
assert api.content.status_code == 401
with pytest.raises(ValueError):
api.get_products_size()
api.get_products()
def test_footprints():
api = SentinelAPI(
environ.get('SENTINEL_USER'),
environ.get('SENTINEL_PASSWORD')
)
api.query(get_coordinates('tests/map.geojson'), datetime(2014, 10, 10), datetime(2014, 12, 31), producttype="GRD")
expected_footprints = geojson.loads(open('tests/expected_search_footprints.geojson', 'r').read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_footprints_s1():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query(get_coordinates('tests/map.geojson'),
datetime(2014, 10, 10),
datetime(2014, 12, 31),
producttype="GRD")
with open('tests/expected_search_footprints_s1.geojson',
'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_get_products_invalid_json():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.post('https://scihub.copernicus.eu/apihub/search?format=json',
text="{Invalid JSON response",
status_code=200)
with pytest.raises(SentinelAPIError) as excinfo:
api.query(area=get_coordinates("tests/map.geojson"),
initial_date="20151219",
end_date="20151228",
platformname="Sentinel-2")
assert excinfo.value.msg == "API response not valid. JSON decoding failed."
def test_footprints_s2():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
with open('tests/expected_search_footprints_s2.geojson',
'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.get_footprints()) == set(expected_footprints)
def test_s2_cloudcover():
api = SentinelAPI(**_api_auth)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]"
)
assert len(api.get_products()) == 3
assert api.get_products()[0]["id"] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef"
assert api.get_products()[1]["id"] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert api.get_products()[2]["id"] == "0848f6b8-5730-4759-850e-fc9945d42296"
class Watcher:
def __init__(self, location):
self.options = Options(location)
self.sql = Esa_Sql(self.options)
self.api = SentinelAPI(self.options.user, self.options.password)
self.products = []
check_and_clean_log_file()
def watch(self):
try:
self.find_candidate_products()
self.filter_for_unknown_products()
self.filter_for_subscription_intersection()
self.insert_products_in_db()
finally:
self.options.set_running('0')
log.info("Done")
def find_candidate_products(self):
log.info("Finding candidate products at ESA")
self.candidate_products = self.api.query(limit=self.options.num_back,
producttype="SLC",
ingestiondate=(f"{self.options.last_search_time}Z",
datetime.isoformat(datetime.now())+"Z"))
self.candidate_products.update(self.api.query(limit=self.options.num_back,
producttype="GRD",
ingestiondate=(f"{self.options.last_search_time}Z",
datetime.isoformat(datetime.now())+"Z")))
self.options.update_last_search_time()
log.info("Inspecting {} products".format(len(self.candidate_products)))
def filter_for_unknown_products(self):
for product in self.candidate_products:
if not self.sql.check_pg_db_for_product(
self.candidate_products[product]['identifier']):
self.products.append(
{'granule': self.candidate_products[product]['identifier'],
'url': self.candidate_products[product]['link_icon'],
'location': self.candidate_products[product]['footprint']}
)
log.info(f"{product['identifier']} is unknown to ASF")
def filter_for_subscription_intersection(self):
for product in self.products:
if not self.sql.check_hyp3_db_for_intersecting_subscription(product):
log.info(f"{product['identifier']} did not match any Hyp3 subscriptions")
self.products.remove(product)
def insert_products_in_db(self):
for product in self.products:
self.sql.insert_product_in_db(product)
def test_get_products_size():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
assert api.get_products_size() == 63.58
api.query_raw(
"S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E")
assert len(api.get_products()) > 0
# Rounded to zero
assert api.get_products_size() == 0
def test_SentinelAPI_connection():
api = SentinelAPI(**_api_auth)
api.query(**_small_query)
assert api.url.startswith(
'https://scihub.copernicus.eu/apihub/search?format=json&rows={rows}'.format(
rows=api.page_size
)
)
assert api.last_query == (
'(beginPosition:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z]) '
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")')
assert api.last_status_code == 200
def search(
user, password, tile, geojson, start, end, download, md5,
sentinel1, sentinel2, cloud, footprints, path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if cloud:
search_kwargs.update(
{"platformname": "Sentinel-2",
"cloudcoverpercentage": "[0 TO %s]" % cloud})
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
if tile:
api.query(point = get_coordinates(tile = tile), initial_date = start, end_date = end, **search_kwargs)
elif geojson:
api.query(area = get_coordinates(geojson_file = geojson), initial_date = start, end_date = end, **search_kwargs)
else:
raise ValueError("Either a --geojson or --tile arguments must be given.")
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
result = api.download_all(path, checksum=md5)
if md5 is True:
corrupt_scenes = [(path, info["id"]) for path, info in result.items() if info is not None]
if len(corrupt_scenes) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for corrupt_tuple in corrupt_scenes:
outfile.write("%s : %s\n" % corrupt_tuple)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
print('---')
print(
'%s scenes found with a total size of %.2f GB' %
(len(api.get_products()), api.get_products_size()))
def test_s2_cloudcover():
api = SentinelAPI(
environ.get('SENTINEL_USER'),
environ.get('SENTINEL_PASSWORD')
)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]"
)
assert len(api.get_products()) == 2
assert api.get_products()[0]["id"] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert api.get_products()[1]["id"] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_s2_cloudcover():
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]")
assert len(api.get_products()) == 3
assert api.get_products(
)[0]["id"] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef"
assert api.get_products(
)[1]["id"] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert api.get_products(
)[2]["id"] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_get_products_size():
api = SentinelAPI(**_api_auth)
api.query(
get_coordinates('tests/map.geojson'),
"20151219", "20151228", platformname="Sentinel-2"
)
assert api.get_products_size() == 63.58
# reset products
api.products = []
# load new very small query
api.load_query("S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E")
assert len(api.get_products()) > 0
# Rounded to zero
assert api.get_products_size() == 0
def test_to_geopandas():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
gdf = api.to_geodataframe(products)
class SentinelDL(object):
def __init__(self, user, password, platform='Sentinel-2'):
self.api = SentinelAPI(user, password,
'https://scihub.copernicus.eu/dhus')
self.platform = platform
def query_bb(self, bb, ts0, ts1, ccrange=(0, 30)):
"""Search by polygon, time and cloud range
return products
"""
# search by polygon, time, and Hub query keywords
if type(bb) is not tuple:
raise TypeError(
"Bounding box must be a tuple of min/max values of Lon/Lat")
footprint = geojson_to_wkt(SentinelDL.geojson_bb(*bb))
return self.api.query(footprint,
date=(ts0, ts1),
platformname=self.platform,
cloudcoverpercentage=ccrange)
@staticmethod
def geojson_bb(lon0, lat0, lon1, lat1):
"""Get a bounding box area as a GeoJSON formatted dictionary
"""
poly = Polygon([(lon0, lat0), (lon1, lat0), (lon1, lat1), (lon1, lat0),
(lon0, lat0)])
return geopandas.GeoSeries([poly]).__geo_interface__
def download_s1(user, password, dir_raw, dir_nc, start_date, end_date,
footprint):
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus/')
#footprint = "POLYGON((73 11, 74 11, 74 14, 73 14, 73 11))"
products = api.query(footprint,
date=(start_date, end_date),
producttype='GRD')
#print(products)
for product in products:
productInfo = api.get_product_odata(product)
title = productInfo['title']
print(title)
file_nc = os.path.join(dir_nc, "%s_VV.nc" % title)
file_wkt = os.path.join(os.path.dirname(dir_nc), 'wkt',
"%s.wkt" % title)
if not os.path.exists(file_wkt):
pFootPrint = productInfo['footprint']
file = open(file_wkt, "a")
file.write(pFootPrint)
file.close()
if not os.path.exists(file_nc):
api.download(product, dir_raw, checksum=True)
def download_error_image(img_date,geo_img,img_id,username,password):
'''
After read error file(image_error.txt) you can get image info which you failed from COG Sentinel-2, you can use this info with this function
if you have more than 1 image, you can download with for loop.
You can find img_date, geo_img and img_id information in image_error.txt file.
api,target_image_id=download_error_image(img_date,geo_img,img_id,username,password)
api.download(target_image_id,directory_path='.')
api.download('7be30c50-31fc-48c4-ab45-fddea9be7877',directory_path='.')
if you get error like >> Product 7be30c50-31fc-48c4-ab45-fddea9be7877 is not online. Triggering retrieval from long term archive.
Go to https://sentinelsat.readthedocs.io/en/stable/api.html#lta-products
username and password should be string
'''
api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
day_before =img_date- datetime.timedelta(days=1)
day_after =img_date + datetime.timedelta(days=1)
footprint = geojson_to_wkt(geo_img)
products = api.query(footprint,
#date = ('20181219', date(2018, 12, 29)),
date=(day_before,day_after),
platformname = 'Sentinel-2',
)
sat_df=api.to_geodataframe(products)
result=sat_df.loc[sat_df['title']==img_id]
return api,result.index.values[0]
def Sen2Download(self, dprofile):
download_profile_args = [
dprofile.username,
dprofile.password,
dprofile.daysdiff,
dprofile.shape_file_path,
dprofile.download_dir,
dprofile.concurrency,
]
username, password, daysdiff, shape_file, directory_path, concurrency = download_profile_args
logger.info(
f'Sentinel-1 Downloads starting with dprofile = {dprofile}')
api = SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus')
#shapefileto wkt
footprint = geojson_to_wkt(read_geojson(shape_file))
#dates to search
end_date = datetime.datetime.now()
daysdiff = datetime.timedelta(days=daysdiff)
start_date = end_date - daysdiff
#Search for data
products = api.query(footprint,
date=(start_date, end_date),
platformname='Sentinel-2',
producttype='S2MSI1C',
cloudcoverpercentage=(0, 30))
self.DownloadProducts(self, products, dprofile)
def test_get_products_size():
api = SentinelAPI(**_api_auth)
api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
assert api.get_products_size() == 63.58
# reset products
api.products = []
# load new very small query
api.load_query(
"S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E")
assert len(api.get_products()) > 0
# Rounded to zero
assert api.get_products_size() == 0
def retrieve_mata_data(filename,username,password):
url = 'https://scihub.copernicus.eu/dhus'
info = filename.split('_')
satellite = info[0]
mode = info[1]
product = info[2]
orbitnumber = np.int(info[7])
time_start = np.int(info[5].split('T')[0])-1
time_end = str(np.int(time_start+2))
time_start = str(time_start)
api = SentinelAPI(username, password, url)
products = api.query(
beginposition=(time_start,time_end),
platformname='Sentinel-1',
producttype=product,
sensoroperationalmode=mode,
polarisationmode='VV VH',
orbitnumber=orbitnumber
)
products_df = api.to_dataframe(products)
index = -1
for i in range(len(products_df)):
if products_df['title'][i] in filename:
index = i
return products_df.iloc[index]
def test_get_products_invalid_json():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.get(
'https://scihub.copernicus.eu/apihub/search?format=json&rows=15000&q=(beginPosition:[2015-12-19T00:00:00Z TO 2015-12-28T00:00:00Z]) AND (footprint:"Intersects(POLYGON((-66.2695312 -8.0592296,-66.2695312 0.7031074,-57.3046875 0.7031074,-57.3046875 -8.0592296,-66.2695312 -8.0592296)))") AND (platformname:Sentinel-2)',
text="Invalid JSON response", status_code=200
)
api.query(
area=get_coordinates("tests/map.geojson"),
initial_date="20151219",
end_date="20151228",
platformname="Sentinel-2"
)
with pytest.raises(ValueError) as val_err:
api.get_products()
assert val_err.value.message == "API response not valid. JSON decoding failed."
def get_products(login_json, coordinates, date_start, date_end, download_dir):
with open(login_json, 'r') as fp:
LOGIN_INFO = json.load(fp)
USER_NAME, PASSWORD = list(LOGIN_INFO.values())
# connect to the API
api = SentinelAPI(USER_NAME, PASSWORD, 'https://scihub.copernicus.eu/dhus')
# define a map polygon
geojson = Polygon(coordinates=coordinates)
# search by polygon, time, and Hub query keywords
footprint = geojson_to_wkt(geojson)
dates = (date_start, date_end) # (date(2018, 4, 1), date(2018, 4, 11))
# June to July maps
products = api.query(
footprint,
date=dates,
platformname='Sentinel-2',
# producttype='S2MSI2A',
area_relation='Intersects', # area of interest is inside footprint
cloudcoverpercentage=(0, 40))
# download all results from the search
api.download_all(products, directory_path=download_dir)
# product_id = list(products.keys())[0]
# api.download(id=product_id, directory_path=download_dir)
# GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries
return api.to_geodataframe(products)
def test_get_products_invalid_json():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.post(
'https://scihub.copernicus.eu/apihub/search?format=json',
text="{Invalid JSON response", status_code=200
)
with pytest.raises(SentinelAPIError) as excinfo:
api.query(
area=get_coordinates("tests/map.geojson"),
initial_date="20151219",
end_date="20151228",
platformname="Sentinel-2"
)
api.get_products()
assert excinfo.value.msg == "API response not valid. JSON decoding failed."
def search(
user, password, geojson, start, end, download, md5,
sentinel1, sentinel2, cloud, footprints, path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if cloud:
search_kwargs.update(
{"platformname": "Sentinel-2",
"cloudcoverpercentage": "[0 TO %s]" % cloud})
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
api.query(get_coordinates(geojson), start, end, **search_kwargs)
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
result = api.download_all(path, checksum=md5)
if md5 is True:
corrupt_scenes = [(path, info["id"]) for path, info in result.items() if info is not None]
if len(corrupt_scenes) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for corrupt_tuple in corrupt_scenes:
outfile.write("%s : %s\n" % corrupt_tuple)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
print('---')
print(
'%s scenes found with a total size of %.2f GB' %
(len(api.get_products()), api.get_products_size()))
def test_to_pandas():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
df = api.to_dataframe(products)
assert 'S2A_OPER_PRD_MSIL1C_PDMC_20151228T112701_R110_V20151227T142229_20151227T142229' in df.index
def download_scenes(period):
date_from = period.date_from
date_to = period.date_to
# Check if result has already been done
scene_filename = 's1_{dfrom}_{dto}.tif'.format(
dfrom=period.date_from.strftime('%Y%m'),
dto=period.date_to.strftime('%Y%m'))
scene_path = os.path.join(RESULTS_PATH, scene_filename)
if os.path.exists(scene_path):
print(
"Sentinel-1 mosaic for period {}-{} already done:".format(
date_from, date_to), scene_path)
return
# Prepare API client for download
api = SentinelAPI(settings.SCIHUB_USER, settings.SCIHUB_PASS,
settings.SCIHUB_URL)
# Query scenes
footprint = geojson_to_wkt(read_geojson(AOI_PATH))
products = api.query(footprint,
date=(date_from, date_to),
platformname='Sentinel-1',
producttype='GRD',
polarisationmode='VV VH',
orbitdirection='ASCENDING')
for k, p in products.items():
print((k, p['summary']))
os.makedirs(S1_RAW_PATH, exist_ok=True)
# Filter already downloaded products
products_to_download = {
k: v
for k, v in products.items() if not os.path.exists(
os.path.join(S1_RAW_PATH, '{}.zip'.format(v['title'])))
}
# Download products
results = api.download_all(products_to_download,
directory_path=S1_RAW_PATH)
products = list(products.values())
# Process the images of each product
with mp.Pool(settings.S1_PROC_NUM_JOBS) as pool:
pool.map(process_product, products)
# Create a median composite from all images of each band, generate extra
# bands and concatenate results into a single multiband imafge.
superimpose(products)
median(products, period)
generate_vvvh(period)
concatenate_results(period)
clip_result(period)
clean_temp_files(period)
def s5down(datum=str(dt.date.today()), product_type='no2'):
# this function will download s5p data of given product_type for given date
# e.g. s5down('2019-08-15','no2')
if product_type == 'no2':
strtype = 'L2__NO2___'
if product_type == 'aerosols':
strtype = 'L2__AER_AI'
datum = dt.datetime.strptime(datum, '%Y-%m-%d').date()
time_in = dt.datetime.combine(datum, dt.time(0, 0))
time_out = dt.datetime.combine(datum, dt.time(23, 59))
api = SentinelAPI('s5pguest', 's5pguest',
'https://s5phub.copernicus.eu/dhus')
#coordinates for CZ:
footprint = 'POLYGON((12.278971773041526 48.69059060056844,18.98957262575027 48.69059060056844,18.98957262575027 51.081759060281655,12.278971773041526 51.081759060281655,12.278971773041526 48.69059060056844))'
products = api.query(footprint,
date=(time_in, time_out),
platformname='Sentinel-5',
producttype=strtype)
print('there are ' + str(len(products)) + ' products found')
a = api.download_all(products)
geotiffs = []
for product_ID in iter(products):
product_path = a[0][product_ID]['path']
print('converting ' + product_path + ' to geotiff')
geotiffs.append(s5p2geotiff(product_path, product_type))
if not geotiffs:
print('some error happened, no geotiffs generated')
clean_downloaded(products, a)
return None
tifstring = ''
for tif in geotiffs:
tifstring = tifstring + ' ' + tif
print('merging geotiffs to ' + str(datum) +
'.tif and cropping for CZ extents')
outfile = str(datum) + '.' + product_type + '.tif'
tmpfile = 'tmp.tif'
os.system(
'gdal_merge.py -o merged.tif -of GTiff -ul_lr 11.3867 51.4847 19.943 47.7933 -a_nodata 9999 '
+ tifstring)
if product_type == 'no2':
#need to compute 1000x
gdal_calc = 'gdal_calc.py -A merged.tif --outfile=' + tmpfile + ' --calc="(A*1000 > 0)*(A * 1000 < 0.7)*(A * 1000)" --overwrite'
print(gdal_calc)
os.system(gdal_calc)
else:
tmpfile = 'merged.tif'
#now oversample using cubic..
gdalwarp = 'gdalwarp -tr 0.015 0.015 -r cubicspline -dstnodata 9999 -srcnodata 9999 ' + tmpfile + ' ' + outfile
#gdalwarp = 'gdalwarp -s_srs EPSG:4326 -t_srs EPSG:4326 -tr 0.015 0.015 -r cubicspline -dstnodata 9999 -srcnodata 9999 temp1000.tif '+outfile
print(gdalwarp)
os.system(gdalwarp)
print('(the file will be also saved as {}.tif)'.format(product_type))
copyfile(outfile, '../data/' + product_type + '.tif')
#cleaning
clean_downloaded(products, a)
return geotiffs
def test_s2_cloudcover():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2",
cloudcoverpercentage="[0 TO 10]")
assert len(products) == 3
assert products[0]["id"] == "6ed0b7de-3435-43df-98bf-ad63c8d077ef"
assert products[1]["id"] == "37ecee60-23d8-4ec2-a65f-2de24f51d30e"
assert products[2]["id"] == "0848f6b8-5730-4759-850e-fc9945d42296"
def test_to_dict():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
dictionary = api.to_dict(products)
# check the type
assert isinstance(dictionary, dict)
# check if dictionary has id key
assert 'S2A_OPER_PRD_MSIL1C_PDMC_20151228T112701_R110_V20151227T142229_20151227T142229' in dictionary
def download_s2(user, password, dir_raw, dir_nc, start_date, end_date, footprint, pr_status):
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus/')
#footprint = "POLYGON((73 11, 74 11, 74 14, 73 14, 73 11))"
#products = api.query(footprint, date=(start_date, end_date), producttype='S2MSI1C')
products = api.query(footprint, date=(start_date, end_date),
producttype='S2MSI1C',cloudcoverpercentage = (0,20))
#print(products)
l = ['S2A_MSIL1C_20180601T051651_N0206_R062_T43PFN_20180601T082308', 'S2A_MSIL1C_20180621T051651_N0206_R062_T43PFN_20180621T081647', 'S2B_MSIL1C_20180613T050649_N0206_R019_T43PFN_20180613T084228',
'S2A_MSIL1C_20180601T051651_N0206_R062_T43PFP_20180601T082308', 'S2A_MSIL1C_20180621T051651_N0206_R062_T43PFP_20180621T081647', 'S2B_MSIL1C_20180613T050649_N0206_R019_T43PFP_20180613T084228',
'S2A_MSIL1C_20180608T050651_N0206_R019_T43PFN_20180608T084904', 'S2A_MSIL1C_20180628T050651_N0206_R019_T43PFN_20180628T081023', 'S2B_MSIL1C_20180616T051649_N0206_R062_T43PFN_20180616T090733',
'S2A_MSIL1C_20180608T050651_N0206_R019_T43PFP_20180608T084904', 'S2A_MSIL1C_20180628T050651_N0206_R019_T43PFP_20180628T081023', 'S2B_MSIL1C_20180616T051649_N0206_R062_T43PFP_20180616T090733',
'S2A_MSIL1C_20180611T051651_N0206_R062_T43PFN_20180611T081245', 'S2B_MSIL1C_20180603T050649_N0206_R019_T43PFN_20180603T084545', 'S2B_MSIL1C_20180623T050649_N0206_R019_T43PFN_20180623T084444',
'S2A_MSIL1C_20180611T051651_N0206_R062_T43PFP_20180611T081245', 'S2B_MSIL1C_20180603T050649_N0206_R019_T43PFP_20180603T084545', 'S2B_MSIL1C_20180623T050649_N0206_R019_T43PFP_20180623T084444',
'S2A_MSIL1C_20180618T050651_N02206_R019_T43PFN_20180618T085607', 'S2B_MSIL1C_20180606T051649_N0206_R062_T43PFN_20180606T104751', 'S2B_MSIL1C_20180626T051649_N0206_R062_T43PFN_20180626T090058',
'S2A_MSIL1C_20180618T050651_N0206_R019_T43PFP_20180618T085607', 'S2B_MSIL1C_20180606T051649_N0206_R062_T43PFP_20180606T104751', 'S2B_MSIL1C_20180626T051649_N0206_R062_T43PFP_20180626T090058']
for product in products:
productInfo = api.get_product_odata(product)
title = productInfo['title']
if title in l:
continue
tileNo_time = '%s_%s' % (title.split('_')[5], title.split('_')[2])
try:
downloadFlag = not pr_status[tileNo_time]
except KeyError:
pr_status[tileNo_time] = False
downloadFlag =True
print "no error"
#file_nc = os.path.join(dir_nc, "%s_VV.nc"%os.path.basename(title).split("_")[4])
#file_nc = os.path.join(dir_nc, "%s_VV.nc" % title[17:48])
file_wkt = os.path.join(os.path.dirname(dir_nc), "wkt/%s.wkt" % tileNo_time)
if not os.path.exists(file_wkt):
pFootPrint = productInfo['footprint']
file = open(file_wkt, "a")
file.write(pFootPrint)
file.close()
if downloadFlag and not title in l:
api.download(product, dir_raw, checksum=True)
l.append(title)
return pr_status
def period_worker(start_date, period, ii_t, auth, CATALOG_ROOT):
api = SentinelAPI(auth['U'], auth['P'])
min_date = start_date + timedelta(days=period * ii_t)
max_date = start_date + timedelta(days=period * (ii_t + 1))
S2_L2A_products = api.query(date=(min_date, max_date),
platformname='Sentinel-2',
producttype='S2MSI2A')
S2_L2A_df = pd.DataFrame.from_dict(S2_products, orient='index')
S2_L2A_df.to_parquet(
os.path.join(
CATALOG_ROOT,
f'S2_L2A_{str(ii_t)}_{min_date.isoformat()[0:10]}_{max_date.isoformat()[0:10]}.parquet'
))
S2_L1C_products = api.query(date=(min_date, max_date),
platformname='Sentinel-2',
producttype='S2MSI1C')
S2_L1C_df = pd.DataFrame.from_dict(S2_L1C_products, orient='index')
S2_L1C_df.to_parquet(
os.path.join(
CATALOG_ROOT,
f'S2_L1C_{str(ii_t)}_{min_date.isoformat()[0:10]}_{max_date.isoformat()[0:10]}.parquet'
))
S1_products = api.query(date=(min_date, max_date),
platformname='Sentinel-1',
producttype='GRD')
S1_df = pd.DataFrame.from_dict(S1_products, orient='index')
S1_df.to_parquet(
os.path.join(
CATALOG_ROOT,
f'S1_{str(ii_t)}_{min_date.isoformat()[0:10]}_{max_date.isoformat()[0:10]}.parquet'
))
def test_footprints_s2():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
with open('tests/expected_search_footprints_s2.geojson',
'r') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(api.to_geojson(products)) == set(expected_footprints)
def _search_on_hub(user, password, hub_address, **search_keywords):
# Connect to the hub and search
try:
print(SentinelAPI.format_query(**search_keywords))
hub = SentinelAPI(user, password, hub_address)
products = hub.query(**search_keywords)
except SentinelAPIError as e:
print(e)
print(SentinelAPI.format_query(**search_keywords))
products = {}
return products
def search(user, password, geojson, start, end, download, footprints, path, query):
"""Search for Sentinel-1 products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password)
if query is not None:
query = dict([i.split('=') for i in query.split(',')])
api.query(get_coordinates(geojson), start, end, **query)
else:
api.query(get_coordinates(geojson), start, end)
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
api.download_all(path)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
def test_SentinelAPI():
api = SentinelAPI(
environ.get('SENTINEL_USER'),
environ.get('SENTINEL_PASSWORD')
)
api.query('0 0,1 1,0 1,0 0', datetime(2015, 1, 1), datetime(2015, 1, 2))
assert api.url == 'https://scihub.esa.int/dhus/search?format=json&rows=15000' + \
'&q=(ingestionDate:[2015-01-01T00:00:00Z TO 2015-01-02T00:00:00Z]) ' + \
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")'
assert api.content.status_code == 200
now = datetime.now()
api.format_url('0 0,1 1,0 1,0 0', end_date=now)
last_24h = format_date(now - timedelta(hours=24))
assert api.url == 'https://scihub.esa.int/dhus/search?format=json&rows=15000' + \
'&q=(ingestionDate:[%s TO %s]) ' % (last_24h, format_date(now)) + \
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))")'
api.format_url('0 0,1 1,0 1,0 0', end_date=now, producttype='SLC')
assert api.url == 'https://scihub.esa.int/dhus/search?format=json&rows=15000' + \
'&q=(ingestionDate:[%s TO %s]) ' % (last_24h, format_date(now)) + \
'AND (footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))") ' + \
'AND (producttype:SLC)'
def cli(user, password, geometry, start, end, uuid, name, download, sentinel, producttype,
instrument, cloud, footprints, path, query, url, order_by, limit):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your Copernicus Open Access Hub user and password, you must pass a geojson file
containing the geometry of the area you want to search for or the UUIDs of the products. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
_set_logger_handler()
api = SentinelAPI(user, password, url)
search_kwargs = {}
if sentinel and not (producttype or instrument):
search_kwargs["platformname"] = "Sentinel-" + sentinel
if instrument and not producttype:
search_kwargs["instrumentshortname"] = instrument
if producttype:
search_kwargs["producttype"] = producttype
if cloud:
if sentinel not in ['2', '3']:
logger.error('Cloud cover is only supported for Sentinel 2 and 3.')
raise ValueError('Cloud cover is only supported for Sentinel 2 and 3.')
search_kwargs["cloudcoverpercentage"] = (0, cloud)
if query is not None:
search_kwargs.update((x.split('=') for x in query.split(',')))
if geometry is not None:
search_kwargs['area'] = geojson_to_wkt(read_geojson(geometry))
if uuid is not None:
uuid_list = [x.strip() for x in uuid.split(',')]
products = {}
for productid in uuid_list:
try:
products[productid] = api.get_product_odata(productid)
except SentinelAPIError as e:
if 'Invalid key' in e.msg:
logger.error('No product with ID \'%s\' exists on server', productid)
elif name is not None:
search_kwargs["identifier"] = name
products = api.query(order_by=order_by, limit=limit, **search_kwargs)
else:
start = start or "19000101"
end = end or "NOW"
products = api.query(date=(start, end),
order_by=order_by, limit=limit, **search_kwargs)
if footprints is True:
footprints_geojson = api.to_geojson(products)
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
product_infos, failed_downloads = api.download_all(products, path)
if len(failed_downloads) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for failed_id in failed_downloads:
outfile.write("%s : %s\n" % (failed_id, products[failed_id]['title']))
else:
for product_id, props in products.items():
if uuid is None:
logger.info('Product %s - %s', product_id, props['summary'])
else: # querying uuids has no summary key
logger.info('Product %s - %s - %s MB', product_id, props['title'],
round(int(props['size']) / (1024. * 1024.), 2))
if uuid is None:
logger.info('---')
logger.info('%s scenes found with a total size of %.2f GB',
len(products), api.get_products_size(products))
def search(
user, password, geojson, start, end, download, md5, sentinel, producttype,
instrument, sentinel1, sentinel2, cloud, footprints, path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if sentinel and not (producttype or instrument):
search_kwargs.update({"platformname": "Sentinel-" + sentinel})
if instrument and not producttype:
search_kwargs.update({"instrumentshortname": instrument})
if producttype:
search_kwargs.update({"producttype": producttype})
if cloud:
if sentinel not in ['2', '3']:
logger.error('Cloud cover is only supported for Sentinel 2 and 3.')
raise ValueError('Cloud cover is only supported for Sentinel 2 and 3.')
search_kwargs.update({"cloudcoverpercentage": "[0 TO %s]" % cloud})
# DEPRECATED: to be removed with next major release
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
logger.info('DEPRECATED: Please use --sentinel instead')
# DEPRECATED: to be removed with next major release
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
logger.info('DEPRECATED: Please use --sentinel instead')
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
wkt = geojson_to_wkt(read_geojson(geojson))
products = api.query(wkt, start, end, **search_kwargs)
if footprints is True:
footprints_geojson = api.to_geojson(products)
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
product_infos, failed_downloads = api.download_all(products, path, checksum=md5)
if md5 is True:
if len(failed_downloads) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for failed_id in failed_downloads:
outfile.write("%s : %s\n" % (failed_id, products[failed_id]['title']))
else:
for product_id, props in products.items():
logger.info('Product %s - %s' % (product_id, props['summary']))
logger.info('---')
logger.info(
'%s scenes found with a total size of %.2f GB' %
(len(products), api.get_products_size(products)))
