def test_api_query_format_escape_spaces():
query = SentinelAPI.format_query(ingestiondate=('NOW-1DAY', 'NOW'))
assert query == 'ingestiondate:[NOW-1DAY TO NOW]'
query = SentinelAPI.format_query(ingestiondate='[NOW-1DAY TO NOW]')
assert query == 'ingestiondate:[NOW-1DAY TO NOW]'
query = SentinelAPI.format_query(ingestiondate=' [NOW-1DAY TO NOW] ')
assert query == 'ingestiondate:[NOW-1DAY TO NOW]'
query = SentinelAPI.format_query(relativeorbitnumber=' {101 TO 103} ')
assert query == 'relativeorbitnumber:{101 TO 103}'
query = SentinelAPI.format_query(filename='S3A_OL_2* ')
assert query == 'filename:S3A_OL_2*'
query = SentinelAPI.format_query(timeliness='Non Time Critical')
assert query == r'timeliness:Non\ Time\ Critical'
query = SentinelAPI.format_query(timeliness='Non\tTime\tCritical')
assert query == r'timeliness:Non\ Time\ Critical'
api = SentinelAPI(**_api_auth)
assert api.count(timeliness='Non Time Critical') > 0
# Allow for regex weirdness
query = SentinelAPI.format_query(timeliness='.+ Critical')
assert query == r'timeliness:.+\ Critical'
assert api.count(timeliness='.+ Critical') > 0
query = SentinelAPI.format_query(identifier='/S[123 ]A.*/')
assert query == r'identifier:/S[123 ]A.*/'
assert api.count(identifier='/S[123 ]A.*/') > 0
def test_unicode_support():
api = SentinelAPI(**_api_kwargs)
test_str = u'٩(●̮̮̃•̃)۶:'
with pytest.raises(SentinelAPIError) as excinfo:
api.count(raw=test_str)
assert test_str == excinfo.value.response.json(
)['feed']['opensearch:Query']['searchTerms']
with pytest.raises(SentinelAPIError) as excinfo:
api.get_product_odata(test_str)
assert test_str in excinfo.value.response.json(
)['error']['message']['value']
def test_quote_symbol_bug():
# A test to check if plus symbol handling works correctly on the server side
# It used to raise an error but has since been fixed
# https://github.com/SentinelDataHub/DataHubSystem/issues/23
api = SentinelAPI(**_api_kwargs)
q = 'beginposition:[2017-05-30T00:00:00Z TO 2017-05-31T00:00:00Z+1DAY]'
count = api.count(raw=q)
assert count > 0
def test_quote_symbol_bug():
# A test to check if plus symbol handling works correctly on the server side
# It used to raise an error but has since been fixed
# https://github.com/SentinelDataHub/DataHubSystem/issues/23
api = SentinelAPI(**_api_kwargs)
q = 'beginposition:[2017-05-30T00:00:00Z TO 2017-05-31T00:00:00Z+1DAY]'
count = api.count(raw=q)
assert count > 0
def job():
#Checking Sentinel data on SciHub, based on search requirements
api = SentinelAPI(oah_user, oah_pass,
'https://apihub.copernicus.eu/apihub/')
count = api.count(area=wkt,
date=(start_date, end_date),
platformname=platformname,
area_relation='Contains',
raw=None,
cloudcoverpercentage=(min_cloud, max_cloud),
limit=20,
processinglevel=processinglevel)
now = datetime.now()
now = now.strftime("%d/%m/%Y %H:%M:%S")
print(now + ' - Checking for new data')
if count > 0:
# Write available image data to dataframe
products = api.query(area=wkt,
date=(start_date, end_date),
platformname=platformname,
area_relation='Contains',
raw=None,
cloudcoverpercentage=(min_cloud, max_cloud),
limit=20,
processinglevel=processinglevel)
products_df = api.to_dataframe(products)
detail = products_df.iat[0, 4]
# Get and format important data of satellite imagery
img_sat = products_df.iloc[0, 36] # Get satellite name
img_proc_lvl = products_df.iloc[0, 37] # Get image processing level
img_date = products_df.iloc[0, 4][6:16] # Get acquisition date
img_time = products_df.iloc[0, 4][17:25] # Get acquisition time
img_cloud = str(products_df.iloc[0,
21])[:5] + ' %' # Get cloud coverage
#Prepare e-mail content
subject = "New satellite image available - " + img_date
body = "Properites if the new satellite imagery is the following.\n\n" + 'Satellite: ' + img_sat + '\n' + 'Processing level: ' + img_proc_lvl + '\n' + 'Timestamp of imagery: ' + img_date + ', ' + img_time + '\n' + 'Cloud cover percentage: ' + img_cloud
message = f'Subject:{subject}\n\n{body}'
#Send e-mail and go to sleep
context = ssl.create_default_context()
with smtplib.SMTP_SSL(smtp_server, port, context=context) as server:
server.login(sender_email, password)
server.sendmail(sender_email, receiver_email,
message.encode("utf8"))
now = datetime.now()
now = now.strftime("%d/%m/%Y %H:%M:%S")
print(now + ' - Mail has been sent')
time.sleep(82800) #23 hours
return
else:
# If no new image available, print message
print(now + ' - There is no new data available')
return
def search_download_sen2_data(user, password, area_polygon, datum, cloudcover,
downloadpath):
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt
import zipfile
import os
import geopandas as gpd
# connect to API
api = SentinelAPI(user,
password,
api_url="https://scihub.copernicus.eu/apihub/")
#### Test
outlines_geojson = gpd.read_file("outlines.shp")
# Avoid Fiona Bug https://github.com/Toblerity/Fiona/issues/438
try:
os.remove("outline.geojson")
except OSError:
pass
outlines_geojson.to_file("outline.geojson", driver="GeoJSON")
area_polygon = "outline.geojson"
##### End test
# Search for products matching query
products = api.query(area=geojson_to_wkt(read_geojson(area_polygon)),
date=datum,
platformname="Sentinel-2",
producttype="S2MSI1C",
cloudcoverpercentage=cloudcover)
# count number of products matching query
print(
"Tiles found:",
api.count(area=geojson_to_wkt(read_geojson(area_polygon)),
date=datum,
platformname="Sentinel-2",
producttype="S2MSI1C",
cloudcoverpercentage=cloudcover), ", Total size: ",
api.get_products_size(products), "GB. Now downloading those tiles")
# downloading all products
download_zip = api.download_all(products, directory_path=downloadpath)
# Unzip files, delete
for key in download_zip[0].keys():
with zipfile.ZipFile(download_zip[0][key]['path']) as zip_file:
for member in zip_file.namelist():
filename = os.path.basename(member)
if not filename:
continue
source = zip_file.open(member)
source.close()
os.remove(download_zip[0][key]['path'])
def test_order_by():
api = SentinelAPI(**_api_auth)
kwargs = dict(area=geojson_to_wkt(
read_geojson(FIXTURES_DIR + '/map.geojson')),
date=("20151219", "20161019"),
platformname="Sentinel-2",
cloudcoverpercentage=(0, 10),
order_by="cloudcoverpercentage, -beginposition")
# Check that order_by works correctly also in cases where pagination is required
expected_count = api.count(**kwargs)
assert expected_count > 100
products = api.query(**kwargs)
assert len(products) == expected_count
vals = [x["cloudcoverpercentage"] for x in products.values()]
assert sorted(vals) == vals
def test_too_long_query():
api = SentinelAPI(**_api_kwargs)
# Test whether our limit calculation is reasonably correct and
# that a relevant error message is provided
def create_query(n):
return " a_-.*:,?+~!" * n
# Expect no error
q = create_query(164)
assert 0.99 < SentinelAPI.check_query_length(q) < 1.0
with pytest.raises(SentinelAPIError) as excinfo:
api.count(raw=q)
assert "Invalid query string" in excinfo.value.msg
# Expect HTTP status 500 Internal Server Error
q = create_query(165)
assert 1.0 <= SentinelAPI.check_query_length(q) < 1.01
with pytest.raises(SentinelAPIError) as excinfo:
api.count(raw=q)
assert excinfo.value.response.status_code == 500
assert ("Request Entity Too Large" in excinfo.value.msg
or "Request-URI Too Long" in excinfo.value.msg)
def job():
#Sentinel műholdképek keresése SciHub-on, keresési kritériumok alapján
api = SentinelAPI(oah_user,oah_pass, 'https://apihub.copernicus.eu/apihub/')
count=api.count(area=wkt, date=(start_date, end_date), platformname=platformname,area_relation='Contains',raw=None,cloudcoverpercentage=(min_cloud,max_cloud),limit=20, processinglevel = processinglevel)
now = datetime.now()
now = now.strftime("%d/%m/%Y %H:%M:%S")
print(now+' - Új adat keresése')
if count>0:
# Elérhető műholdképek adatainak dataframe-be írása
products = api.query(area=wkt, date=(start_date, end_date), platformname=platformname,area_relation='Contains',raw=None,cloudcoverpercentage=(min_cloud,max_cloud),limit=20, processinglevel = processinglevel)
products_df = api.to_dataframe(products)
detail=products_df.iat[0,4]
# E-mailbe írandó adatok formázása
img_sat=products_df.iloc[0,36] # Műhold név
img_proc_lvl=products_df.iloc[0,37] # Feldolgozási szint
img_date=products_df.iloc[0,4][6:16] # Felvétel rögzítésének dátuma
img_time=products_df.iloc[0,4][17:25] # Felvétel rögzítésének időpontja
img_cloud=str(products_df.iloc[0,21])[:5]+' %' # Felvétel felhőzöttsége
#E-mail tartalom előkészítése
subject="Új műholdkép - "+img_date
body="A vizsgált területről készült új műholdkép adatai.\n\n"+'Műhold: '+img_sat+'\n'+'Feldolgozási szint: '+img_proc_lvl+'\n'+'Felvétel rögzítve: '+img_date+', '+img_time+'\n'+'Felvétel felhőzöttsége: '+img_cloud
message=f'Subject:{subject}\n\n{body}'
#E-mail küldése és keresés szüneteltetése
context = ssl.create_default_context()
with smtplib.SMTP_SSL(smtp_server, port, context=context) as server:
server.login(sender_email, password)
server.sendmail(sender_email, receiver_email, message.encode("utf8"))
now = datetime.now()
now = now.strftime("%d/%m/%Y %H:%M:%S")
print(now+' - E-mail elküldve')
time.sleep(82800) #23 óra
return
else:
# Jelezzen ha nincs új műholdkép
print(now+' - Nem érhető el új műholdkép')
return
def test_count():
api = SentinelAPI(**_api_kwargs)
count = api.count(None, ("20150101", "20151231"))
assert count > 100000
def test_count():
api = SentinelAPI(**_api_kwargs)
count = api.count(None, ("20150101", "20151231"))
assert count > 100000