def get_sentinel(pts,
output_dir,
producttype,
start=datetime.date(2019, 1, 1),
end=datetime.date(2020, 1, 1),
**kwargs):
""""Query sentinel images of type producttype intersecting pts and randomly select one between start and end"""
config = configparser.ConfigParser()
config.read("sentinel_scihub.ini")
api = SentinelAPI(config["login"]["username"], config["login"]["password"])
for pt in pts:
products = api.query(area=f"{pt[0]}, {pt[1]}",
date=(start, end),
producttype=producttype,
**kwargs)
if len(products) == 0:
print("No products found, skipping")
return False
else:
product_id = random.choice(list(products.keys()))
print(product_id)
try:
api.download(product_id, directory_path=output_dir)
return True
except SentinelAPILTAError:
print("product offline, skipping")
return False
def test_scihub_unresponsive():
timeout_connect = 6
timeout_read = 6.6
timeout = (timeout_connect, timeout_read)
api = SentinelAPI("mock_user", "mock_password", timeout=timeout)
with requests_mock.mock() as rqst:
rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ConnectTimeout)
with pytest.raises(requests.exceptions.ConnectTimeout):
api.query(**_small_query)
with pytest.raises(requests.exceptions.ConnectTimeout):
api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.ConnectTimeout):
api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.ConnectTimeout):
api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
with requests_mock.mock() as rqst:
rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ReadTimeout)
with pytest.raises(requests.exceptions.ReadTimeout):
api.query(**_small_query)
with pytest.raises(requests.exceptions.ReadTimeout):
api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.ReadTimeout):
api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.ReadTimeout):
api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
def download_extract_s2(scene_uuid, down_dir, original_scene_dir):
"""
Download a single S2 scene from ESA via sentinelsat
based upon uuid.
"""
# if unzipped .SAFE file doesn't exist then we must do something
if not os.path.exists(original_scene_dir):
# if downloaded .zip file doesn't exist then download it
if not os.path.exists(original_scene_dir.replace('.SAFE/','.zip')):
print ( 'Downloading ESA scene zip: {}'.format(os.path.basename(original_scene_dir)) )
esa_api = SentinelAPI('tmj21','Welcome12!')
esa_api.download(scene_uuid, down_dir, checksum=True)
# extract downloaded .zip file
print ( 'Extracting ESA scene: {}'.format(original_scene_dir) )
zip_ref = zipfile.ZipFile(original_scene_dir.replace('.SAFE/','.zip'), 'r')
zip_ref.extractall(os.path.dirname(down_dir))
zip_ref.close()
else:
print ( 'ESA scene already extracted: {}'.format(original_scene_dir) )
# remove zipped scene but onliy if unzipped
if os.path.exists(original_scene_dir) & os.path.exists(original_scene_dir.replace('.SAFE/','.zip')):
print ( 'Deleting ESA scene zip: {}'.format(original_scene_dir.replace('.SAFE/','.zip')) )
os.remove(original_scene_dir.replace('.SAFE/','.zip'))
def download_image(self):
"""
Returns
-------
image_path : Directory
Filepath to downloaded image after unzipping
date : String
Date of downloaded image
time : String
Zulu time of downloaded image
"""
api = SentinelAPI(self.username, self.password,
'https://scihub.copernicus.eu/dhus')
products = self.get_image_products()
key = products.index[0]
title = products['title'][0]
date, time = str(products['endposition'][0]).split(' ')
os.chdir(self.image_dir)
print('\n >>> DOWNLOADING IMAGE')
api.download(key)
with ZipFile(title + '.zip', 'r') as zipObject:
filenames = zipObject.namelist()
for file in filenames:
if file.endswith('TCI_10m.jp2'):
image_path = zipObject.extract(file)
os.remove(title + '.zip')
return image_path, date, time
def down_imgs(inTbl, imgIDcol, outFolder=None):
"""
Download Images in Table
"""
import os
from sentinelsat import SentinelAPI
from glass.g.rd.shp import shp_to_obj
from glass.cons.sentinel import con_datahub
of = outFolder if outFolder else os.path.dirname(inTbl)
# Tbl to df
df_img = shp_to_obj(inTbl)
# API Instance
user, password = con_datahub()
api = SentinelAPI(user, password, URL_COPERNICUS)
# Download Images
for idx, row in df_img.iterrows():
# Check if file already exists
outFile = os.path.join(outFolder, row.identifier + '.zip')
if os.path.exists(outFile):
print('IMG already exists')
continue
else:
api.download(row[imgIDcol], directory_path=outFolder)
def dowload_img(list_index, dst_folder):
# connect to the API
api = SentinelAPI(data_hub['user'], data_hub['password'],
'https://scihub.copernicus.eu/dhus')
for i in list_index:
title = get_title(conn_string,
schema='metadado_img',
table='metadado_sentinel',
uuid=i)
file_already_download = is_file_in_folder(folder=path_home /
FOLDER_NAME,
file_name=title,
file_extention='.zip')
if file_already_download:
insert_date_hour_db(conn_string=conn_string,
schema='metadado_img',
table='metadado_sentinel',
column='date_download_img',
uuid=i)
elif TO_DOWNLOAD:
api.download(i, directory_path=dst_folder)
insert_date_hour_db(conn_string=conn_string,
schema='metadado_img',
table='metadado_sentinel',
column='date_download_img',
uuid=i)
def satdownload_zip(product_id, directory='./', api=None):
if api is None:
api = SentinelAPI(USERNAME, PASSWORD,
'https://scihub.copernicus.eu/dhus')
try:
print('Downloading {}...'.format(product_id))
api.download(product_id, directory)
except Exception as inst:
print('Wrong product id.')
def get_data(name,
AREA,
start_date='20190101',
end_date='20190128',
platformname='Sentinel-2',
processinglevel='Level-2A'):
#apiのクエリに次のが必要。中身はPOLYGON((..))っていうstr型
footprint_geojson = geojson_to_wkt(
read_geojson('./location/' + str(name) + '.geojson'))
# use sentinelAPI
user = ''
password = ''
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus')
products = api.query(
footprint_geojson,
date=(start_date, end_date), #取得希望期間の入力
platformname=platformname,
processinglevel=processinglevel, ##2016年はL1C
cloudcoverpercentage=(0, 100)) #被雲率（0％〜100％）
print("この期間の画像の数は" + str(len(products)) + "枚です")
#この後で雲の被覆率ごとにデータを並べ替えて、1個ダウンロードする。
products_gdf = api.to_geodataframe(products)
products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'],
ascending=[True]).head()
#ファイルがまだなければデータダウンロード。作業ディレクトリにzipファイルがダウンロードされる
for i in range(3): #3回までチャレンジ
uuid = products_gdf_sorted.iloc[i]["uuid"]
product_title = products_gdf_sorted.iloc[i][
"title"] #S2A_MSIL2A_20190101T01405... みたいな
product_date = products_gdf_sorted.iloc[i]["summary"].split(
',')[0].split()[1][:10]
print("この期間で1番被覆率が低いのは" + product_date + "日")
if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True:
print("新規にデータをダウンロードします")
try:
api.download(uuid)
except:
print("ダウンロード不可")
else:
break
else:
break
if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True:
#ダウンロードしたzipファイルを解凍
#str(product_title) + '.SAFE っていうフォルダが生成される
file_name = str(product_title) + '.zip'
with zipfile.ZipFile(file_name) as zf:
zf.extractall()
shutil.move(str(product_title) + '.zip', './data_geo/')
shutil.move(str(product_title) + '.SAFE', './data_geo/')
return product_title
def download(username, password, scene_id, directory_path, url=None):
"""A function for downloading scenes from Scihub with an API"""
# connect to the API
if url is None:
api = SentinelAPI(username, password)
else:
api = SentinelAPI(username, password, api_url=url)
api.download(scene_id, directory_path, True)
def _down_btn_clicked(self):
username = self.entry_username.get()
password = self.entry_password.get()
product_id = self.entry_product_id.get()
#print(username, password,product_id)
api = SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus')
print("Downloading started...")
api.download(product_id)
def download_sentinel(current_date, past_date, json, ndvi_tiles, df_predios,
ndvi_dir, rgb_dir, ngb_dir, sentinel_dir):
# Variables datos
geojson_file = 'Tiles-chile.geojson'
cloud_directory = ''
#Storage access
client = storage.Client.from_service_account_json(
'ADL-forestal-segmentation-7dc429779824.json')
bucket = client.get_bucket('ranger-app')
# Sentinel API
api = SentinelAPI('matias-arauco', 'arauco2019',
'https://scihub.copernicus.eu/apihub/')
footprint = geojson_to_wkt(json)
products = api.query(footprint,
date=(past_date, current_date),
platformname='Sentinel-2',
cloudcoverpercentage=(0, 30))
products_df = api.to_dataframe(products)
products_df_sorted = products_df.loc[products_df['tileid'].isnull()]
products_df_sorted = products_df_sorted.sort_values(
['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True])
print(products_df_sorted)
index = products_df_sorted.index
ite = 0
for i in index:
file_name = products_df_sorted['title'][ite][:]
year, month, day = products_df_sorted['title'][ite][
11:15], products_df_sorted['title'][ite][
15:17], products_df_sorted['title'][ite][17:19]
tile = products_df_sorted['title'][ite][39:44]
date = year + '-' + month + '-' + day
print('Descargando el Tile: ', tile, ', con fecha: ', date)
api.download(i, directory_path=sentinel_dir)
upload_file(bucket, sentinel_dir, file_name + '.zip')
ndvi_clipping(sentinel_dir, file_name, tile, date, ndvi_tiles,
df_predios, ndvi_dir, rgb_dir, ngb_dir, bucket)
os.remove(sentinel_dir + file_name + '.zip')
ite = ite + 1
def download(self, scene_id: str, output: str, **kwargs) -> str:
"""Try to download data from Copernicus.
Raises:
DownloadError when scene not found.
DataOfflineError when scene is not available/offline.
"""
meta = self.api.query(filename=f'{scene_id}*')
if len(meta) < 0:
raise DownloadError(f'Scene id {scene_id} not found.')
api = self.api
# When parallel support set, get an available client from Redis
if self.parallel:
client = self.clients.get_user()
api = SentinelAPI(client.username,
client.password,
show_progressbars=self.progress)
uuid = list(meta)[0]
entry = api.download(uuid, output)
if not entry['Online']:
raise DataOfflineError(scene_id)
return entry['path']
def test_scihub_unresponsive():
api = SentinelAPI("mock_user", "mock_password")
with requests_mock.mock() as rqst:
rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ConnectTimeout)
with pytest.raises(requests.exceptions.Timeout) as excinfo:
api.query(**_small_query)
with pytest.raises(requests.exceptions.Timeout) as excinfo:
api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.Timeout) as excinfo:
api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.Timeout) as excinfo:
api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
def download_from_scihub(product_uuid, out_folder, user, passwd):
"""
Downloads and unzips product_uuid from scihub
Parameters
----------
product_uuid
The product UUID (4dfB4-432df....)
out_folder
The folder to save the .SAFE file to
user
Scihub username
passwd
Scihub password
Notes
-----
If interrupted mid-download, there will be a .incomplete file in the download folder. You might need to remove
this for further processing.
"""
api = SentinelAPI(user, passwd)
log.info("Downloading {} from scihub".format(product_uuid))
prod = api.download(product_uuid, out_folder)
if not prod:
log.error("{} failed to download".format(product_uuid))
zip_path = os.path.join(out_folder, prod['title'] + ".zip")
log.info("Unzipping {} to {}".format(zip_path, out_folder))
zip_ref = zipfile.ZipFile(zip_path, 'r')
zip_ref.extractall(out_folder)
zip_ref.close()
log.info("Removing {}".format(zip_path))
os.remove(zip_path)
def test_SentinelAPI_wrong_credentials():
api = SentinelAPI("wrong_user", "wrong_password")
with pytest.raises(SentinelAPIError) as excinfo:
api.query(**_small_query)
assert excinfo.value.response.status_code == 401
with pytest.raises(SentinelAPIError) as excinfo:
api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
assert excinfo.value.response.status_code == 401
with pytest.raises(SentinelAPIError) as excinfo:
api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
assert excinfo.value.response.status_code == 401
with pytest.raises(SentinelAPIError) as excinfo:
api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
assert excinfo.value.response.status_code == 401
class sentinelWrapper:
def __init__(self):
logger.info("connect to sentinel API")
# connection to API for search queries and download requests
self.api = SentinelAPI(config.copernicusUser, config.copernicusPW,
config.copernicusURL)
logger.info("sentinel API connected")
def getSentinelProducts(self, lat, lon, dateFrom, dateTo, platform,
**kwargs):
logger.info("start sentinel query")
# convert geolocation coordinates to wkt format
footprint = geojson_to_wkt(Point((lon, lat)))
# prepare parameter for cloud coverage
if "cloudcoverpercentage" in kwargs:
kwargs["cloudcoverpercentage"] = (0,
kwargs["cloudcoverpercentage"])
# search query
result = self.api.query(footprint,
date=(dateFrom, dateTo),
platformname=platform,
**kwargs)
logger.info("sentinel query complete")
return result
# download multiple sentinel products (list of product IDs)
def downloadSentinelProducts(self, products):
logger.info("start downloading sentinel product list")
self.api.download_all(products, config.bigTilesDir)
logger.info("download complete")
# download sentinel product with certain product ID
def downloadSentinelProduct(self, productID):
logger.info("start downloading sentinel product")
self.api.download(productID, config.bigTilesDir)
logger.info("download complete")
def get_sentinel_images(reef, start_date, end_date, num_images, user,
password):
"""
Method to download Sentinel-2 images using Sentinel API
Params - 1. reef (str) - Coral reef object
2. start_date (str) - starting date of sentinel images
3. end_date (str) - end date of sentinel images
4. num_images (int) - number of sentinel-2 images to download
5. user (str) - username on scihub.copernicus.eu
6. password (str) - password on scihub.copernicus.eu
"""
#login into api
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus')
#load in geojson of reef
reef_path = reef.get_path()
reef_gjson_fp = os.path.join(reef_path, reef.get_reef_name() + '.geojson')
reef_footprint = geojson_to_wkt(read_geojson(reef_gjson_fp))
#query sentinel sat api
products = api.query(reef_footprint,date = (start_date, end_date),platformname = 'Sentinel-2'\
,area_relation = 'Intersects',processinglevel = 'Level-2A',\
order_by = 'cloudcoverpercentage')
#creating folder for saving sentinel images
sentinel_path = os.path.join(reef_path, 'SAFE files')
if not os.path.exists(sentinel_path):
os.makedirs(sentinel_path)
#downloading num_images
for i, x in enumerate(products.items()):
k, v = x[0], x[1]
if i < num_images:
api.download(k, directory_path=sentinel_path)
#unzipping files
for file in os.listdir(sentinel_path):
if file.endswith('.zip'):
file_path = os.path.join(sentinel_path, file)
out_path = os.path.join(sentinel_path, file.split('.')[0])
if os.path.exists(file_path) and not os.path.exists(out_path):
with zipfile.ZipFile(file_path, "r") as zip_ref:
zip_ref.extractall(sentinel_path)
os.remove(file_path)
def test_SentinelAPI_wrong_credentials(small_query):
api = SentinelAPI("wrong_user", "wrong_password")
@contextmanager
def assert_exception():
with pytest.raises(UnauthorizedError) as excinfo:
yield
assert excinfo.value.response.status_code == 401
assert "Invalid user name or password" in excinfo.value.msg
with assert_exception():
api.query(**small_query)
with assert_exception():
api.get_product_odata("8df46c9e-a20c-43db-a19a-4240c2ed3b8b")
with assert_exception():
api.download("8df46c9e-a20c-43db-a19a-4240c2ed3b8b")
with assert_exception():
api.download_all(["8df46c9e-a20c-43db-a19a-4240c2ed3b8b"])
def test_download(tmpdir):
api = SentinelAPI(**_api_auth)
uuid = "1f62a176-c980-41dc-b3a1-c735d660c910"
filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E"
expected_path = tmpdir.join(filename + ".zip")
# Download normally
product_info = api.download(uuid, str(tmpdir), checksum=True)
assert expected_path.samefile(product_info["path"])
assert product_info["title"] == filename
assert product_info["size"] == expected_path.size()
hash = expected_path.computehash()
modification_time = expected_path.mtime()
expected_product_info = product_info
del expected_product_info['path']
# File exists, test with checksum
# Expect no modification
product_info = api.download(uuid, str(tmpdir), check_existing=True)
assert expected_path.mtime() == modification_time
del product_info['path']
assert product_info == expected_product_info
# File exists, test without checksum
# Expect no modification
product_info = api.download(uuid, str(tmpdir), check_existing=False)
assert expected_path.mtime() == modification_time
del product_info['path']
assert product_info == expected_product_info
# Create invalid file, expect re-download
with expected_path.open("wb") as f:
f.seek(expected_product_info["size"] - 1)
f.write(b'\0')
assert expected_path.computehash("md5") != hash
product_info = api.download(uuid, str(tmpdir), check_existing=True)
assert expected_path.computehash("md5") == hash
del product_info['path']
assert product_info == expected_product_info
# Test continue
with expected_path.open("rb") as f:
content = f.read()
with expected_path.open("wb") as f:
f.write(content[:100])
assert expected_path.computehash("md5") != hash
product_info = api.download(uuid, str(tmpdir), check_existing=True)
assert expected_path.computehash("md5") == hash
del product_info['path']
assert product_info == expected_product_info
# Test MD5 check
with expected_path.open("wb") as f:
f.write(b'abcd' * 100)
assert expected_path.computehash("md5") != hash
with pytest.raises(InvalidChecksumError):
api.download(uuid, str(tmpdir), check_existing=True, checksum=True)
def test_download(tmpdir):
api = SentinelAPI(**_api_auth)
uuid = "1f62a176-c980-41dc-b3a1-c735d660c910"
filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E"
expected_path = tmpdir.join(filename + ".zip")
# Download normally
product_info = api.download(uuid, str(tmpdir), checksum=True)
assert expected_path.samefile(product_info["path"])
assert product_info["title"] == filename
assert product_info["size"] == expected_path.size()
hash = expected_path.computehash()
modification_time = expected_path.mtime()
expected_product_info = product_info
del expected_product_info['path']
# File exists, test with checksum
# Expect no modification
product_info = api.download(uuid, str(tmpdir), check_existing=True)
assert expected_path.mtime() == modification_time
del product_info['path']
assert product_info == expected_product_info
# File exists, test without checksum
# Expect no modification
product_info = api.download(uuid, str(tmpdir), check_existing=False)
assert expected_path.mtime() == modification_time
del product_info['path']
assert product_info == expected_product_info
# Create invalid file, expect re-download
with expected_path.open("wb") as f:
f.seek(expected_product_info["size"] - 1)
f.write(b'\0')
assert expected_path.computehash("md5") != hash
product_info = api.download(uuid, str(tmpdir), check_existing=True)
assert expected_path.computehash("md5") == hash
del product_info['path']
assert product_info == expected_product_info
# Test continue
with expected_path.open("rb") as f:
content = f.read()
with expected_path.open("wb") as f:
f.write(content[:100])
assert expected_path.computehash("md5") != hash
product_info = api.download(uuid, str(tmpdir), check_existing=True)
assert expected_path.computehash("md5") == hash
del product_info['path']
assert product_info == expected_product_info
# Test MD5 check
with expected_path.open("wb") as f:
f.write(b'abcd' * 100)
assert expected_path.computehash("md5") != hash
with pytest.raises(InvalidChecksumError):
api.download(uuid, str(tmpdir), check_existing=True, checksum=True)
def download_extract_s2_esa(scene_uuid, down_dir, original_scene_dir):
"""
Download a single S2 scene from ESA via sentinelsat
based upon uuid.
Assumes esa hub creds stored as env variables.
:param scene_uuid: S2 download uuid from sentinelsat query
:param down_dir: directory in which to create a downloaded product dir
:param original_scene_dir:
:return:
"""
# if unzipped .SAFE file doesn't exist then we must do something
if not os.path.exists(original_scene_dir):
# if downloaded .zip file doesn't exist then download it
if not os.path.exists(original_scene_dir.replace('.SAFE/', '.zip')):
logging.info('Downloading ESA scene zip: {}'.format(
os.path.basename(original_scene_dir)))
copernicus_username = os.getenv("COPERNICUS_USERNAME")
copernicus_pwd = os.getenv("COPERNICUS_PWD")
logging.debug(f"ESA username: {copernicus_username}")
esa_api = SentinelAPI(copernicus_username, copernicus_pwd)
esa_api.download(scene_uuid, down_dir, checksum=True)
# extract downloaded .zip file
logging.info('Extracting ESA scene: {}'.format(original_scene_dir))
zip_ref = zipfile.ZipFile(original_scene_dir.replace('.SAFE/', '.zip'),
'r')
zip_ref.extractall(os.path.dirname(down_dir))
zip_ref.close()
else:
logging.warning(
'ESA scene already extracted: {}'.format(original_scene_dir))
# remove zipped scene but onliy if unzipped
if os.path.exists(original_scene_dir) & os.path.exists(
original_scene_dir.replace('.SAFE/', '.zip')):
logging.info('Deleting ESA scene zip: {}'.format(
original_scene_dir.replace('.SAFE/', '.zip')))
os.remove(original_scene_dir.replace('.SAFE/', '.zip'))
def test_SentinelAPI_wrong_credentials():
api = SentinelAPI(
"wrong_user",
"wrong_password"
)
with pytest.raises(SentinelAPIError) as excinfo:
api.query(**_small_query)
assert excinfo.value.response.status_code == 401
with pytest.raises(SentinelAPIError) as excinfo:
api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
assert excinfo.value.response.status_code == 401
with pytest.raises(SentinelAPIError) as excinfo:
api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
assert excinfo.value.response.status_code == 401
with pytest.raises(SentinelAPIError) as excinfo:
api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
assert excinfo.value.response.status_code == 401
class SentinelDownloader:
user = attr.ib(type=User)
area = attr.ib(type=str)
date = attr.ib(type=datetime.datetime)
def __attrs_post_init__(self):
self.month_range = relativedelta(months=1)
self.api = SentinelAPI(self.user.name, self.user.password,
'https://scihub.copernicus.eu/dhus')
def download_items(self):
for scene in self.scenes[:1]:
path = os.path.join(PATH_DATA_IN, scene)
os.mkdir(path)
self.api.download(scene, path)
@property
def scenes(self):
end_positions = [[
abs((self.items_metadata[key]["endposition"] -
self.date)).total_seconds(), key
] for key in self.items_metadata]
sorted_dates = sorted(end_positions, key=lambda x: x[0])
small_difference = sorted_dates[0][0]
final_dates = [
el[1] for el in sorted_dates if el[0] == small_difference
]
return final_dates
@lazy_property
def items_metadata(self):
return self.api.query(area=self.area,
date=(self.date - self.month_range,
self.date + self.month_range),
platformname='Sentinel-2',
cloudcoverpercentage="[0 TO 20]")
def test_scihub_unresponsive(small_query):
timeout_connect = 6
timeout_read = 6.6
timeout = (timeout_connect, timeout_read)
api = SentinelAPI("mock_user", "mock_password", timeout=timeout)
with requests_mock.mock() as rqst:
rqst.request(requests_mock.ANY,
requests_mock.ANY,
exc=requests.exceptions.ConnectTimeout)
with pytest.raises(requests.exceptions.ConnectTimeout):
api.query(**small_query)
with pytest.raises(requests.exceptions.ConnectTimeout):
api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.ConnectTimeout):
api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.ConnectTimeout):
api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
with requests_mock.mock() as rqst:
rqst.request(requests_mock.ANY,
requests_mock.ANY,
exc=requests.exceptions.ReadTimeout)
with pytest.raises(requests.exceptions.ReadTimeout):
api.query(**small_query)
with pytest.raises(requests.exceptions.ReadTimeout):
api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.ReadTimeout):
api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b')
with pytest.raises(requests.exceptions.ReadTimeout):
api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
class Downloader:
def __init__(self, str_username, str_password, str_link):
self.api = SentinelAPI(str_username, str_password, str_link)
self.products = None
def search_polygon(self, footprint: object, str_date_start: str,
str_date_end: str, str_platform_name: str, percentage: object):
print('searching')
self.products = self.api.query(footprint,
date=(str_date_start, str_date_end),
platformname=str_platform_name,
cloudcoverpercentage=(percentage[0], percentage[1]))
size = self.api.get_products_size(self.products)
print(f'found {size}GiB of data')
# print(self.products)
def download_zip(self, path):
self.api.download_all(self.products, path, max_attempt, True)
def download_products(self, path, download_file):
if download_file:
self.download_zip(path)
print('downloaded')
df_products = self.api.to_dataframe(self.products)
return df_products
def download_geoproduct(self, path, download_file):
if download_file:
self.download_zip(path)
# print('download Geos')
gdf_products = self.api.to_geodataframe(self.products)
return gdf_products
def download_json(self):
return self.api.to_geojson(self.products)
def download_one(self, key, path):
self.api.download(key, path, True)
def download(product: str,
api: SentinelAPI,
udate: str,
lat: float,
lng: float,
force: str = 'false'):
"""Download Sentinel-2 tile user selected
Parameters
----------
product : str
Product key of selceted tile
api : SentinelAPI
API interface to Sentinel-2 data hub
udate : str
Selected date in format YYYY.MM.DD
lat : float
Target latitude
lng : float
Target longitude
force : str, optional
Force download by removing existing downloaded zip files, by default 'false'
Returns
-------
str
Path to downloaded zip of tile
"""
logger.info('DOWNLOADING SENTINEL-2 DATA')
output_pth = os.path.join(const.MODIS_TERRA, 'sentinel', udate,
f'lat{lat}_lng{lng}')
if not os.path.exists(output_pth):
logger.debug(f'creating directory: {output_pth}')
os.makedirs(output_pth)
if force == 'true': # if force then remove existing downloads
files = glob(os.path.join(output_pth, '*'))
for f in files:
logger.debug(f"removing: {f}")
os.remove(f)
try: # download target tile
logger.info(f'Downloading {product}')
info = api.download(product, directory_path=output_pth)
return info
except Exception as e:
logger.error(e)
def main():
if len(sys.argv) < 2:
print(" Usage: python3 get_past_scenes.py [year] [month]")
return 1
api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
logging.info(api.api_url)
t0 = datetime(int(sys.argv[1]), int(sys.argv[2]), 1, 0, 0, 0)
tf = t0 + timedelta(days=12)
# search by polygon, time, and SciHub query keywords
footprint = geojson_to_wkt(
read_geojson(home['parameters'] + '/extent_' + location['region'] +
'.geojson'))
products_s1a = api.query(footprint,
date=(date(t0.year, t0.month,
t0.day), date(tf.year, tf.month,
tf.day)),
producttype="GRD",
platformname='Sentinel-1')
unavailable = []
for uuid in products_s1a:
product_info = api.get_product_odata(uuid)
if any(product_info['title'] in s for s in os.listdir(sarIn)):
logging.info('Skipping ' + uuid + '. Already exists in ' + sarIn)
continue
logging.info('Is ' + uuid + ' online?')
logging.info(product_info['Online'])
if not product_info['Online']:
logging.info('Requesting unavailable uuids')
api.download(uuid)
unavailable = unavailable + [uuid]
else:
logging.info('Downloading available uuids')
api.download(uuid, directory_path=sarIn)
logging.info(
'Sleeping 30 minutes (the API does not allow intensive requests)')
time.sleep(30 * 60)
while len(unavailable) > 0:
for uuid in unavailable:
product_info = api.get_product_odata(uuid)
if product_info['Online']:
logging.info(uuid + ' is available! Downloading:')
api.download(uuid, directory_path=sarIn)
unavailable.remove(uuid)
time.sleep(600)
return 0
def download_from_scihub(product_uuid, out_folder, user, passwd):
"""
Downloads and unzips product_uuid from scihub
Parameters
----------
product_uuid : str
The product UUID (4dfB4-432df....)
out_folder : str
The folder to save the .SAFE file to
user : str
Scihub username
passwd : str
Scihub password
Notes
-----
If interrupted mid-download, there will be a .incomplete file in the download folder. You might need to remove
this for further processing.
"""
api = SentinelAPI(user, passwd)
api.api_url = "https://apihub.copernicus.eu/apihub/"
log.info("Downloading {} from scihub".format(product_uuid))
prod = api.download(product_uuid, out_folder)
if not prod:
log.error("{} not found. Please check.".format(product_uuid))
if not prod["Online"]:
log.info(
"{} is being retrieved from long-term archive. Please try again later."
.format(product_uuid))
return 1
zip_path = os.path.join(out_folder, prod['title'] + ".zip")
log.info("Unzipping {} to {}".format(zip_path, out_folder))
zip_ref = zipfile.ZipFile(zip_path, 'r')
zip_ref.extractall(out_folder)
zip_ref.close()
log.info("Removing {}".format(zip_path))
os.remove(zip_path)
return 0
def download_products(products, data_dir, username, password, unzip=False, max_retries=3, verbose=True):
"""Downloads a set of Sentinel-2 products.
Parameters:
products : dict
A dict with Sentinel-2 products, as returned by SentinelAPI.query() or
search_products(); keys must be Sentinel-2 UUIDs and values must be
dicts with at least the "identifier" property.
data_dir : string
The path to the directory where the products should be downloaded
username, password : string
Copernicus Open Access Hub username and password
Options:
unzip : boolean (default: False)
Whether to unzip the downloaded products (to the same dir)
max_retries : integer (default: 3)
The maximum number of retires
verbose : boolean (default: True)
Whether to report progress to screen
Returns:
None
"""
if username in ["", None] or password in ["", None]:
print("Error: you must provide your Copernicus credentials; you can set them in download_datasets.py")
sys.exit()
# Login to Copernicus
api = SentinelAPI(username, password)
for product_id, product_info in products.items():
tile = product_info["identifier"].split("_")[5][1:]
tile_dir_path = os.path.join(data_dir, "T{}".format(tile))
zip_fname = product_info["identifier"] + ".zip"
zip_fpath = os.path.join(tile_dir_path, zip_fname)
if os.path.isfile(zip_fpath):
# Skip download if the zip already exists
if verbose:
print("Already downloaded: {}".format(product_info["identifier"]))
else:
# Create tile subdir if not found
if not os.path.exists(tile_dir_path):
os.makedirs(tile_dir_path)
# Download (and check with MD5) product
if verbose:
print(product_info["identifier"])
tries_left = max_retries
while tries_left > 0:
try:
result = api.download(product_id, tile_dir_path)
break
except SentinelAPI.InvalidChecksumError:
tries_left -= 1
if verbose:
if tries_left > 0:
print("Bad MD5 checksum! Retrying.")
else:
print("Bad MD5 checksum! Max retries reached; skipping.")
os.remove(zip_fpath)
except:
tries_left -= 1
if verbose:
if tries_left > 0:
print("Unknown error! Retrying.")
else:
print("Unknown error! Skipping.")
if verbose:
print("Downloaded {}".format(zip_fname))
# Unzip if asked
if unzip:
unzipped_fname = product_info["identifier"] + ".SAFE"
unzipped_path = os.path.join(tile_dir_path, unzipped_fname)
if os.path.exists(unzipped_path):
if verbose:
print("Dataset already unzipped")
else:
if verbose:
print("Unzipping dataset ...")
with zipfile.ZipFile(zip_fpath, "r") as zip:
zip.extractall(tile_dir_path)
if verbose:
print("Unzipped {}".format(unzipped_fname))
class SentinelDownloader(object):
def __init__(self, user, password, api_url='https://scihub.copernicus.eu/apihub'):
self._apiname = api_url
self._user = user
self._password = password
# init logger
root = logging.getLogger()
root.addHandler(logging.StreamHandler(
sys.stderr
))
if self._apiname == 'https://scihub.copernicus.eu/apihub':
try:
from sentinelsat import SentinelAPI
except ImportError as e:
gs.fatal(_("Module requires sentinelsat library: {}").format(e))
# connect SciHub via API
self._api = SentinelAPI(self._user, self._password,
api_url=self._apiname
)
elif self._apiname == 'USGS_EE':
try:
import landsatxplore.api
from landsatxplore.errors import EarthExplorerError
except ImportError as e:
gs.fatal(_("Module requires landsatxplore library: {}").format(e))
api_login = False
while api_login is False:
# avoid login conflict in possible parallel execution
try:
self._api = landsatxplore.api.API(self._user,
self._password)
api_login = True
except EarthExplorerError as e:
time.sleep(1)
self._products_df_sorted = None
def filter(self, area, area_relation,
clouds=None, producttype=None, limit=None, query={},
start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None):
args = {}
if clouds:
args['cloudcoverpercentage'] = (0, int(clouds))
if relativeorbitnumber:
args['relativeorbitnumber'] = relativeorbitnumber
if producttype.startswith('S2') and int(relativeorbitnumber) > 143:
gs.warning("This relative orbit number is out of range")
elif int(relativeorbitnumber) > 175:
gs.warning(_("This relative orbit number is out of range"))
if producttype:
args['producttype'] = producttype
if producttype.startswith('S2'):
args['platformname'] = 'Sentinel-2'
else:
args['platformname'] = 'Sentinel-1'
if not start:
start = 'NOW-60DAYS'
else:
start = start.replace('-', '')
if not end:
end = 'NOW'
else:
end = end.replace('-', '')
if query:
redefined = [value for value in args.keys() if value in query.keys()]
if redefined:
gs.warning(_("Query overrides already defined options ({})").format(
','.join(redefined)
))
args.update(query)
gs.verbose(_("Query: area={} area_relation={} date=({}, {}) args={}").format(
area, area_relation, start, end, args
))
products = self._api.query(
area=area, area_relation=area_relation,
date=(start, end),
**args
)
products_df = self._api.to_dataframe(products)
if len(products_df) < 1:
gs.message(_("No product found"))
return
# sort and limit to first sorted product
if sortby:
self._products_df_sorted = products_df.sort_values(
sortby,
ascending=[asc] * len(sortby)
)
else:
self._products_df_sorted = products_df
if limit:
self._products_df_sorted = self._products_df_sorted.head(int(limit))
gs.message(_("{} Sentinel product(s) found").format(len(self._products_df_sorted)))
def list(self):
if self._products_df_sorted is None:
return
id_kw = ('uuid', 'entity_id')
identifier_kw = ('identifier', 'display_id')
cloud_kw = ('cloudcoverpercentage', 'cloud_cover')
time_kw = ('beginposition', 'acquisition_date')
kw_idx = 1 if self._apiname == 'USGS_EE' else 0
for idx in range(len(self._products_df_sorted[id_kw[kw_idx]])):
if cloud_kw[kw_idx] in self._products_df_sorted:
ccp = '{0:2.0f}%'.format(
float(self._products_df_sorted[cloud_kw[kw_idx]][idx]))
else:
ccp = 'cloudcover_NA'
print_str = '{0} {1}'.format(
self._products_df_sorted[id_kw[kw_idx]][idx],
self._products_df_sorted[identifier_kw[kw_idx]][idx])
if kw_idx == 1:
time_string = self._products_df_sorted[time_kw[kw_idx]][idx]
else:
time_string = self._products_df_sorted[
time_kw[kw_idx]][idx].strftime("%Y-%m-%dT%H:%M:%SZ")
print_str += ' {0} {1}'.format(time_string, ccp)
if kw_idx == 0:
print_str += ' {0}'.format(
self._products_df_sorted['producttype'][idx])
print(print_str)
def download(self, output, sleep=False, maxretry=False,
datasource='ESA_COAH'):
if self._products_df_sorted is None:
return
create_dir(output)
gs.message(_("Downloading data into <{}>...").format(output))
if datasource == 'USGS_EE':
from landsatxplore.earthexplorer import EarthExplorer
from landsatxplore.errors import EarthExplorerError
from zipfile import ZipFile
ee_login = False
while ee_login is False:
# avoid login conflict in possible parallel execution
try:
ee = EarthExplorer(self._user, self._password)
ee_login = True
except EarthExplorerError as e:
time.sleep(1)
for idx in range(len(self._products_df_sorted['entity_id'])):
scene = self._products_df_sorted['entity_id'][idx]
identifier = self._products_df_sorted['display_id'][idx]
zip_file = os.path.join(output, '{}.zip'.format(identifier))
gs.message(_("Downloading {}...").format(identifier))
try:
ee.download(identifier=identifier, output_dir=output, timeout=600)
except EarthExplorerError as e:
gs.fatal(_(e))
ee.logout()
# extract .zip to get "usual" .SAFE
with ZipFile(zip_file, 'r') as zip:
safe_name = zip.namelist()[0].split('/')[0]
outpath = os.path.join(output, safe_name)
zip.extractall(path=output)
gs.message(_("Downloaded to <{}>").format(outpath))
try:
os.remove(zip_file)
except Exception as e:
gs.warning(_("Unable to remove {0}:{1}").format(
zip_file, e))
elif datasource == "ESA_COAH":
for idx in range(len(self._products_df_sorted['uuid'])):
gs.message('{} -> {}.SAFE'.format(
self._products_df_sorted['uuid'][idx],
os.path.join(output, self._products_df_sorted['identifier'][idx])
))
# download
out = self._api.download(self._products_df_sorted['uuid'][idx],
output)
if sleep:
x = 1
online = out['Online']
while not online:
# sleep is in minutes so multiply by 60
time.sleep(int(sleep) * 60)
out = self._api.download(self._products_df_sorted['uuid'][idx],
output)
x += 1
if x > maxretry:
online = True
elif datasource == 'GCS':
for scene_id in self._products_df_sorted['identifier']:
gs.message(_("Downloading {}...").format(scene_id))
dl_code = download_gcs(scene_id, output)
if dl_code == 0:
gs.message(_("Downloaded to {}").format(
os.path.join(output, '{}.SAFE'.format(scene_id))))
else:
# remove incomplete file
del_folder = os.path.join(output,
'{}.SAFE'.format(scene_id))
try:
shutil.rmtree(del_folder)
except Exception as e:
gs.warning(_("Unable to removed unfinished "
"download {}".format(del_folder)))
def save_footprints(self, map_name):
if self._products_df_sorted is None:
return
if self._apiname == 'USGS_EE':
gs.fatal(_(
"USGS Earth Explorer does not support footprint download."))
try:
from osgeo import ogr, osr
except ImportError as e:
gs.fatal(_("Option <footprints> requires GDAL library: {}").format(e))
gs.message(_("Writing footprints into <{}>...").format(map_name))
driver = ogr.GetDriverByName("GPKG")
tmp_name = gs.tempfile() + '.gpkg'
data_source = driver.CreateDataSource(tmp_name)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
# features can be polygons or multi-polygons
layer = data_source.CreateLayer(str(map_name), srs, ogr.wkbMultiPolygon)
# attributes
attrs = OrderedDict([
("uuid", ogr.OFTString),
("ingestiondate", ogr.OFTString),
("cloudcoverpercentage", ogr.OFTInteger),
("producttype", ogr.OFTString),
("identifier", ogr.OFTString)
])
# Sentinel-1 data does not have cloudcoverpercentage
prod_types = [type for type in self._products_df_sorted["producttype"]]
s1_types = ["SLC", "GRD"]
if any(type in prod_types for type in s1_types):
del attrs["cloudcoverpercentage"]
for key in attrs.keys():
field = ogr.FieldDefn(key, attrs[key])
layer.CreateField(field)
# features
for idx in range(len(self._products_df_sorted['uuid'])):
wkt = self._products_df_sorted['footprint'][idx]
feature = ogr.Feature(layer.GetLayerDefn())
newgeom = ogr.CreateGeometryFromWkt(wkt)
# convert polygons to multi-polygons
newgeomtype = ogr.GT_Flatten(newgeom.GetGeometryType())
if newgeomtype == ogr.wkbPolygon:
multigeom = ogr.Geometry(ogr.wkbMultiPolygon)
multigeom.AddGeometryDirectly(newgeom)
feature.SetGeometry(multigeom)
else:
feature.SetGeometry(newgeom)
for key in attrs.keys():
if key == 'ingestiondate':
value = self._products_df_sorted[key][idx].strftime("%Y-%m-%dT%H:%M:%SZ")
else:
value = self._products_df_sorted[key][idx]
feature.SetField(key, value)
layer.CreateFeature(feature)
feature = None
data_source = None
# coordinates of footprints are in WKT -> fp precision issues
# -> snap
gs.run_command('v.import', input=tmp_name, output=map_name,
layer=map_name, snap=1e-10, quiet=True
)
def get_products_from_uuid_usgs(self, uuid_list):
scenes = []
for uuid in uuid_list:
metadata = self._api.metadata(uuid, 'SENTINEL_2A')
scenes.append(metadata)
scenes_df = pandas.DataFrame.from_dict(scenes)
self._products_df_sorted = scenes_df
gs.message(_("{} Sentinel product(s) found").format(
len(self._products_df_sorted)))
def set_uuid(self, uuid_list):
"""Set products by uuid.
TODO: Find better implementation
:param uuid: uuid to download
"""
if self._apiname == 'USGS_EE':
self.get_products_from_uuid_usgs(uuid_list)
else:
from sentinelsat.sentinel import SentinelAPIError
self._products_df_sorted = {'uuid': []}
for uuid in uuid_list:
try:
odata = self._api.get_product_odata(uuid, full=True)
except SentinelAPIError as e:
gs.error(_("{0}. UUID {1} skipped".format(e, uuid)))
continue
for k, v in odata.items():
if k == 'id':
k = 'uuid'
elif k == 'Sensing start':
k = 'beginposition'
elif k == 'Product type':
k = 'producttype'
elif k == 'Cloud cover percentage':
k = 'cloudcoverpercentage'
elif k == 'Identifier':
k = 'identifier'
elif k == 'Ingestion Date':
k = 'ingestiondate'
elif k == 'footprint':
pass
else:
continue
if k not in self._products_df_sorted:
self._products_df_sorted[k] = []
self._products_df_sorted[k].append(v)
def filter_USGS(self, area, area_relation, clouds=None, producttype=None,
limit=None, query={}, start=None, end=None, sortby=[],
asc=True, relativeorbitnumber=None):
if area_relation != 'Intersects':
gs.fatal(_(
"USGS Earth Explorer only supports area_relation"
" 'Intersects'"))
if relativeorbitnumber:
gs.fatal(_(
"USGS Earth Explorer does not support 'relativeorbitnumber'"
" option."))
if producttype and producttype != 'S2MSI1C':
gs.fatal(_(
"USGS Earth Explorer only supports producttype S2MSI1C"))
if query:
if not any(key in query for key in ['identifier', 'filename',
'usgs_identifier']):
gs.fatal(_(
"USGS Earth Explorer only supports query options"
" 'filename', 'identifier' or 'usgs_identifier'."))
if 'usgs_identifier' in query:
# get entityId from usgs identifier and directly save results
usgs_id = query['usgs_identifier']
check_s2l1c_identifier(usgs_id, source='usgs')
# entity_id = self._api.lookup('SENTINEL_2A', [usgs_id],
# inverse=True)
entity_id = self._api.get_entity_id([usgs_id], 'SENTINEL_2A')
self.get_products_from_uuid_usgs(entity_id)
return
else:
if "filename" in query:
esa_id = query['filename'].replace('.SAFE', '')
else:
esa_id = query['identifier']
check_s2l1c_identifier(esa_id, source='esa')
esa_prod_id = esa_id.split('_')[-1]
utm_tile = esa_id.split('_')[-2]
acq_date = esa_id.split('_')[2].split('T')[0]
acq_date_string = '{0}-{1}-{2}'.format(
acq_date[:4], acq_date[4:6], acq_date[6:])
start_date = end_date = acq_date_string
# build the USGS style S2-identifier
if utm_tile.startswith('T'):
utm_tile_base = utm_tile[1:]
bbox = get_bbox_from_S2_UTMtile(utm_tile_base)
else:
# get coordinate pairs from wkt string
str_1 = 'POLYGON(('
str_2 = '))'
coords = area[area.find(str_1)+len(str_1):area.rfind(str_2)].split(',')
# add one space to first pair for consistency
coords[0] = ' ' + coords[0]
lons = [float(pair.split(' ')[1]) for pair in coords]
lats = [float(pair.split(' ')[2]) for pair in coords]
bbox = (min(lons), min(lats), max(lons), max(lats))
start_date = start
end_date = end
usgs_args = {
'dataset': 'SENTINEL_2A',
'bbox': bbox,
'start_date': start_date,
'end_date': end_date
}
if clouds:
usgs_args['max_cloud_cover'] = clouds
if limit:
usgs_args['max_results'] = limit
scenes = self._api.search(**usgs_args)
self._api.logout()
if query:
# check if the UTM-Tile is correct, remove otherwise
for scene in scenes:
if scene['display_id'].split('_')[1] != utm_tile:
scenes.remove(scene)
# remove redundant scene
if len(scenes) == 2:
for scene in scenes:
prod_id = scene['display_id'].split('_')[-1]
if prod_id != esa_prod_id:
scenes.remove(scene)
if len(scenes) < 1:
gs.message(_("No product found"))
return
scenes_df = pandas.DataFrame.from_dict(scenes)
if sortby:
# replace sortby keywords with USGS keywords
for idx, keyword in enumerate(sortby):
if keyword == 'cloudcoverpercentage':
sortby[idx] = 'cloud_cover'
# turn cloudcover to float to make it sortable
scenes_df['cloud_cover'] = pandas.to_numeric(
scenes_df['cloud_cover'])
elif keyword == 'ingestiondate':
sortby[idx] = 'acquisition_date'
# what does sorting by footprint mean
elif keyword == 'footprint':
sortby[idx] = 'display_id'
self._products_df_sorted = scenes_df.sort_values(
sortby,
ascending=[asc] * len(sortby), ignore_index=True
)
else:
self._products_df_sorted = scenes_df
gs.message(_("{} Sentinel product(s) found").format(
len(self._products_df_sorted)))
def test_download(tmpdir):
api = SentinelAPI(**_api_auth)
uuid = "1f62a176-c980-41dc-b3a1-c735d660c910"
filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E"
expected_path = tmpdir.join(filename + ".zip")
tempfile_path = tmpdir.join(filename + ".zip.incomplete")
# Download normally
product_info = api.download(uuid, str(tmpdir), checksum=True)
assert expected_path.samefile(product_info["path"])
assert not tempfile_path.check(exists=1)
assert product_info["title"] == filename
assert product_info["size"] == expected_path.size()
assert product_info["downloaded_bytes"] == expected_path.size()
hash = expected_path.computehash("md5")
modification_time = expected_path.mtime()
expected_product_info = product_info
# File exists, expect nothing to happen
product_info = api.download(uuid, str(tmpdir))
assert not tempfile_path.check(exists=1)
assert expected_path.mtime() == modification_time
expected_product_info["downloaded_bytes"] = 0
assert product_info == expected_product_info
# Create invalid but full-sized tempfile, expect re-download
expected_path.move(tempfile_path)
with tempfile_path.open("wb") as f:
f.seek(expected_product_info["size"] - 1)
f.write(b'\0')
assert tempfile_path.computehash("md5") != hash
product_info = api.download(uuid, str(tmpdir))
assert expected_path.check(exists=1, file=1)
assert expected_path.computehash("md5") == hash
expected_product_info["downloaded_bytes"] = expected_product_info["size"]
assert product_info == expected_product_info
# Create invalid tempfile, without checksum check
# Expect continued download and no exception
dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa'
with tempfile_path.open("wb") as f:
f.write(dummy_content)
expected_path.remove()
product_info = api.download(uuid, str(tmpdir), checksum=False)
assert not tempfile_path.check(exists=1)
assert expected_path.check(exists=1, file=1)
assert expected_path.computehash("md5") != hash
expected_product_info["downloaded_bytes"] = expected_product_info["size"] - len(dummy_content)
assert product_info == expected_product_info
# Create invalid tempfile, with checksum check
# Expect continued download and exception raised
dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa'
with tempfile_path.open("wb") as f:
f.write(dummy_content)
expected_path.remove()
with pytest.raises(InvalidChecksumError):
api.download(uuid, str(tmpdir), checksum=True)
assert not tempfile_path.check(exists=1)
assert not expected_path.check(exists=1, file=1)
tmpdir.remove()
def test_check_existing(tmpdir):
api = SentinelAPI(**_api_auth)
ids = [
"5618ce1b-923b-4df2-81d9-50b53e5aded9",
"d8340134-878f-4891-ba4f-4df54f1e3ab4",
"1f62a176-c980-41dc-b3a1-c735d660c910"
]
names = [
"S1A_WV_OCN__2SSV_20150526T081641_20150526T082418_006090_007E3E_104C",
"S1A_WV_OCN__2SSV_20150526T211029_20150526T211737_006097_007E78_134A",
"S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E"
]
paths = [tmpdir.join(fn + ".zip") for fn in names]
path_strings = list(map(str, paths))
# Init files used for testing
api.download(ids[0], str(tmpdir))
# File #1: complete and correct
assert paths[0].check(exists=1, file=1)
# File #2: complete but incorrect
with paths[1].open("wb") as f:
size = 130102
f.seek(size - 1)
f.write(b'\0')
# File #3: incomplete
dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa'
with paths[2].open("wb") as f:
f.write(dummy_content)
assert paths[2].check(exists=1, file=1)
# Test
expected = {str(paths[1]), str(paths[2])}
result = api.check_files(ids=ids, directory=str(tmpdir))
assert set(result) == expected
assert result[paths[1]][0]['id'] == ids[1]
assert result[paths[2]][0]['id'] == ids[2]
assert paths[0].check(exists=1, file=1)
assert paths[1].check(exists=1, file=1)
assert paths[2].check(exists=1, file=1)
result = api.check_files(paths=path_strings)
assert set(result) == expected
assert result[paths[1]][0]['id'] == ids[1]
assert result[paths[2]][0]['id'] == ids[2]
assert paths[0].check(exists=1, file=1)
assert paths[1].check(exists=1, file=1)
assert paths[2].check(exists=1, file=1)
result = api.check_files(paths=path_strings, delete=True)
assert set(result) == expected
assert result[paths[1]][0]['id'] == ids[1]
assert result[paths[2]][0]['id'] == ids[2]
assert paths[0].check(exists=1, file=1)
assert not paths[1].check(exists=1, file=1)
assert not paths[2].check(exists=1, file=1)
missing_file = str(
tmpdir.join(
"S1A_EW_GRDH_1SDH_20141003T003840_20141003T003920_002658_002F54_4DD1.zip"
))
result = api.check_files(paths=[missing_file])
assert set(result) == {missing_file}
assert result[missing_file][0]['id']
with pytest.raises(ValueError):
api.check_files(ids=ids)
with pytest.raises(ValueError):
api.check_files()
tmpdir.remove()
def test_download_invalid_id():
api = SentinelAPI(**_api_auth)
uuid = "1f62a176-c980-41dc-xxxx-c735d660c910"
with pytest.raises(SentinelAPIError) as excinfo:
api.download(uuid)
assert 'Invalid key' in excinfo.value.msg
def test_download(tmpdir):
api = SentinelAPI(**_api_auth)
uuid = "1f62a176-c980-41dc-b3a1-c735d660c910"
filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E"
expected_path = tmpdir.join(filename + ".zip")
tempfile_path = tmpdir.join(filename + ".zip.incomplete")
# Download normally
product_info = api.download(uuid, str(tmpdir), checksum=True)
assert expected_path.samefile(product_info["path"])
assert not tempfile_path.check(exists=1)
assert product_info["title"] == filename
assert product_info["size"] == expected_path.size()
assert product_info["downloaded_bytes"] == expected_path.size()
hash = expected_path.computehash("md5")
modification_time = expected_path.mtime()
expected_product_info = product_info
# File exists, expect nothing to happen
product_info = api.download(uuid, str(tmpdir))
assert not tempfile_path.check(exists=1)
assert expected_path.mtime() == modification_time
expected_product_info["downloaded_bytes"] = 0
assert product_info == expected_product_info
# Create invalid but full-sized tempfile, expect re-download
expected_path.move(tempfile_path)
with tempfile_path.open("wb") as f:
f.seek(expected_product_info["size"] - 1)
f.write(b'\0')
assert tempfile_path.computehash("md5") != hash
product_info = api.download(uuid, str(tmpdir))
assert expected_path.check(exists=1, file=1)
assert expected_path.computehash("md5") == hash
expected_product_info["downloaded_bytes"] = expected_product_info["size"]
assert product_info == expected_product_info
# Create invalid tempfile, without checksum check
# Expect continued download and no exception
dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa'
with tempfile_path.open("wb") as f:
f.write(dummy_content)
expected_path.remove()
product_info = api.download(uuid, str(tmpdir), checksum=False)
assert not tempfile_path.check(exists=1)
assert expected_path.check(exists=1, file=1)
assert expected_path.computehash("md5") != hash
expected_product_info["downloaded_bytes"] = expected_product_info[
"size"] - len(dummy_content)
assert product_info == expected_product_info
# Create invalid tempfile, with checksum check
# Expect continued download and exception raised
dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa'
with tempfile_path.open("wb") as f:
f.write(dummy_content)
expected_path.remove()
with pytest.raises(InvalidChecksumError):
api.download(uuid, str(tmpdir), checksum=True)
assert not tempfile_path.check(exists=1)
assert not expected_path.check(exists=1, file=1)
tmpdir.remove()
def test_check_existing(tmpdir):
api = SentinelAPI(**_api_auth)
ids = [
"5618ce1b-923b-4df2-81d9-50b53e5aded9",
"d8340134-878f-4891-ba4f-4df54f1e3ab4",
"1f62a176-c980-41dc-b3a1-c735d660c910"
]
names = ["S1A_WV_OCN__2SSV_20150526T081641_20150526T082418_006090_007E3E_104C",
"S1A_WV_OCN__2SSV_20150526T211029_20150526T211737_006097_007E78_134A",
"S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E"]
paths = [tmpdir.join(fn + ".zip") for fn in names]
path_strings = list(map(str, paths))
# Init files used for testing
api.download(ids[0], str(tmpdir))
# File #1: complete and correct
assert paths[0].check(exists=1, file=1)
# File #2: complete but incorrect
with paths[1].open("wb") as f:
size = 130102
f.seek(size - 1)
f.write(b'\0')
# File #3: incomplete
dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa'
with paths[2].open("wb") as f:
f.write(dummy_content)
assert paths[2].check(exists=1, file=1)
# Test
expected = {str(paths[1]), str(paths[2])}
result = api.check_files(ids=ids, directory=str(tmpdir))
assert set(result) == expected
assert result[paths[1]][0]['id'] == ids[1]
assert result[paths[2]][0]['id'] == ids[2]
assert paths[0].check(exists=1, file=1)
assert paths[1].check(exists=1, file=1)
assert paths[2].check(exists=1, file=1)
result = api.check_files(paths=path_strings)
assert set(result) == expected
assert result[paths[1]][0]['id'] == ids[1]
assert result[paths[2]][0]['id'] == ids[2]
assert paths[0].check(exists=1, file=1)
assert paths[1].check(exists=1, file=1)
assert paths[2].check(exists=1, file=1)
result = api.check_files(paths=path_strings, delete=True)
assert set(result) == expected
assert result[paths[1]][0]['id'] == ids[1]
assert result[paths[2]][0]['id'] == ids[2]
assert paths[0].check(exists=1, file=1)
assert not paths[1].check(exists=1, file=1)
assert not paths[2].check(exists=1, file=1)
missing_file = str(tmpdir.join(
"S1A_EW_GRDH_1SDH_20141003T003840_20141003T003920_002658_002F54_4DD1.zip"))
result = api.check_files(paths=[missing_file])
assert set(result) == {missing_file}
assert result[missing_file][0]['id']
with pytest.raises(ValueError):
api.check_files(ids=ids)
with pytest.raises(ValueError):
api.check_files()
tmpdir.remove()
def test_download_invalid_id():
api = SentinelAPI(**_api_auth)
uuid = "1f62a176-c980-41dc-xxxx-c735d660c910"
with pytest.raises(SentinelAPIError) as excinfo:
api.download(uuid)
assert 'Invalid key' in excinfo.value.msg