def s1_download(argument_list):
"""
This function will download S1 products from ASF mirror.
:param url: the url to the file you want to download
:param filename: the absolute path to where the downloaded file should
be written to
:param uname: ESA's scihub username
:param pword: ESA's scihub password
:return:
"""
url = argument_list[0]
filename = argument_list[1]
uname = argument_list[2]
pword = argument_list[3]
#session = SessionWithHeaderRedirection(uname, pword)
print(' INFO: Downloading scene to: {}'.format(filename))
# submit the request using the session
#response = session.get(url, stream=True)
# raise an exception in case of http errors
#response.raise_for_status()
# get download size
#total_length = int(response.headers.get('content-length', 0))
# define chunk_size
#chunk_size = 1024
# check if file is partially downloaded
if os.path.exists(filename):
os.remove(filename)
zip_test = 1
while zip_test is not None and zip_test <= 10:
command = 'wget --check-certificate=off -c -nv --http-user='******' --http-passwd="' + pword + '" -O ' + filename + ' ' + url
process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True)
astdout, astderr = process.communicate()
print(astdout, astderr)
print(' INFO: Checking the zip archive of {} for inconsistency'.format(
filename))
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downlaoded again
if zip_test is not None:
print(' INFO: {} did not pass the zip test. \
Re-downloading the full scene.'.format(filename))
if os.path.exists(filename):
os.remove(filename)
# otherwise we change the status to True
else:
print(' INFO: {} passed the zip test.'.format(filename))
with open(str('{}.downloaded'.format(filename)), 'w') as file:
file.write('successfully downloaded \n')
def restore_download_dir(input_directory, download_dir):
"""Create the OST download directory structure from downloaded files
In case data is already downloaded to a single folder, this function can
be used to create a OST compliant structure of the download directory.
:param input_directory: the directory, where the downloaded files
are located
:type input_directory: str/Path
:param download_dir: the high-level directory compliant with OST
:type download_dir: str/Path
"""
if isinstance(input_directory, str):
input_directory = Path(input_directory)
if isinstance(download_dir, str):
download_dir = Path(download_dir)
for scene in list(input_directory.glob('*zip')):
# get scene
s1scene = S1Scene(scene.name[:-4])
# create download path and file
file_path = s1scene.download_path(download_dir, True)
# check zipfile
logger.info(f'Checking zip file {str(scene)} for inconsistency.')
zip_test = h.check_zipfile(str(scene))
if not zip_test:
logger.info('Passed zip file test.')
scene.rename(file_path)
# add downloaded (should be zip checked in future)
with open(file_path.with_suffix('.downloaded'), 'w+') as file:
file.write('successfully downloaded \n')
else:
logger.info(
f'File {str(scene)} is corrupted and will not be moved.'
)
def restore_download_dir(input_directory, download_dir):
'''Function to create the OST download directory structure
In case data is already downloaded to a single folder, this function can
be used to create a OST compliant structure of the download directory.
Args:
input_directory: the directory, where the dwonloaded files are located
download_dir: the high-level directory compliant with OST
'''
from ost.helpers import helpers as h
for scene_in in glob.glob(opj(input_directory, '*zip')):
# get scene
scene = S1Scene(os.path.basename(scene_in)[:-4])
# create download path and file
filepath = scene._download_path(download_dir, True)
# check zipfile
print(
' INFO: Checking zip file {} for inconsistency.'.format(scene_in))
zip_test = h.check_zipfile(scene_in)
if not zip_test:
print(' INFO: Passed')
# move file
os.rename(scene_in, filepath)
# add downloaded (should be zip checked in future)
f = open(filepath + ".downloaded", "w+")
f.close()
else:
print(' INFO: File {} is corrupted and will not be moved.')
def s1_download(uuid,
filename,
uname,
pword,
base_url='https://scihub.copernicus.eu/apihub'):
"""Single scene download function for Copernicus scihub/apihub
:param uuid: product's uuid
:param filename: local path for the download
:param uname: username of Copernicus' scihub
:param pword: password of Copernicus' scihub
:param base_url:
:return:
"""
# get out the arguments
if isinstance(filename, str):
filename = Path(filename)
# ask for credentials in case they are not defined as input
if not uname or not pword:
ask_credentials()
# define url
url = f'{base_url}/odata/v1/Products(\'{uuid}\')/$value'
# get first response for file Size
response = requests.get(url, stream=True, auth=(uname, pword))
# check response
if response.status_code == 401:
raise ValueError(' ERROR: Username/Password are incorrect.')
elif response.status_code != 200:
print(' ERROR: Something went wrong, will try again in 30 seconds.')
response.raise_for_status()
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
first_byte = filename.stat().st_size if filename.exists() else 0
if first_byte >= total_length:
return
zip_test = 1
while zip_test is not None:
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": f"bytes={first_byte}-{total_length}"}
logger.info(f'Downloading scene to: {filename.name}')
response = requests.get(url,
headers=header,
stream=True,
auth=(uname, pword))
# actual download
with open(filename, "ab") as file:
pbar = tqdm.tqdm(total=total_length,
initial=first_byte,
unit='B',
unit_scale=True,
desc=' INFO: Downloading: ')
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# update first_byte
first_byte = filename.stat().st_size
# zipFile check
logger.info(f'Checking zip archive {filename.name} for inconsistency')
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downloaded again
if zip_test is not None:
logger.info(
f'{filename.name} did not pass the zip test. Re-downloading '
f'the full scene.')
filename.unlink()
first_byte = 0
# otherwise we change the status to True
else:
logger.info(f'{filename.name} passed the zip test.')
with open(filename.with_suffix('.downloaded'), 'w') as file:
file.write('successfully downloaded \n')
def s1_download(argument_list):
'''Function to download a single Sentinel-1 product from Copernicus scihub
This function will download S1 products from ESA's apihub.
Args:
argument_list: a list with 4 entries (this is used to enable parallel
execution)
argument_list[0] is the product's uuid
argument_list[1] is the local path for the download
argument_list[2] is the username of Copernicus' scihub
argument_list[3] is the password of Copernicus' scihub
'''
# get out the arguments
uuid = argument_list[0]
filename = argument_list[1]
uname = argument_list[2]
pword = argument_list[3]
# ask for username and password in case you have not defined as input
if not uname:
print(' If you do not have a Copernicus Scihub user'
' account go to: https://scihub.copernicus.eu')
uname = input(' Your Copernicus Scihub Username:'******' Your Copernicus Scihub Password:'******'https://scihub.copernicus.eu/apihub/odata/v1/'
'Products(\'{}\')/$value'.format(uuid))
# get first response for file Size
response = requests.get(url, stream=True, auth=(uname, pword))
# check response
if response.status_code == 401:
raise ValueError(' ERROR: Username/Password are incorrect.')
elif response.status_code != 200:
print(' ERROR: Something went wrong, will try again in 30 seconds.')
response.raise_for_status()
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
if os.path.exists(filename):
first_byte = os.path.getsize(filename)
else:
first_byte = 0
if first_byte >= total_length:
return total_length
zip_test = 1
while zip_test is not None:
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": "bytes={}-{}".format(first_byte, total_length)}
print(' INFO: Downloading scene to: {}'.format(filename))
response = requests.get(url,
headers=header,
stream=True,
auth=(uname, pword))
# actual download
with open(filename, "ab") as file:
if total_length is None:
file.write(response.content)
else:
pbar = tqdm.tqdm(total=total_length,
initial=first_byte,
unit='B',
unit_scale=True,
desc=' INFO: Downloading: ')
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# update first_byte
first_byte = os.path.getsize(filename)
# zipFile check
print(' INFO: Checking the zip archive of {} for inconsistency'.format(
filename))
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downlaoded again
if zip_test is not None:
print(' INFO: {} did not pass the zip test. \
Re-downloading the full scene.'.format(filename))
os.remove(filename)
first_byte = 0
# otherwise we change the status to True
else:
print(' INFO: {} passed the zip test.'.format(filename))
with open(str('{}.downloaded'.format(filename)), 'w') as file:
file.write('successfully downloaded \n')
def peps_download(argument_list):
"""Single scene download function for Copernicus scihub/apihub
:param argument_list:
a list with 4 entries (this is used to enable parallel execution)
argument_list[0]: product's url
argument_list[1]: local path for the download
argument_list[2]: username of Copernicus' scihub
argument_list[3]: password of Copernicus' scihub
:return:
"""
url, filename, uname, pword = argument_list
filename = Path(filename)
# get first response for file Size
response = requests.get(url, stream=True, auth=(uname, pword))
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
if filename.exists():
first_byte = filename.stat().st_size
if first_byte == total_length:
logger.info(f'{filename.name} already downloaded.')
else:
logger.info(f'Continue downloading scene to: {filename.name}')
else:
logger.info(f'Downloading scene to: {filename.resolve()}')
first_byte = 0
if first_byte >= total_length:
return total_length
zip_test = 1
while zip_test is not None and zip_test <= 10:
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": f"bytes={first_byte}-{total_length}"}
response = requests.get(url,
headers=header,
stream=True,
auth=(uname, pword))
# actual download
with open(filename, "ab") as file:
if total_length is None:
file.write(response.content)
else:
pbar = tqdm.tqdm(total=total_length,
initial=first_byte,
unit='B',
unit_scale=True,
desc=' INFO: Downloading: ')
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# updated fileSize
first_byte = filename.stat().st_size
# zipFile check
logger.info(
f'Checking the zip archive of {filename.name} for inconsistency')
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downlaoded again
if zip_test is not None:
logger.info(f'{filename.name} did not pass the zip test. '
f'Re-downloading the full scene.')
filename.unlink()
first_byte = 0
# otherwise we change the status to True
else:
logger.info(f'{filename} passed the zip test.')
with open(filename.with_suffix('.downloaded'), 'w') as file:
file.write('successfully downloaded \n')
def s1_download(argument_list):
"""
This function will download S1 products from ASF mirror.
:param url: the url to the file you want to download
:param filename: the absolute path to where the downloaded file should
be written to
:param uname: ESA's scihub username
:param pword: ESA's scihub password
:return:
"""
url = argument_list[0]
filename = argument_list[1]
uname = argument_list[2]
pword = argument_list[3]
session = SessionWithHeaderRedirection(uname, pword)
print(' INFO: Downloading scene to: {}'.format(filename))
# submit the request using the session
response = session.get(url, stream=True)
# raise an exception in case of http errors
response.raise_for_status()
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
if os.path.exists(filename):
first_byte = os.path.getsize(filename)
else:
first_byte = 0
zip_test = 1
while zip_test is not None and zip_test <= 10:
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": "bytes={}-{}".format(first_byte, total_length)}
response = session.get(url, headers=header, stream=True)
# actual download
with open(filename, "ab") as file:
if total_length is None:
file.write(response.content)
else:
pbar = tqdm.tqdm(total=total_length,
initial=first_byte,
unit='B',
unit_scale=True,
desc=' INFO: Downloading ')
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# updated fileSize
first_byte = os.path.getsize(filename)
print(' INFO: Checking the zip archive of {} for inconsistency'.format(
filename))
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downlaoded again
if zip_test is not None:
print(' INFO: {} did not pass the zip test. \
Re-downloading the full scene.'.format(filename))
if os.path.exists(filename):
os.remove(filename)
first_byte = 0
# otherwise we change the status to True
else:
print(' INFO: {} passed the zip test.'.format(filename))
with open(str('{}.downloaded'.format(filename)), 'w') as file:
file.write('successfully downloaded \n')
def download_srtm_tile(url):
snap_aux = Path.home().joinpath('.snap/auxdata/dem/SRTM 1Sec HGT/')
if not snap_aux.exists():
raise RuntimeError(' Snap aux folder not found')
filename = snap_aux.joinpath(url.split('/')[-1])
# get first response for file Size
response = requests.get(url, stream=True)
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
first_byte = filename.stat().st_size if filename.exists() else 0
if first_byte >= total_length:
return
zip_test = 1
while zip_test is not None:
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": f"bytes={first_byte}-{total_length}"}
# logger.info(f'Downloading scene to: {filename.name}')
response = requests.get(
url, headers=header, stream=True
)
# actual download
with open(filename, "ab") as file:
pbar = tqdm.tqdm(
total=total_length, initial=first_byte, unit='B',
unit_scale=True, desc=' INFO: Downloading: '
)
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# update first_byte
first_byte = filename.stat().st_size
# zipFile check
logger.info(f'Checking zip archive {filename.name} for inconsistency')
zip_test = h.check_zipfile(filename)
if zip_test is not None:
logger.info(
f'{filename.name} did not pass the zip test. Re-downloading '
f'the full scene.'
)
filename.unlink()
first_byte = 0
# otherwise we change the status to True
else:
logger.info(f'{filename.name} passed the zip test.')
def s1_download(argument_list):
'''Function to download a single Sentinel-1 product from CNES' PEPS
Args:
argument_list: a list with 4 entries (this is used to enable parallel
execution)
argument_list[0] is the product's url
argument_list[1] is the local path for the download
argument_list[2] is the username of CNES' PEPS
argument_list[3] is the password of CNES' PEPS
'''
url = argument_list[0]
filename = argument_list[1]
uname = argument_list[2]
pword = argument_list[3]
# get first response for file Size
response = requests.get(url, stream=True, auth=(uname, pword))
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
if os.path.exists(filename):
first_byte = os.path.getsize(filename)
if first_byte == total_length:
print(' INFO: {} already downloaded.'.format(filename))
else:
print(' INFO: Continue downloading scene to: {}'.format(filename))
else:
print(' INFO: Downloading scene to: {}'.format(filename))
first_byte = 0
if first_byte >= total_length:
return total_length
zip_test = 1
while zip_test is not None and zip_test <= 10:
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": "bytes={}-{}".format(first_byte, total_length)}
response = requests.get(url,
headers=header,
stream=True,
auth=(uname, pword))
# actual download
with open(filename, "ab") as file:
if total_length is None:
file.write(response.content)
else:
pbar = tqdm.tqdm(total=total_length,
initial=first_byte,
unit='B',
unit_scale=True,
desc=' INFO: Downloading: ')
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# updated fileSize
first_byte = os.path.getsize(filename)
# zipFile check
print(' INFO: Checking the zip archive of {} for inconsistency'.format(
filename))
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downlaoded again
if zip_test is not None:
print(' INFO: {} did not pass the zip test. \
Re-downloading the full scene.'.format(filename))
os.remove(filename)
first_byte = 0
# otherwise we change the status to True
else:
print(' INFO: {} passed the zip test.'.format(filename))
with open(str('{}.downloaded'.format(filename)), 'w') as file:
file.write('successfully downloaded \n')
def onda_download(argument_list):
"""Single scene download function for Copernicus scihub/apihub
:param argument_list:
a list with 4 entries (this is used to enable parallel execution)
argument_list[0]: product's uuid
argument_list[1]: local path for the download
argument_list[2]: username of ONDA Dias
argument_list[3]: password of ONDA Dias
:return:
"""
# get out the arguments
uuid, filename, uname, pword = argument_list
filename = Path(filename)
# ask for username and password in case you have not defined as input
if not uname:
print(' If you do not have a ONDA DIAS user'
' account go to: https://www.onda-dias.eu/cms/')
uname = input(' Your ONDA DIAS Username:'******' Your ONDA DIAS Password:'******'' around the product uuid)
url = (f'https://catalogue.onda-dias.eu/dias-catalogue/'
f'Products({uuid})/$value')
# get first response for file Size
response = requests.get(url, stream=True, auth=(uname, pword))
# check response
if response.status_code == 401:
raise ValueError(' ERROR: Username/Password are incorrect.')
elif response.status_code != 200:
print(' ERROR: Something went wrong, will try again in 30 seconds.')
response.raise_for_status()
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
if filename.exists():
first_byte = filename.stat().st_size
else:
first_byte = 0
if first_byte >= total_length:
return total_length
zip_test = 1
while zip_test is not None:
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": f"bytes={first_byte}-{total_length}"}
logger.info(f'Downloading scene to: {filename.resolve()}')
response = requests.get(url,
headers=header,
stream=True,
auth=(uname, pword))
# actual download
with open(filename, "ab") as file:
if total_length is None:
file.write(response.content)
else:
pbar = tqdm.tqdm(total=total_length,
initial=first_byte,
unit='B',
unit_scale=True,
desc=' INFO: Downloading: ')
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# update first_byte
first_byte = filename.stat().st_size
# zipFile check
logger.info(
f'Checking the zip archive of {filename.name} for inconsistency')
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downlaoded again
if zip_test is not None:
logger.info(f'{filename.name} did not pass the zip test. '
f'Re-downloading the full scene.')
filename.unlink()
first_byte = 0
# otherwise we change the status to True
else:
logger.info(f'{filename.name} passed the zip test.')
with open(filename.with_suffix('.downloaded'), 'w') as file:
file.write('successfully downloaded \n')
def asf_download(url, filename, uname, pword):
"""
This function will download S1 products from ASF mirror.
:param url: the url to the file you want to download
:param filename: the absolute path to where the downloaded file should
be written to
:param uname: ESA's scihub username
:param pword: ESA's scihub password
:return:
"""
# extract list of args
if isinstance(filename, str):
filename = Path(filename)
with requests.Session() as session:
session.auth = (uname, pword)
request = session.request('get', url)
response = session.get(request.url, auth=(uname, pword), stream=True)
# raise an exception in case of http errors
response.raise_for_status()
logger.info(f'Downloading scene to: {filename}')
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
if filename.exists():
first_byte = filename.stat().st_size
else:
first_byte = 0
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": f"bytes={first_byte}-{total_length}"}
response = session.get(url, headers=header, stream=True)
# actual download
with open(filename, "ab") as file:
if total_length is None:
file.write(response.content)
else:
pbar = tqdm.tqdm(total=total_length, initial=first_byte,
unit='B', unit_scale=True,
desc=' INFO: Downloading ')
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# update file size
first_byte = filename.stat().st_size
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downloaded again
if zip_test is not None:
if filename.exists():
filename.unlink()
raise DownloadError(
f'{filename.name} did not pass the zip test. '
f'Re-downloading the full scene.'
)
else:
logger.info(f'{filename.name} passed the zip test.')
with open(filename.with_suffix('.downloaded'), 'w+') as file:
file.write('successfully downloaded \n')