def restore_download_dir(input_directory, download_dir):
'''Function to create the OST download directory structure
In case data is already downloaded to a single folder, this function can
be used to create a OST compliant structure of the download directory.
Args:
input_directory: the directory, where the dwonloaded files are located
download_dir: the high-level directory compliant with OST
'''
from ost.helpers import utils as h
for scene_in in glob.glob(opj(input_directory, '*zip')):
# get scene
scene = S1Scene(os.path.basename(scene_in)[:-4])
# create download path and file
filepath = scene._download_path(download_dir, True)
# check zipfile
logger.debug(
'INFO: Checking zip file {} for inconsistency.'.format(scene_in)
)
zip_test = h.check_zipfile(scene_in)
if not zip_test:
logger.debug('INFO: Passed')
# move file
os.rename(scene_in, filepath)
# add downloaded (should be zip checked in future)
f = open(filepath+".downloaded", "w+")
f.close()
else:
logger.debug('INFO: File {} is corrupted and will not be moved.')
def test_asf_download(s1_grd_notnr_ost_product, mirror=2):
herbert_uname = HERBERT_USER['uname']
herbert_password = HERBERT_USER['asf_pword']
df = pd.DataFrame({'identifier': [s1_grd_notnr_ost_product[1].scene_id]})
with TemporaryDirectory(dir=os.getcwd()) as temp:
download_dir, missing_products = download_sentinel1(
inventory_df=df,
download_dir=temp,
mirror=mirror,
concurrent=1,
uname=herbert_uname,
pword=herbert_password)
from ost.helpers.utils import check_zipfile
product_path = s1_grd_notnr_ost_product[1].get_path(
download_dir=download_dir, data_mount='/eodata')
return_code = check_zipfile(product_path)
assert return_code is None
def s1_download(argument_list):
'''Function to download a single Sentinel-1 product from CNES' PEPS
Args:
argument_list: a list with 4 entries (this is used to enable parallel
execution)
argument_list[0] is the product's url
argument_list[1] is the local path for the download
argument_list[2] is the username of CNES' PEPS
argument_list[3] is the password of CNES' PEPS
'''
url = argument_list[0]
filename = argument_list[1]
uname = argument_list[2]
pword = argument_list[3]
# get first response for file Size
response = requests.get(url, stream=True, auth=(uname, pword))
# get download size
total_length = int(response.headers.get('content-length', 0))
# define chunk_size
chunk_size = 1024
# check if file is partially downloaded
if os.path.exists(filename):
first_byte = os.path.getsize(filename)
if first_byte == total_length:
logger.debug('INFO: {} already downloaded.'.format(filename))
else:
logger.debug(
'INFO: Continue downloading scene to: {}'.format(filename))
else:
logger.debug('INFO: Downloading scene to: {}'.format(filename))
first_byte = 0
if first_byte >= total_length:
return total_length
zip_test = 1
while zip_test is not None and zip_test <= 10:
while first_byte < total_length:
# get byte offset for already downloaded file
header = {"Range": "bytes={}-{}".format(first_byte, total_length)}
response = requests.get(url,
headers=header,
stream=True,
auth=(uname, pword))
# actual download
with open(filename, "ab") as file:
if total_length is None:
file.write(response.content)
else:
pbar = tqdm.tqdm(total=total_length,
initial=first_byte,
unit='B',
unit_scale=True,
desc=' INFO: Downloading: ')
for chunk in response.iter_content(chunk_size):
if chunk:
file.write(chunk)
pbar.update(chunk_size)
pbar.close()
# updated fileSize
first_byte = os.path.getsize(filename)
# zipFile check
logger.debug(
'INFO: Checking the zip archive of {} for inconsistency'.format(
filename))
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downlaoded again
if zip_test is not None:
logger.debug('INFO: {} did not pass the zip test. \
Re-downloading the full scene.'.format(filename))
os.remove(filename)
first_byte = 0
# otherwise we change the status to True
else:
logger.debug('INFO: {} passed the zip test.'.format(filename))
with open(str('{}.downloaded'.format(filename)), 'w') as file:
file.write('successfully downloaded \n')
def s1_download(argument_list):
'''Function to download a single Sentinel-1 product from Copernicus scihub
This function will download S1 products from ESA's apihub.
Args:
argument_list: a list with 4 entries (this is used to enable parallel
execution)
argument_list[0] is the product's uuid
argument_list[1] is the local path for the download
argument_list[2] is the username of Copernicus' scihub
argument_list[3] is the password of Copernicus' scihub
'''
# get out the arguments
uuid = argument_list[0]
filename = argument_list[1]
uname = argument_list[2]
pword = argument_list[3]
# ask for username and password in case you have not defined as input
if not uname:
logger.debug('If you do not have a Copernicus Scihub user'
' account go to: https://scihub.copernicus.eu')
uname = input(' Your Copernicus Scihub Username:'******' Your Copernicus Scihub Password:'******'https://scihub.copernicus.eu/apihub/odata/v1/'
'Products(\'{}\')/$value'.format(uuid))
# get first response for file Size
response = requests.get(url, stream=True, auth=(uname, pword))
# check response
if response.status_code == 401:
raise ValueError(' ERROR: Username/Password are incorrect.')
elif response.status_code == 404:
logger.debug('Product %s missing from the archive, continuing.',
filename.split('/')[-1])
return filename.split('/')[-1]
elif response.status_code != 200:
logger.debug(
'ERROR: Something went wrong, will try again in 30 seconds.')
response.raise_for_status()
password_manager = urlreq.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, "https://scihub.copernicus.eu/apihub/",
uname, pword)
cookie_jar = CookieJar()
opener = urlreq.build_opener(urlreq.HTTPBasicAuthHandler(password_manager),
urlreq.HTTPCookieProcessor(cookie_jar))
urlreq.install_opener(opener)
# get download size
total_length = int(response.headers.get('content-length', 0))
# check if file is partially downloaded
if os.path.exists(filename):
first_byte = os.path.getsize(filename)
else:
first_byte = 0
if first_byte >= total_length:
return str('{}.downloaded'.format(filename))
zip_test = 1
while zip_test is not None and zip_test <= 10:
logger.debug('INFO: Downloading scene to: {}'.format(filename))
with TqdmUpTo(unit='B',
unit_scale=True,
miniters=1,
desc=url.split('/')[-1]) as t:
filename, headers = urlreq.urlretrieve(url,
filename=filename,
reporthook=t.update_to)
# zipFile check
logger.debug(
'INFO: Checking the zip archive of {} for inconsistency'.format(
filename))
zip_test = h.check_zipfile(filename)
# if it did not pass the test, remove the file
# in the while loop it will be downlaoded again
if zip_test is not None:
logger.debug('INFO: {} did not pass the zip test. \
Re-downloading the full scene.'.format(filename))
os.remove(filename)
first_byte = 0
# otherwise we change the status to True
else:
logger.debug('INFO: {} passed the zip test.'.format(filename))
with open(str('{}.downloaded'.format(filename)), 'w') as file:
file.write('successfully downloaded \n')
return str('{}.downloaded'.format(filename))