def s1_download(argument_list): """ This function will download S1 products from ASF mirror. :param url: the url to the file you want to download :param filename: the absolute path to where the downloaded file should be written to :param uname: ESA's scihub username :param pword: ESA's scihub password :return: """ url = argument_list[0] filename = argument_list[1] uname = argument_list[2] pword = argument_list[3] #session = SessionWithHeaderRedirection(uname, pword) print(' INFO: Downloading scene to: {}'.format(filename)) # submit the request using the session #response = session.get(url, stream=True) # raise an exception in case of http errors #response.raise_for_status() # get download size #total_length = int(response.headers.get('content-length', 0)) # define chunk_size #chunk_size = 1024 # check if file is partially downloaded if os.path.exists(filename): os.remove(filename) zip_test = 1 while zip_test is not None and zip_test <= 10: command = 'wget --check-certificate=off -c -nv --http-user='******' --http-passwd="' + pword + '" -O ' + filename + ' ' + url process = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) astdout, astderr = process.communicate() print(astdout, astderr) print(' INFO: Checking the zip archive of {} for inconsistency'.format( filename)) zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downlaoded again if zip_test is not None: print(' INFO: {} did not pass the zip test. \ Re-downloading the full scene.'.format(filename)) if os.path.exists(filename): os.remove(filename) # otherwise we change the status to True else: print(' INFO: {} passed the zip test.'.format(filename)) with open(str('{}.downloaded'.format(filename)), 'w') as file: file.write('successfully downloaded \n')
def restore_download_dir(input_directory, download_dir): """Create the OST download directory structure from downloaded files In case data is already downloaded to a single folder, this function can be used to create a OST compliant structure of the download directory. :param input_directory: the directory, where the downloaded files are located :type input_directory: str/Path :param download_dir: the high-level directory compliant with OST :type download_dir: str/Path """ if isinstance(input_directory, str): input_directory = Path(input_directory) if isinstance(download_dir, str): download_dir = Path(download_dir) for scene in list(input_directory.glob('*zip')): # get scene s1scene = S1Scene(scene.name[:-4]) # create download path and file file_path = s1scene.download_path(download_dir, True) # check zipfile logger.info(f'Checking zip file {str(scene)} for inconsistency.') zip_test = h.check_zipfile(str(scene)) if not zip_test: logger.info('Passed zip file test.') scene.rename(file_path) # add downloaded (should be zip checked in future) with open(file_path.with_suffix('.downloaded'), 'w+') as file: file.write('successfully downloaded \n') else: logger.info( f'File {str(scene)} is corrupted and will not be moved.' )
def restore_download_dir(input_directory, download_dir): '''Function to create the OST download directory structure In case data is already downloaded to a single folder, this function can be used to create a OST compliant structure of the download directory. Args: input_directory: the directory, where the dwonloaded files are located download_dir: the high-level directory compliant with OST ''' from ost.helpers import helpers as h for scene_in in glob.glob(opj(input_directory, '*zip')): # get scene scene = S1Scene(os.path.basename(scene_in)[:-4]) # create download path and file filepath = scene._download_path(download_dir, True) # check zipfile print( ' INFO: Checking zip file {} for inconsistency.'.format(scene_in)) zip_test = h.check_zipfile(scene_in) if not zip_test: print(' INFO: Passed') # move file os.rename(scene_in, filepath) # add downloaded (should be zip checked in future) f = open(filepath + ".downloaded", "w+") f.close() else: print(' INFO: File {} is corrupted and will not be moved.')
def s1_download(uuid, filename, uname, pword, base_url='https://scihub.copernicus.eu/apihub'): """Single scene download function for Copernicus scihub/apihub :param uuid: product's uuid :param filename: local path for the download :param uname: username of Copernicus' scihub :param pword: password of Copernicus' scihub :param base_url: :return: """ # get out the arguments if isinstance(filename, str): filename = Path(filename) # ask for credentials in case they are not defined as input if not uname or not pword: ask_credentials() # define url url = f'{base_url}/odata/v1/Products(\'{uuid}\')/$value' # get first response for file Size response = requests.get(url, stream=True, auth=(uname, pword)) # check response if response.status_code == 401: raise ValueError(' ERROR: Username/Password are incorrect.') elif response.status_code != 200: print(' ERROR: Something went wrong, will try again in 30 seconds.') response.raise_for_status() # get download size total_length = int(response.headers.get('content-length', 0)) # define chunk_size chunk_size = 1024 # check if file is partially downloaded first_byte = filename.stat().st_size if filename.exists() else 0 if first_byte >= total_length: return zip_test = 1 while zip_test is not None: while first_byte < total_length: # get byte offset for already downloaded file header = {"Range": f"bytes={first_byte}-{total_length}"} logger.info(f'Downloading scene to: {filename.name}') response = requests.get(url, headers=header, stream=True, auth=(uname, pword)) # actual download with open(filename, "ab") as file: pbar = tqdm.tqdm(total=total_length, initial=first_byte, unit='B', unit_scale=True, desc=' INFO: Downloading: ') for chunk in response.iter_content(chunk_size): if chunk: file.write(chunk) pbar.update(chunk_size) pbar.close() # update first_byte first_byte = filename.stat().st_size # zipFile check logger.info(f'Checking zip archive {filename.name} for inconsistency') zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downloaded again if zip_test is not None: logger.info( f'{filename.name} did not pass the zip test. Re-downloading ' f'the full scene.') filename.unlink() first_byte = 0 # otherwise we change the status to True else: logger.info(f'{filename.name} passed the zip test.') with open(filename.with_suffix('.downloaded'), 'w') as file: file.write('successfully downloaded \n')
def s1_download(argument_list): '''Function to download a single Sentinel-1 product from Copernicus scihub This function will download S1 products from ESA's apihub. Args: argument_list: a list with 4 entries (this is used to enable parallel execution) argument_list[0] is the product's uuid argument_list[1] is the local path for the download argument_list[2] is the username of Copernicus' scihub argument_list[3] is the password of Copernicus' scihub ''' # get out the arguments uuid = argument_list[0] filename = argument_list[1] uname = argument_list[2] pword = argument_list[3] # ask for username and password in case you have not defined as input if not uname: print(' If you do not have a Copernicus Scihub user' ' account go to: https://scihub.copernicus.eu') uname = input(' Your Copernicus Scihub Username:'******' Your Copernicus Scihub Password:'******'https://scihub.copernicus.eu/apihub/odata/v1/' 'Products(\'{}\')/$value'.format(uuid)) # get first response for file Size response = requests.get(url, stream=True, auth=(uname, pword)) # check response if response.status_code == 401: raise ValueError(' ERROR: Username/Password are incorrect.') elif response.status_code != 200: print(' ERROR: Something went wrong, will try again in 30 seconds.') response.raise_for_status() # get download size total_length = int(response.headers.get('content-length', 0)) # define chunk_size chunk_size = 1024 # check if file is partially downloaded if os.path.exists(filename): first_byte = os.path.getsize(filename) else: first_byte = 0 if first_byte >= total_length: return total_length zip_test = 1 while zip_test is not None: while first_byte < total_length: # get byte offset for already downloaded file header = {"Range": "bytes={}-{}".format(first_byte, total_length)} print(' INFO: Downloading scene to: {}'.format(filename)) response = requests.get(url, headers=header, stream=True, auth=(uname, pword)) # actual download with open(filename, "ab") as file: if total_length is None: file.write(response.content) else: pbar = tqdm.tqdm(total=total_length, initial=first_byte, unit='B', unit_scale=True, desc=' INFO: Downloading: ') for chunk in response.iter_content(chunk_size): if chunk: file.write(chunk) pbar.update(chunk_size) pbar.close() # update first_byte first_byte = os.path.getsize(filename) # zipFile check print(' INFO: Checking the zip archive of {} for inconsistency'.format( filename)) zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downlaoded again if zip_test is not None: print(' INFO: {} did not pass the zip test. \ Re-downloading the full scene.'.format(filename)) os.remove(filename) first_byte = 0 # otherwise we change the status to True else: print(' INFO: {} passed the zip test.'.format(filename)) with open(str('{}.downloaded'.format(filename)), 'w') as file: file.write('successfully downloaded \n')
def peps_download(argument_list): """Single scene download function for Copernicus scihub/apihub :param argument_list: a list with 4 entries (this is used to enable parallel execution) argument_list[0]: product's url argument_list[1]: local path for the download argument_list[2]: username of Copernicus' scihub argument_list[3]: password of Copernicus' scihub :return: """ url, filename, uname, pword = argument_list filename = Path(filename) # get first response for file Size response = requests.get(url, stream=True, auth=(uname, pword)) # get download size total_length = int(response.headers.get('content-length', 0)) # define chunk_size chunk_size = 1024 # check if file is partially downloaded if filename.exists(): first_byte = filename.stat().st_size if first_byte == total_length: logger.info(f'{filename.name} already downloaded.') else: logger.info(f'Continue downloading scene to: {filename.name}') else: logger.info(f'Downloading scene to: {filename.resolve()}') first_byte = 0 if first_byte >= total_length: return total_length zip_test = 1 while zip_test is not None and zip_test <= 10: while first_byte < total_length: # get byte offset for already downloaded file header = {"Range": f"bytes={first_byte}-{total_length}"} response = requests.get(url, headers=header, stream=True, auth=(uname, pword)) # actual download with open(filename, "ab") as file: if total_length is None: file.write(response.content) else: pbar = tqdm.tqdm(total=total_length, initial=first_byte, unit='B', unit_scale=True, desc=' INFO: Downloading: ') for chunk in response.iter_content(chunk_size): if chunk: file.write(chunk) pbar.update(chunk_size) pbar.close() # updated fileSize first_byte = filename.stat().st_size # zipFile check logger.info( f'Checking the zip archive of {filename.name} for inconsistency') zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downlaoded again if zip_test is not None: logger.info(f'{filename.name} did not pass the zip test. ' f'Re-downloading the full scene.') filename.unlink() first_byte = 0 # otherwise we change the status to True else: logger.info(f'{filename} passed the zip test.') with open(filename.with_suffix('.downloaded'), 'w') as file: file.write('successfully downloaded \n')
def s1_download(argument_list): """ This function will download S1 products from ASF mirror. :param url: the url to the file you want to download :param filename: the absolute path to where the downloaded file should be written to :param uname: ESA's scihub username :param pword: ESA's scihub password :return: """ url = argument_list[0] filename = argument_list[1] uname = argument_list[2] pword = argument_list[3] session = SessionWithHeaderRedirection(uname, pword) print(' INFO: Downloading scene to: {}'.format(filename)) # submit the request using the session response = session.get(url, stream=True) # raise an exception in case of http errors response.raise_for_status() # get download size total_length = int(response.headers.get('content-length', 0)) # define chunk_size chunk_size = 1024 # check if file is partially downloaded if os.path.exists(filename): first_byte = os.path.getsize(filename) else: first_byte = 0 zip_test = 1 while zip_test is not None and zip_test <= 10: while first_byte < total_length: # get byte offset for already downloaded file header = {"Range": "bytes={}-{}".format(first_byte, total_length)} response = session.get(url, headers=header, stream=True) # actual download with open(filename, "ab") as file: if total_length is None: file.write(response.content) else: pbar = tqdm.tqdm(total=total_length, initial=first_byte, unit='B', unit_scale=True, desc=' INFO: Downloading ') for chunk in response.iter_content(chunk_size): if chunk: file.write(chunk) pbar.update(chunk_size) pbar.close() # updated fileSize first_byte = os.path.getsize(filename) print(' INFO: Checking the zip archive of {} for inconsistency'.format( filename)) zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downlaoded again if zip_test is not None: print(' INFO: {} did not pass the zip test. \ Re-downloading the full scene.'.format(filename)) if os.path.exists(filename): os.remove(filename) first_byte = 0 # otherwise we change the status to True else: print(' INFO: {} passed the zip test.'.format(filename)) with open(str('{}.downloaded'.format(filename)), 'w') as file: file.write('successfully downloaded \n')
def download_srtm_tile(url): snap_aux = Path.home().joinpath('.snap/auxdata/dem/SRTM 1Sec HGT/') if not snap_aux.exists(): raise RuntimeError(' Snap aux folder not found') filename = snap_aux.joinpath(url.split('/')[-1]) # get first response for file Size response = requests.get(url, stream=True) # get download size total_length = int(response.headers.get('content-length', 0)) # define chunk_size chunk_size = 1024 # check if file is partially downloaded first_byte = filename.stat().st_size if filename.exists() else 0 if first_byte >= total_length: return zip_test = 1 while zip_test is not None: while first_byte < total_length: # get byte offset for already downloaded file header = {"Range": f"bytes={first_byte}-{total_length}"} # logger.info(f'Downloading scene to: {filename.name}') response = requests.get( url, headers=header, stream=True ) # actual download with open(filename, "ab") as file: pbar = tqdm.tqdm( total=total_length, initial=first_byte, unit='B', unit_scale=True, desc=' INFO: Downloading: ' ) for chunk in response.iter_content(chunk_size): if chunk: file.write(chunk) pbar.update(chunk_size) pbar.close() # update first_byte first_byte = filename.stat().st_size # zipFile check logger.info(f'Checking zip archive {filename.name} for inconsistency') zip_test = h.check_zipfile(filename) if zip_test is not None: logger.info( f'{filename.name} did not pass the zip test. Re-downloading ' f'the full scene.' ) filename.unlink() first_byte = 0 # otherwise we change the status to True else: logger.info(f'{filename.name} passed the zip test.')
def s1_download(argument_list): '''Function to download a single Sentinel-1 product from CNES' PEPS Args: argument_list: a list with 4 entries (this is used to enable parallel execution) argument_list[0] is the product's url argument_list[1] is the local path for the download argument_list[2] is the username of CNES' PEPS argument_list[3] is the password of CNES' PEPS ''' url = argument_list[0] filename = argument_list[1] uname = argument_list[2] pword = argument_list[3] # get first response for file Size response = requests.get(url, stream=True, auth=(uname, pword)) # get download size total_length = int(response.headers.get('content-length', 0)) # define chunk_size chunk_size = 1024 # check if file is partially downloaded if os.path.exists(filename): first_byte = os.path.getsize(filename) if first_byte == total_length: print(' INFO: {} already downloaded.'.format(filename)) else: print(' INFO: Continue downloading scene to: {}'.format(filename)) else: print(' INFO: Downloading scene to: {}'.format(filename)) first_byte = 0 if first_byte >= total_length: return total_length zip_test = 1 while zip_test is not None and zip_test <= 10: while first_byte < total_length: # get byte offset for already downloaded file header = {"Range": "bytes={}-{}".format(first_byte, total_length)} response = requests.get(url, headers=header, stream=True, auth=(uname, pword)) # actual download with open(filename, "ab") as file: if total_length is None: file.write(response.content) else: pbar = tqdm.tqdm(total=total_length, initial=first_byte, unit='B', unit_scale=True, desc=' INFO: Downloading: ') for chunk in response.iter_content(chunk_size): if chunk: file.write(chunk) pbar.update(chunk_size) pbar.close() # updated fileSize first_byte = os.path.getsize(filename) # zipFile check print(' INFO: Checking the zip archive of {} for inconsistency'.format( filename)) zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downlaoded again if zip_test is not None: print(' INFO: {} did not pass the zip test. \ Re-downloading the full scene.'.format(filename)) os.remove(filename) first_byte = 0 # otherwise we change the status to True else: print(' INFO: {} passed the zip test.'.format(filename)) with open(str('{}.downloaded'.format(filename)), 'w') as file: file.write('successfully downloaded \n')
def onda_download(argument_list): """Single scene download function for Copernicus scihub/apihub :param argument_list: a list with 4 entries (this is used to enable parallel execution) argument_list[0]: product's uuid argument_list[1]: local path for the download argument_list[2]: username of ONDA Dias argument_list[3]: password of ONDA Dias :return: """ # get out the arguments uuid, filename, uname, pword = argument_list filename = Path(filename) # ask for username and password in case you have not defined as input if not uname: print(' If you do not have a ONDA DIAS user' ' account go to: https://www.onda-dias.eu/cms/') uname = input(' Your ONDA DIAS Username:'******' Your ONDA DIAS Password:'******'' around the product uuid) url = (f'https://catalogue.onda-dias.eu/dias-catalogue/' f'Products({uuid})/$value') # get first response for file Size response = requests.get(url, stream=True, auth=(uname, pword)) # check response if response.status_code == 401: raise ValueError(' ERROR: Username/Password are incorrect.') elif response.status_code != 200: print(' ERROR: Something went wrong, will try again in 30 seconds.') response.raise_for_status() # get download size total_length = int(response.headers.get('content-length', 0)) # define chunk_size chunk_size = 1024 # check if file is partially downloaded if filename.exists(): first_byte = filename.stat().st_size else: first_byte = 0 if first_byte >= total_length: return total_length zip_test = 1 while zip_test is not None: while first_byte < total_length: # get byte offset for already downloaded file header = {"Range": f"bytes={first_byte}-{total_length}"} logger.info(f'Downloading scene to: {filename.resolve()}') response = requests.get(url, headers=header, stream=True, auth=(uname, pword)) # actual download with open(filename, "ab") as file: if total_length is None: file.write(response.content) else: pbar = tqdm.tqdm(total=total_length, initial=first_byte, unit='B', unit_scale=True, desc=' INFO: Downloading: ') for chunk in response.iter_content(chunk_size): if chunk: file.write(chunk) pbar.update(chunk_size) pbar.close() # update first_byte first_byte = filename.stat().st_size # zipFile check logger.info( f'Checking the zip archive of {filename.name} for inconsistency') zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downlaoded again if zip_test is not None: logger.info(f'{filename.name} did not pass the zip test. ' f'Re-downloading the full scene.') filename.unlink() first_byte = 0 # otherwise we change the status to True else: logger.info(f'{filename.name} passed the zip test.') with open(filename.with_suffix('.downloaded'), 'w') as file: file.write('successfully downloaded \n')
def asf_download(url, filename, uname, pword): """ This function will download S1 products from ASF mirror. :param url: the url to the file you want to download :param filename: the absolute path to where the downloaded file should be written to :param uname: ESA's scihub username :param pword: ESA's scihub password :return: """ # extract list of args if isinstance(filename, str): filename = Path(filename) with requests.Session() as session: session.auth = (uname, pword) request = session.request('get', url) response = session.get(request.url, auth=(uname, pword), stream=True) # raise an exception in case of http errors response.raise_for_status() logger.info(f'Downloading scene to: {filename}') # get download size total_length = int(response.headers.get('content-length', 0)) # define chunk_size chunk_size = 1024 # check if file is partially downloaded if filename.exists(): first_byte = filename.stat().st_size else: first_byte = 0 while first_byte < total_length: # get byte offset for already downloaded file header = {"Range": f"bytes={first_byte}-{total_length}"} response = session.get(url, headers=header, stream=True) # actual download with open(filename, "ab") as file: if total_length is None: file.write(response.content) else: pbar = tqdm.tqdm(total=total_length, initial=first_byte, unit='B', unit_scale=True, desc=' INFO: Downloading ') for chunk in response.iter_content(chunk_size): if chunk: file.write(chunk) pbar.update(chunk_size) pbar.close() # update file size first_byte = filename.stat().st_size zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downloaded again if zip_test is not None: if filename.exists(): filename.unlink() raise DownloadError( f'{filename.name} did not pass the zip test. ' f'Re-downloading the full scene.' ) else: logger.info(f'{filename.name} passed the zip test.') with open(filename.with_suffix('.downloaded'), 'w+') as file: file.write('successfully downloaded \n')