def get_sentinel(pts, output_dir, producttype, start=datetime.date(2019, 1, 1), end=datetime.date(2020, 1, 1), **kwargs): """"Query sentinel images of type producttype intersecting pts and randomly select one between start and end""" config = configparser.ConfigParser() config.read("sentinel_scihub.ini") api = SentinelAPI(config["login"]["username"], config["login"]["password"]) for pt in pts: products = api.query(area=f"{pt[0]}, {pt[1]}", date=(start, end), producttype=producttype, **kwargs) if len(products) == 0: print("No products found, skipping") return False else: product_id = random.choice(list(products.keys())) print(product_id) try: api.download(product_id, directory_path=output_dir) return True except SentinelAPILTAError: print("product offline, skipping") return False
def test_scihub_unresponsive(): timeout_connect = 6 timeout_read = 6.6 timeout = (timeout_connect, timeout_read) api = SentinelAPI("mock_user", "mock_password", timeout=timeout) with requests_mock.mock() as rqst: rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ConnectTimeout) with pytest.raises(requests.exceptions.ConnectTimeout): api.query(**_small_query) with pytest.raises(requests.exceptions.ConnectTimeout): api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ConnectTimeout): api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ConnectTimeout): api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b']) with requests_mock.mock() as rqst: rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ReadTimeout) with pytest.raises(requests.exceptions.ReadTimeout): api.query(**_small_query) with pytest.raises(requests.exceptions.ReadTimeout): api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ReadTimeout): api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ReadTimeout): api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
def download_extract_s2(scene_uuid, down_dir, original_scene_dir): """ Download a single S2 scene from ESA via sentinelsat based upon uuid. """ # if unzipped .SAFE file doesn't exist then we must do something if not os.path.exists(original_scene_dir): # if downloaded .zip file doesn't exist then download it if not os.path.exists(original_scene_dir.replace('.SAFE/','.zip')): print ( 'Downloading ESA scene zip: {}'.format(os.path.basename(original_scene_dir)) ) esa_api = SentinelAPI('tmj21','Welcome12!') esa_api.download(scene_uuid, down_dir, checksum=True) # extract downloaded .zip file print ( 'Extracting ESA scene: {}'.format(original_scene_dir) ) zip_ref = zipfile.ZipFile(original_scene_dir.replace('.SAFE/','.zip'), 'r') zip_ref.extractall(os.path.dirname(down_dir)) zip_ref.close() else: print ( 'ESA scene already extracted: {}'.format(original_scene_dir) ) # remove zipped scene but onliy if unzipped if os.path.exists(original_scene_dir) & os.path.exists(original_scene_dir.replace('.SAFE/','.zip')): print ( 'Deleting ESA scene zip: {}'.format(original_scene_dir.replace('.SAFE/','.zip')) ) os.remove(original_scene_dir.replace('.SAFE/','.zip'))
def download_image(self): """ Returns ------- image_path : Directory Filepath to downloaded image after unzipping date : String Date of downloaded image time : String Zulu time of downloaded image """ api = SentinelAPI(self.username, self.password, 'https://scihub.copernicus.eu/dhus') products = self.get_image_products() key = products.index[0] title = products['title'][0] date, time = str(products['endposition'][0]).split(' ') os.chdir(self.image_dir) print('\n >>> DOWNLOADING IMAGE') api.download(key) with ZipFile(title + '.zip', 'r') as zipObject: filenames = zipObject.namelist() for file in filenames: if file.endswith('TCI_10m.jp2'): image_path = zipObject.extract(file) os.remove(title + '.zip') return image_path, date, time
def down_imgs(inTbl, imgIDcol, outFolder=None): """ Download Images in Table """ import os from sentinelsat import SentinelAPI from glass.g.rd.shp import shp_to_obj from glass.cons.sentinel import con_datahub of = outFolder if outFolder else os.path.dirname(inTbl) # Tbl to df df_img = shp_to_obj(inTbl) # API Instance user, password = con_datahub() api = SentinelAPI(user, password, URL_COPERNICUS) # Download Images for idx, row in df_img.iterrows(): # Check if file already exists outFile = os.path.join(outFolder, row.identifier + '.zip') if os.path.exists(outFile): print('IMG already exists') continue else: api.download(row[imgIDcol], directory_path=outFolder)
def dowload_img(list_index, dst_folder): # connect to the API api = SentinelAPI(data_hub['user'], data_hub['password'], 'https://scihub.copernicus.eu/dhus') for i in list_index: title = get_title(conn_string, schema='metadado_img', table='metadado_sentinel', uuid=i) file_already_download = is_file_in_folder(folder=path_home / FOLDER_NAME, file_name=title, file_extention='.zip') if file_already_download: insert_date_hour_db(conn_string=conn_string, schema='metadado_img', table='metadado_sentinel', column='date_download_img', uuid=i) elif TO_DOWNLOAD: api.download(i, directory_path=dst_folder) insert_date_hour_db(conn_string=conn_string, schema='metadado_img', table='metadado_sentinel', column='date_download_img', uuid=i)
def satdownload_zip(product_id, directory='./', api=None): if api is None: api = SentinelAPI(USERNAME, PASSWORD, 'https://scihub.copernicus.eu/dhus') try: print('Downloading {}...'.format(product_id)) api.download(product_id, directory) except Exception as inst: print('Wrong product id.')
def get_data(name, AREA, start_date='20190101', end_date='20190128', platformname='Sentinel-2', processinglevel='Level-2A'): #apiのクエリに次のが必要。中身はPOLYGON((..))っていうstr型 footprint_geojson = geojson_to_wkt( read_geojson('./location/' + str(name) + '.geojson')) # use sentinelAPI user = '' password = '' api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') products = api.query( footprint_geojson, date=(start_date, end_date), #取得希望期間の入力 platformname=platformname, processinglevel=processinglevel, ##2016年はL1C cloudcoverpercentage=(0, 100)) #被雲率(0%〜100%) print("この期間の画像の数は" + str(len(products)) + "枚です") #この後で雲の被覆率ごとにデータを並べ替えて、1個ダウンロードする。 products_gdf = api.to_geodataframe(products) products_gdf_sorted = products_gdf.sort_values(['cloudcoverpercentage'], ascending=[True]).head() #ファイルがまだなければデータダウンロード。作業ディレクトリにzipファイルがダウンロードされる for i in range(3): #3回までチャレンジ uuid = products_gdf_sorted.iloc[i]["uuid"] product_title = products_gdf_sorted.iloc[i][ "title"] #S2A_MSIL2A_20190101T01405... みたいな product_date = products_gdf_sorted.iloc[i]["summary"].split( ',')[0].split()[1][:10] print("この期間で1番被覆率が低いのは" + product_date + "日") if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True: print("新規にデータをダウンロードします") try: api.download(uuid) except: print("ダウンロード不可") else: break else: break if os.path.isfile("./data_geo/" + str(product_title) + '.zip') != True: #ダウンロードしたzipファイルを解凍 #str(product_title) + '.SAFE っていうフォルダが生成される file_name = str(product_title) + '.zip' with zipfile.ZipFile(file_name) as zf: zf.extractall() shutil.move(str(product_title) + '.zip', './data_geo/') shutil.move(str(product_title) + '.SAFE', './data_geo/') return product_title
def download(username, password, scene_id, directory_path, url=None): """A function for downloading scenes from Scihub with an API""" # connect to the API if url is None: api = SentinelAPI(username, password) else: api = SentinelAPI(username, password, api_url=url) api.download(scene_id, directory_path, True)
def _down_btn_clicked(self): username = self.entry_username.get() password = self.entry_password.get() product_id = self.entry_product_id.get() #print(username, password,product_id) api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus') print("Downloading started...") api.download(product_id)
def download_sentinel(current_date, past_date, json, ndvi_tiles, df_predios, ndvi_dir, rgb_dir, ngb_dir, sentinel_dir): # Variables datos geojson_file = 'Tiles-chile.geojson' cloud_directory = '' #Storage access client = storage.Client.from_service_account_json( 'ADL-forestal-segmentation-7dc429779824.json') bucket = client.get_bucket('ranger-app') # Sentinel API api = SentinelAPI('matias-arauco', 'arauco2019', 'https://scihub.copernicus.eu/apihub/') footprint = geojson_to_wkt(json) products = api.query(footprint, date=(past_date, current_date), platformname='Sentinel-2', cloudcoverpercentage=(0, 30)) products_df = api.to_dataframe(products) products_df_sorted = products_df.loc[products_df['tileid'].isnull()] products_df_sorted = products_df_sorted.sort_values( ['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]) print(products_df_sorted) index = products_df_sorted.index ite = 0 for i in index: file_name = products_df_sorted['title'][ite][:] year, month, day = products_df_sorted['title'][ite][ 11:15], products_df_sorted['title'][ite][ 15:17], products_df_sorted['title'][ite][17:19] tile = products_df_sorted['title'][ite][39:44] date = year + '-' + month + '-' + day print('Descargando el Tile: ', tile, ', con fecha: ', date) api.download(i, directory_path=sentinel_dir) upload_file(bucket, sentinel_dir, file_name + '.zip') ndvi_clipping(sentinel_dir, file_name, tile, date, ndvi_tiles, df_predios, ndvi_dir, rgb_dir, ngb_dir, bucket) os.remove(sentinel_dir + file_name + '.zip') ite = ite + 1
def download(self, scene_id: str, output: str, **kwargs) -> str: """Try to download data from Copernicus. Raises: DownloadError when scene not found. DataOfflineError when scene is not available/offline. """ meta = self.api.query(filename=f'{scene_id}*') if len(meta) < 0: raise DownloadError(f'Scene id {scene_id} not found.') api = self.api # When parallel support set, get an available client from Redis if self.parallel: client = self.clients.get_user() api = SentinelAPI(client.username, client.password, show_progressbars=self.progress) uuid = list(meta)[0] entry = api.download(uuid, output) if not entry['Online']: raise DataOfflineError(scene_id) return entry['path']
def test_scihub_unresponsive(): api = SentinelAPI("mock_user", "mock_password") with requests_mock.mock() as rqst: rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ConnectTimeout) with pytest.raises(requests.exceptions.Timeout) as excinfo: api.query(**_small_query) with pytest.raises(requests.exceptions.Timeout) as excinfo: api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.Timeout) as excinfo: api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.Timeout) as excinfo: api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
def download_from_scihub(product_uuid, out_folder, user, passwd): """ Downloads and unzips product_uuid from scihub Parameters ---------- product_uuid The product UUID (4dfB4-432df....) out_folder The folder to save the .SAFE file to user Scihub username passwd Scihub password Notes ----- If interrupted mid-download, there will be a .incomplete file in the download folder. You might need to remove this for further processing. """ api = SentinelAPI(user, passwd) log.info("Downloading {} from scihub".format(product_uuid)) prod = api.download(product_uuid, out_folder) if not prod: log.error("{} failed to download".format(product_uuid)) zip_path = os.path.join(out_folder, prod['title'] + ".zip") log.info("Unzipping {} to {}".format(zip_path, out_folder)) zip_ref = zipfile.ZipFile(zip_path, 'r') zip_ref.extractall(out_folder) zip_ref.close() log.info("Removing {}".format(zip_path)) os.remove(zip_path)
def test_SentinelAPI_wrong_credentials(): api = SentinelAPI("wrong_user", "wrong_password") with pytest.raises(SentinelAPIError) as excinfo: api.query(**_small_query) assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b']) assert excinfo.value.response.status_code == 401
class sentinelWrapper: def __init__(self): logger.info("connect to sentinel API") # connection to API for search queries and download requests self.api = SentinelAPI(config.copernicusUser, config.copernicusPW, config.copernicusURL) logger.info("sentinel API connected") def getSentinelProducts(self, lat, lon, dateFrom, dateTo, platform, **kwargs): logger.info("start sentinel query") # convert geolocation coordinates to wkt format footprint = geojson_to_wkt(Point((lon, lat))) # prepare parameter for cloud coverage if "cloudcoverpercentage" in kwargs: kwargs["cloudcoverpercentage"] = (0, kwargs["cloudcoverpercentage"]) # search query result = self.api.query(footprint, date=(dateFrom, dateTo), platformname=platform, **kwargs) logger.info("sentinel query complete") return result # download multiple sentinel products (list of product IDs) def downloadSentinelProducts(self, products): logger.info("start downloading sentinel product list") self.api.download_all(products, config.bigTilesDir) logger.info("download complete") # download sentinel product with certain product ID def downloadSentinelProduct(self, productID): logger.info("start downloading sentinel product") self.api.download(productID, config.bigTilesDir) logger.info("download complete")
def get_sentinel_images(reef, start_date, end_date, num_images, user, password): """ Method to download Sentinel-2 images using Sentinel API Params - 1. reef (str) - Coral reef object 2. start_date (str) - starting date of sentinel images 3. end_date (str) - end date of sentinel images 4. num_images (int) - number of sentinel-2 images to download 5. user (str) - username on scihub.copernicus.eu 6. password (str) - password on scihub.copernicus.eu """ #login into api api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') #load in geojson of reef reef_path = reef.get_path() reef_gjson_fp = os.path.join(reef_path, reef.get_reef_name() + '.geojson') reef_footprint = geojson_to_wkt(read_geojson(reef_gjson_fp)) #query sentinel sat api products = api.query(reef_footprint,date = (start_date, end_date),platformname = 'Sentinel-2'\ ,area_relation = 'Intersects',processinglevel = 'Level-2A',\ order_by = 'cloudcoverpercentage') #creating folder for saving sentinel images sentinel_path = os.path.join(reef_path, 'SAFE files') if not os.path.exists(sentinel_path): os.makedirs(sentinel_path) #downloading num_images for i, x in enumerate(products.items()): k, v = x[0], x[1] if i < num_images: api.download(k, directory_path=sentinel_path) #unzipping files for file in os.listdir(sentinel_path): if file.endswith('.zip'): file_path = os.path.join(sentinel_path, file) out_path = os.path.join(sentinel_path, file.split('.')[0]) if os.path.exists(file_path) and not os.path.exists(out_path): with zipfile.ZipFile(file_path, "r") as zip_ref: zip_ref.extractall(sentinel_path) os.remove(file_path)
def test_SentinelAPI_wrong_credentials(small_query): api = SentinelAPI("wrong_user", "wrong_password") @contextmanager def assert_exception(): with pytest.raises(UnauthorizedError) as excinfo: yield assert excinfo.value.response.status_code == 401 assert "Invalid user name or password" in excinfo.value.msg with assert_exception(): api.query(**small_query) with assert_exception(): api.get_product_odata("8df46c9e-a20c-43db-a19a-4240c2ed3b8b") with assert_exception(): api.download("8df46c9e-a20c-43db-a19a-4240c2ed3b8b") with assert_exception(): api.download_all(["8df46c9e-a20c-43db-a19a-4240c2ed3b8b"])
def test_download(tmpdir): api = SentinelAPI(**_api_auth) uuid = "1f62a176-c980-41dc-b3a1-c735d660c910" filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E" expected_path = tmpdir.join(filename + ".zip") # Download normally product_info = api.download(uuid, str(tmpdir), checksum=True) assert expected_path.samefile(product_info["path"]) assert product_info["title"] == filename assert product_info["size"] == expected_path.size() hash = expected_path.computehash() modification_time = expected_path.mtime() expected_product_info = product_info del expected_product_info['path'] # File exists, test with checksum # Expect no modification product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.mtime() == modification_time del product_info['path'] assert product_info == expected_product_info # File exists, test without checksum # Expect no modification product_info = api.download(uuid, str(tmpdir), check_existing=False) assert expected_path.mtime() == modification_time del product_info['path'] assert product_info == expected_product_info # Create invalid file, expect re-download with expected_path.open("wb") as f: f.seek(expected_product_info["size"] - 1) f.write(b'\0') assert expected_path.computehash("md5") != hash product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.computehash("md5") == hash del product_info['path'] assert product_info == expected_product_info # Test continue with expected_path.open("rb") as f: content = f.read() with expected_path.open("wb") as f: f.write(content[:100]) assert expected_path.computehash("md5") != hash product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.computehash("md5") == hash del product_info['path'] assert product_info == expected_product_info # Test MD5 check with expected_path.open("wb") as f: f.write(b'abcd' * 100) assert expected_path.computehash("md5") != hash with pytest.raises(InvalidChecksumError): api.download(uuid, str(tmpdir), check_existing=True, checksum=True)
def download_extract_s2_esa(scene_uuid, down_dir, original_scene_dir): """ Download a single S2 scene from ESA via sentinelsat based upon uuid. Assumes esa hub creds stored as env variables. :param scene_uuid: S2 download uuid from sentinelsat query :param down_dir: directory in which to create a downloaded product dir :param original_scene_dir: :return: """ # if unzipped .SAFE file doesn't exist then we must do something if not os.path.exists(original_scene_dir): # if downloaded .zip file doesn't exist then download it if not os.path.exists(original_scene_dir.replace('.SAFE/', '.zip')): logging.info('Downloading ESA scene zip: {}'.format( os.path.basename(original_scene_dir))) copernicus_username = os.getenv("COPERNICUS_USERNAME") copernicus_pwd = os.getenv("COPERNICUS_PWD") logging.debug(f"ESA username: {copernicus_username}") esa_api = SentinelAPI(copernicus_username, copernicus_pwd) esa_api.download(scene_uuid, down_dir, checksum=True) # extract downloaded .zip file logging.info('Extracting ESA scene: {}'.format(original_scene_dir)) zip_ref = zipfile.ZipFile(original_scene_dir.replace('.SAFE/', '.zip'), 'r') zip_ref.extractall(os.path.dirname(down_dir)) zip_ref.close() else: logging.warning( 'ESA scene already extracted: {}'.format(original_scene_dir)) # remove zipped scene but onliy if unzipped if os.path.exists(original_scene_dir) & os.path.exists( original_scene_dir.replace('.SAFE/', '.zip')): logging.info('Deleting ESA scene zip: {}'.format( original_scene_dir.replace('.SAFE/', '.zip'))) os.remove(original_scene_dir.replace('.SAFE/', '.zip'))
def test_SentinelAPI_wrong_credentials(): api = SentinelAPI( "wrong_user", "wrong_password" ) with pytest.raises(SentinelAPIError) as excinfo: api.query(**_small_query) assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') assert excinfo.value.response.status_code == 401 with pytest.raises(SentinelAPIError) as excinfo: api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b']) assert excinfo.value.response.status_code == 401
class SentinelDownloader: user = attr.ib(type=User) area = attr.ib(type=str) date = attr.ib(type=datetime.datetime) def __attrs_post_init__(self): self.month_range = relativedelta(months=1) self.api = SentinelAPI(self.user.name, self.user.password, 'https://scihub.copernicus.eu/dhus') def download_items(self): for scene in self.scenes[:1]: path = os.path.join(PATH_DATA_IN, scene) os.mkdir(path) self.api.download(scene, path) @property def scenes(self): end_positions = [[ abs((self.items_metadata[key]["endposition"] - self.date)).total_seconds(), key ] for key in self.items_metadata] sorted_dates = sorted(end_positions, key=lambda x: x[0]) small_difference = sorted_dates[0][0] final_dates = [ el[1] for el in sorted_dates if el[0] == small_difference ] return final_dates @lazy_property def items_metadata(self): return self.api.query(area=self.area, date=(self.date - self.month_range, self.date + self.month_range), platformname='Sentinel-2', cloudcoverpercentage="[0 TO 20]")
def test_scihub_unresponsive(small_query): timeout_connect = 6 timeout_read = 6.6 timeout = (timeout_connect, timeout_read) api = SentinelAPI("mock_user", "mock_password", timeout=timeout) with requests_mock.mock() as rqst: rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ConnectTimeout) with pytest.raises(requests.exceptions.ConnectTimeout): api.query(**small_query) with pytest.raises(requests.exceptions.ConnectTimeout): api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ConnectTimeout): api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ConnectTimeout): api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b']) with requests_mock.mock() as rqst: rqst.request(requests_mock.ANY, requests_mock.ANY, exc=requests.exceptions.ReadTimeout) with pytest.raises(requests.exceptions.ReadTimeout): api.query(**small_query) with pytest.raises(requests.exceptions.ReadTimeout): api.get_product_odata('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ReadTimeout): api.download('8df46c9e-a20c-43db-a19a-4240c2ed3b8b') with pytest.raises(requests.exceptions.ReadTimeout): api.download_all(['8df46c9e-a20c-43db-a19a-4240c2ed3b8b'])
class Downloader: def __init__(self, str_username, str_password, str_link): self.api = SentinelAPI(str_username, str_password, str_link) self.products = None def search_polygon(self, footprint: object, str_date_start: str, str_date_end: str, str_platform_name: str, percentage: object): print('searching') self.products = self.api.query(footprint, date=(str_date_start, str_date_end), platformname=str_platform_name, cloudcoverpercentage=(percentage[0], percentage[1])) size = self.api.get_products_size(self.products) print(f'found {size}GiB of data') # print(self.products) def download_zip(self, path): self.api.download_all(self.products, path, max_attempt, True) def download_products(self, path, download_file): if download_file: self.download_zip(path) print('downloaded') df_products = self.api.to_dataframe(self.products) return df_products def download_geoproduct(self, path, download_file): if download_file: self.download_zip(path) # print('download Geos') gdf_products = self.api.to_geodataframe(self.products) return gdf_products def download_json(self): return self.api.to_geojson(self.products) def download_one(self, key, path): self.api.download(key, path, True)
def download(product: str, api: SentinelAPI, udate: str, lat: float, lng: float, force: str = 'false'): """Download Sentinel-2 tile user selected Parameters ---------- product : str Product key of selceted tile api : SentinelAPI API interface to Sentinel-2 data hub udate : str Selected date in format YYYY.MM.DD lat : float Target latitude lng : float Target longitude force : str, optional Force download by removing existing downloaded zip files, by default 'false' Returns ------- str Path to downloaded zip of tile """ logger.info('DOWNLOADING SENTINEL-2 DATA') output_pth = os.path.join(const.MODIS_TERRA, 'sentinel', udate, f'lat{lat}_lng{lng}') if not os.path.exists(output_pth): logger.debug(f'creating directory: {output_pth}') os.makedirs(output_pth) if force == 'true': # if force then remove existing downloads files = glob(os.path.join(output_pth, '*')) for f in files: logger.debug(f"removing: {f}") os.remove(f) try: # download target tile logger.info(f'Downloading {product}') info = api.download(product, directory_path=output_pth) return info except Exception as e: logger.error(e)
def main(): if len(sys.argv) < 2: print(" Usage: python3 get_past_scenes.py [year] [month]") return 1 api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus') logging.info(api.api_url) t0 = datetime(int(sys.argv[1]), int(sys.argv[2]), 1, 0, 0, 0) tf = t0 + timedelta(days=12) # search by polygon, time, and SciHub query keywords footprint = geojson_to_wkt( read_geojson(home['parameters'] + '/extent_' + location['region'] + '.geojson')) products_s1a = api.query(footprint, date=(date(t0.year, t0.month, t0.day), date(tf.year, tf.month, tf.day)), producttype="GRD", platformname='Sentinel-1') unavailable = [] for uuid in products_s1a: product_info = api.get_product_odata(uuid) if any(product_info['title'] in s for s in os.listdir(sarIn)): logging.info('Skipping ' + uuid + '. Already exists in ' + sarIn) continue logging.info('Is ' + uuid + ' online?') logging.info(product_info['Online']) if not product_info['Online']: logging.info('Requesting unavailable uuids') api.download(uuid) unavailable = unavailable + [uuid] else: logging.info('Downloading available uuids') api.download(uuid, directory_path=sarIn) logging.info( 'Sleeping 30 minutes (the API does not allow intensive requests)') time.sleep(30 * 60) while len(unavailable) > 0: for uuid in unavailable: product_info = api.get_product_odata(uuid) if product_info['Online']: logging.info(uuid + ' is available! Downloading:') api.download(uuid, directory_path=sarIn) unavailable.remove(uuid) time.sleep(600) return 0
def download_from_scihub(product_uuid, out_folder, user, passwd): """ Downloads and unzips product_uuid from scihub Parameters ---------- product_uuid : str The product UUID (4dfB4-432df....) out_folder : str The folder to save the .SAFE file to user : str Scihub username passwd : str Scihub password Notes ----- If interrupted mid-download, there will be a .incomplete file in the download folder. You might need to remove this for further processing. """ api = SentinelAPI(user, passwd) api.api_url = "https://apihub.copernicus.eu/apihub/" log.info("Downloading {} from scihub".format(product_uuid)) prod = api.download(product_uuid, out_folder) if not prod: log.error("{} not found. Please check.".format(product_uuid)) if not prod["Online"]: log.info( "{} is being retrieved from long-term archive. Please try again later." .format(product_uuid)) return 1 zip_path = os.path.join(out_folder, prod['title'] + ".zip") log.info("Unzipping {} to {}".format(zip_path, out_folder)) zip_ref = zipfile.ZipFile(zip_path, 'r') zip_ref.extractall(out_folder) zip_ref.close() log.info("Removing {}".format(zip_path)) os.remove(zip_path) return 0
def download_products(products, data_dir, username, password, unzip=False, max_retries=3, verbose=True): """Downloads a set of Sentinel-2 products. Parameters: products : dict A dict with Sentinel-2 products, as returned by SentinelAPI.query() or search_products(); keys must be Sentinel-2 UUIDs and values must be dicts with at least the "identifier" property. data_dir : string The path to the directory where the products should be downloaded username, password : string Copernicus Open Access Hub username and password Options: unzip : boolean (default: False) Whether to unzip the downloaded products (to the same dir) max_retries : integer (default: 3) The maximum number of retires verbose : boolean (default: True) Whether to report progress to screen Returns: None """ if username in ["", None] or password in ["", None]: print("Error: you must provide your Copernicus credentials; you can set them in download_datasets.py") sys.exit() # Login to Copernicus api = SentinelAPI(username, password) for product_id, product_info in products.items(): tile = product_info["identifier"].split("_")[5][1:] tile_dir_path = os.path.join(data_dir, "T{}".format(tile)) zip_fname = product_info["identifier"] + ".zip" zip_fpath = os.path.join(tile_dir_path, zip_fname) if os.path.isfile(zip_fpath): # Skip download if the zip already exists if verbose: print("Already downloaded: {}".format(product_info["identifier"])) else: # Create tile subdir if not found if not os.path.exists(tile_dir_path): os.makedirs(tile_dir_path) # Download (and check with MD5) product if verbose: print(product_info["identifier"]) tries_left = max_retries while tries_left > 0: try: result = api.download(product_id, tile_dir_path) break except SentinelAPI.InvalidChecksumError: tries_left -= 1 if verbose: if tries_left > 0: print("Bad MD5 checksum! Retrying.") else: print("Bad MD5 checksum! Max retries reached; skipping.") os.remove(zip_fpath) except: tries_left -= 1 if verbose: if tries_left > 0: print("Unknown error! Retrying.") else: print("Unknown error! Skipping.") if verbose: print("Downloaded {}".format(zip_fname)) # Unzip if asked if unzip: unzipped_fname = product_info["identifier"] + ".SAFE" unzipped_path = os.path.join(tile_dir_path, unzipped_fname) if os.path.exists(unzipped_path): if verbose: print("Dataset already unzipped") else: if verbose: print("Unzipping dataset ...") with zipfile.ZipFile(zip_fpath, "r") as zip: zip.extractall(tile_dir_path) if verbose: print("Unzipped {}".format(unzipped_fname))
class SentinelDownloader(object): def __init__(self, user, password, api_url='https://scihub.copernicus.eu/apihub'): self._apiname = api_url self._user = user self._password = password # init logger root = logging.getLogger() root.addHandler(logging.StreamHandler( sys.stderr )) if self._apiname == 'https://scihub.copernicus.eu/apihub': try: from sentinelsat import SentinelAPI except ImportError as e: gs.fatal(_("Module requires sentinelsat library: {}").format(e)) # connect SciHub via API self._api = SentinelAPI(self._user, self._password, api_url=self._apiname ) elif self._apiname == 'USGS_EE': try: import landsatxplore.api from landsatxplore.errors import EarthExplorerError except ImportError as e: gs.fatal(_("Module requires landsatxplore library: {}").format(e)) api_login = False while api_login is False: # avoid login conflict in possible parallel execution try: self._api = landsatxplore.api.API(self._user, self._password) api_login = True except EarthExplorerError as e: time.sleep(1) self._products_df_sorted = None def filter(self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None): args = {} if clouds: args['cloudcoverpercentage'] = (0, int(clouds)) if relativeorbitnumber: args['relativeorbitnumber'] = relativeorbitnumber if producttype.startswith('S2') and int(relativeorbitnumber) > 143: gs.warning("This relative orbit number is out of range") elif int(relativeorbitnumber) > 175: gs.warning(_("This relative orbit number is out of range")) if producttype: args['producttype'] = producttype if producttype.startswith('S2'): args['platformname'] = 'Sentinel-2' else: args['platformname'] = 'Sentinel-1' if not start: start = 'NOW-60DAYS' else: start = start.replace('-', '') if not end: end = 'NOW' else: end = end.replace('-', '') if query: redefined = [value for value in args.keys() if value in query.keys()] if redefined: gs.warning(_("Query overrides already defined options ({})").format( ','.join(redefined) )) args.update(query) gs.verbose(_("Query: area={} area_relation={} date=({}, {}) args={}").format( area, area_relation, start, end, args )) products = self._api.query( area=area, area_relation=area_relation, date=(start, end), **args ) products_df = self._api.to_dataframe(products) if len(products_df) < 1: gs.message(_("No product found")) return # sort and limit to first sorted product if sortby: self._products_df_sorted = products_df.sort_values( sortby, ascending=[asc] * len(sortby) ) else: self._products_df_sorted = products_df if limit: self._products_df_sorted = self._products_df_sorted.head(int(limit)) gs.message(_("{} Sentinel product(s) found").format(len(self._products_df_sorted))) def list(self): if self._products_df_sorted is None: return id_kw = ('uuid', 'entity_id') identifier_kw = ('identifier', 'display_id') cloud_kw = ('cloudcoverpercentage', 'cloud_cover') time_kw = ('beginposition', 'acquisition_date') kw_idx = 1 if self._apiname == 'USGS_EE' else 0 for idx in range(len(self._products_df_sorted[id_kw[kw_idx]])): if cloud_kw[kw_idx] in self._products_df_sorted: ccp = '{0:2.0f}%'.format( float(self._products_df_sorted[cloud_kw[kw_idx]][idx])) else: ccp = 'cloudcover_NA' print_str = '{0} {1}'.format( self._products_df_sorted[id_kw[kw_idx]][idx], self._products_df_sorted[identifier_kw[kw_idx]][idx]) if kw_idx == 1: time_string = self._products_df_sorted[time_kw[kw_idx]][idx] else: time_string = self._products_df_sorted[ time_kw[kw_idx]][idx].strftime("%Y-%m-%dT%H:%M:%SZ") print_str += ' {0} {1}'.format(time_string, ccp) if kw_idx == 0: print_str += ' {0}'.format( self._products_df_sorted['producttype'][idx]) print(print_str) def download(self, output, sleep=False, maxretry=False, datasource='ESA_COAH'): if self._products_df_sorted is None: return create_dir(output) gs.message(_("Downloading data into <{}>...").format(output)) if datasource == 'USGS_EE': from landsatxplore.earthexplorer import EarthExplorer from landsatxplore.errors import EarthExplorerError from zipfile import ZipFile ee_login = False while ee_login is False: # avoid login conflict in possible parallel execution try: ee = EarthExplorer(self._user, self._password) ee_login = True except EarthExplorerError as e: time.sleep(1) for idx in range(len(self._products_df_sorted['entity_id'])): scene = self._products_df_sorted['entity_id'][idx] identifier = self._products_df_sorted['display_id'][idx] zip_file = os.path.join(output, '{}.zip'.format(identifier)) gs.message(_("Downloading {}...").format(identifier)) try: ee.download(identifier=identifier, output_dir=output, timeout=600) except EarthExplorerError as e: gs.fatal(_(e)) ee.logout() # extract .zip to get "usual" .SAFE with ZipFile(zip_file, 'r') as zip: safe_name = zip.namelist()[0].split('/')[0] outpath = os.path.join(output, safe_name) zip.extractall(path=output) gs.message(_("Downloaded to <{}>").format(outpath)) try: os.remove(zip_file) except Exception as e: gs.warning(_("Unable to remove {0}:{1}").format( zip_file, e)) elif datasource == "ESA_COAH": for idx in range(len(self._products_df_sorted['uuid'])): gs.message('{} -> {}.SAFE'.format( self._products_df_sorted['uuid'][idx], os.path.join(output, self._products_df_sorted['identifier'][idx]) )) # download out = self._api.download(self._products_df_sorted['uuid'][idx], output) if sleep: x = 1 online = out['Online'] while not online: # sleep is in minutes so multiply by 60 time.sleep(int(sleep) * 60) out = self._api.download(self._products_df_sorted['uuid'][idx], output) x += 1 if x > maxretry: online = True elif datasource == 'GCS': for scene_id in self._products_df_sorted['identifier']: gs.message(_("Downloading {}...").format(scene_id)) dl_code = download_gcs(scene_id, output) if dl_code == 0: gs.message(_("Downloaded to {}").format( os.path.join(output, '{}.SAFE'.format(scene_id)))) else: # remove incomplete file del_folder = os.path.join(output, '{}.SAFE'.format(scene_id)) try: shutil.rmtree(del_folder) except Exception as e: gs.warning(_("Unable to removed unfinished " "download {}".format(del_folder))) def save_footprints(self, map_name): if self._products_df_sorted is None: return if self._apiname == 'USGS_EE': gs.fatal(_( "USGS Earth Explorer does not support footprint download.")) try: from osgeo import ogr, osr except ImportError as e: gs.fatal(_("Option <footprints> requires GDAL library: {}").format(e)) gs.message(_("Writing footprints into <{}>...").format(map_name)) driver = ogr.GetDriverByName("GPKG") tmp_name = gs.tempfile() + '.gpkg' data_source = driver.CreateDataSource(tmp_name) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) # features can be polygons or multi-polygons layer = data_source.CreateLayer(str(map_name), srs, ogr.wkbMultiPolygon) # attributes attrs = OrderedDict([ ("uuid", ogr.OFTString), ("ingestiondate", ogr.OFTString), ("cloudcoverpercentage", ogr.OFTInteger), ("producttype", ogr.OFTString), ("identifier", ogr.OFTString) ]) # Sentinel-1 data does not have cloudcoverpercentage prod_types = [type for type in self._products_df_sorted["producttype"]] s1_types = ["SLC", "GRD"] if any(type in prod_types for type in s1_types): del attrs["cloudcoverpercentage"] for key in attrs.keys(): field = ogr.FieldDefn(key, attrs[key]) layer.CreateField(field) # features for idx in range(len(self._products_df_sorted['uuid'])): wkt = self._products_df_sorted['footprint'][idx] feature = ogr.Feature(layer.GetLayerDefn()) newgeom = ogr.CreateGeometryFromWkt(wkt) # convert polygons to multi-polygons newgeomtype = ogr.GT_Flatten(newgeom.GetGeometryType()) if newgeomtype == ogr.wkbPolygon: multigeom = ogr.Geometry(ogr.wkbMultiPolygon) multigeom.AddGeometryDirectly(newgeom) feature.SetGeometry(multigeom) else: feature.SetGeometry(newgeom) for key in attrs.keys(): if key == 'ingestiondate': value = self._products_df_sorted[key][idx].strftime("%Y-%m-%dT%H:%M:%SZ") else: value = self._products_df_sorted[key][idx] feature.SetField(key, value) layer.CreateFeature(feature) feature = None data_source = None # coordinates of footprints are in WKT -> fp precision issues # -> snap gs.run_command('v.import', input=tmp_name, output=map_name, layer=map_name, snap=1e-10, quiet=True ) def get_products_from_uuid_usgs(self, uuid_list): scenes = [] for uuid in uuid_list: metadata = self._api.metadata(uuid, 'SENTINEL_2A') scenes.append(metadata) scenes_df = pandas.DataFrame.from_dict(scenes) self._products_df_sorted = scenes_df gs.message(_("{} Sentinel product(s) found").format( len(self._products_df_sorted))) def set_uuid(self, uuid_list): """Set products by uuid. TODO: Find better implementation :param uuid: uuid to download """ if self._apiname == 'USGS_EE': self.get_products_from_uuid_usgs(uuid_list) else: from sentinelsat.sentinel import SentinelAPIError self._products_df_sorted = {'uuid': []} for uuid in uuid_list: try: odata = self._api.get_product_odata(uuid, full=True) except SentinelAPIError as e: gs.error(_("{0}. UUID {1} skipped".format(e, uuid))) continue for k, v in odata.items(): if k == 'id': k = 'uuid' elif k == 'Sensing start': k = 'beginposition' elif k == 'Product type': k = 'producttype' elif k == 'Cloud cover percentage': k = 'cloudcoverpercentage' elif k == 'Identifier': k = 'identifier' elif k == 'Ingestion Date': k = 'ingestiondate' elif k == 'footprint': pass else: continue if k not in self._products_df_sorted: self._products_df_sorted[k] = [] self._products_df_sorted[k].append(v) def filter_USGS(self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None): if area_relation != 'Intersects': gs.fatal(_( "USGS Earth Explorer only supports area_relation" " 'Intersects'")) if relativeorbitnumber: gs.fatal(_( "USGS Earth Explorer does not support 'relativeorbitnumber'" " option.")) if producttype and producttype != 'S2MSI1C': gs.fatal(_( "USGS Earth Explorer only supports producttype S2MSI1C")) if query: if not any(key in query for key in ['identifier', 'filename', 'usgs_identifier']): gs.fatal(_( "USGS Earth Explorer only supports query options" " 'filename', 'identifier' or 'usgs_identifier'.")) if 'usgs_identifier' in query: # get entityId from usgs identifier and directly save results usgs_id = query['usgs_identifier'] check_s2l1c_identifier(usgs_id, source='usgs') # entity_id = self._api.lookup('SENTINEL_2A', [usgs_id], # inverse=True) entity_id = self._api.get_entity_id([usgs_id], 'SENTINEL_2A') self.get_products_from_uuid_usgs(entity_id) return else: if "filename" in query: esa_id = query['filename'].replace('.SAFE', '') else: esa_id = query['identifier'] check_s2l1c_identifier(esa_id, source='esa') esa_prod_id = esa_id.split('_')[-1] utm_tile = esa_id.split('_')[-2] acq_date = esa_id.split('_')[2].split('T')[0] acq_date_string = '{0}-{1}-{2}'.format( acq_date[:4], acq_date[4:6], acq_date[6:]) start_date = end_date = acq_date_string # build the USGS style S2-identifier if utm_tile.startswith('T'): utm_tile_base = utm_tile[1:] bbox = get_bbox_from_S2_UTMtile(utm_tile_base) else: # get coordinate pairs from wkt string str_1 = 'POLYGON((' str_2 = '))' coords = area[area.find(str_1)+len(str_1):area.rfind(str_2)].split(',') # add one space to first pair for consistency coords[0] = ' ' + coords[0] lons = [float(pair.split(' ')[1]) for pair in coords] lats = [float(pair.split(' ')[2]) for pair in coords] bbox = (min(lons), min(lats), max(lons), max(lats)) start_date = start end_date = end usgs_args = { 'dataset': 'SENTINEL_2A', 'bbox': bbox, 'start_date': start_date, 'end_date': end_date } if clouds: usgs_args['max_cloud_cover'] = clouds if limit: usgs_args['max_results'] = limit scenes = self._api.search(**usgs_args) self._api.logout() if query: # check if the UTM-Tile is correct, remove otherwise for scene in scenes: if scene['display_id'].split('_')[1] != utm_tile: scenes.remove(scene) # remove redundant scene if len(scenes) == 2: for scene in scenes: prod_id = scene['display_id'].split('_')[-1] if prod_id != esa_prod_id: scenes.remove(scene) if len(scenes) < 1: gs.message(_("No product found")) return scenes_df = pandas.DataFrame.from_dict(scenes) if sortby: # replace sortby keywords with USGS keywords for idx, keyword in enumerate(sortby): if keyword == 'cloudcoverpercentage': sortby[idx] = 'cloud_cover' # turn cloudcover to float to make it sortable scenes_df['cloud_cover'] = pandas.to_numeric( scenes_df['cloud_cover']) elif keyword == 'ingestiondate': sortby[idx] = 'acquisition_date' # what does sorting by footprint mean elif keyword == 'footprint': sortby[idx] = 'display_id' self._products_df_sorted = scenes_df.sort_values( sortby, ascending=[asc] * len(sortby), ignore_index=True ) else: self._products_df_sorted = scenes_df gs.message(_("{} Sentinel product(s) found").format( len(self._products_df_sorted)))
def test_download(tmpdir): api = SentinelAPI(**_api_auth) uuid = "1f62a176-c980-41dc-b3a1-c735d660c910" filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E" expected_path = tmpdir.join(filename + ".zip") tempfile_path = tmpdir.join(filename + ".zip.incomplete") # Download normally product_info = api.download(uuid, str(tmpdir), checksum=True) assert expected_path.samefile(product_info["path"]) assert not tempfile_path.check(exists=1) assert product_info["title"] == filename assert product_info["size"] == expected_path.size() assert product_info["downloaded_bytes"] == expected_path.size() hash = expected_path.computehash("md5") modification_time = expected_path.mtime() expected_product_info = product_info # File exists, expect nothing to happen product_info = api.download(uuid, str(tmpdir)) assert not tempfile_path.check(exists=1) assert expected_path.mtime() == modification_time expected_product_info["downloaded_bytes"] = 0 assert product_info == expected_product_info # Create invalid but full-sized tempfile, expect re-download expected_path.move(tempfile_path) with tempfile_path.open("wb") as f: f.seek(expected_product_info["size"] - 1) f.write(b'\0') assert tempfile_path.computehash("md5") != hash product_info = api.download(uuid, str(tmpdir)) assert expected_path.check(exists=1, file=1) assert expected_path.computehash("md5") == hash expected_product_info["downloaded_bytes"] = expected_product_info["size"] assert product_info == expected_product_info # Create invalid tempfile, without checksum check # Expect continued download and no exception dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa' with tempfile_path.open("wb") as f: f.write(dummy_content) expected_path.remove() product_info = api.download(uuid, str(tmpdir), checksum=False) assert not tempfile_path.check(exists=1) assert expected_path.check(exists=1, file=1) assert expected_path.computehash("md5") != hash expected_product_info["downloaded_bytes"] = expected_product_info["size"] - len(dummy_content) assert product_info == expected_product_info # Create invalid tempfile, with checksum check # Expect continued download and exception raised dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa' with tempfile_path.open("wb") as f: f.write(dummy_content) expected_path.remove() with pytest.raises(InvalidChecksumError): api.download(uuid, str(tmpdir), checksum=True) assert not tempfile_path.check(exists=1) assert not expected_path.check(exists=1, file=1) tmpdir.remove()
def test_check_existing(tmpdir): api = SentinelAPI(**_api_auth) ids = [ "5618ce1b-923b-4df2-81d9-50b53e5aded9", "d8340134-878f-4891-ba4f-4df54f1e3ab4", "1f62a176-c980-41dc-b3a1-c735d660c910" ] names = [ "S1A_WV_OCN__2SSV_20150526T081641_20150526T082418_006090_007E3E_104C", "S1A_WV_OCN__2SSV_20150526T211029_20150526T211737_006097_007E78_134A", "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E" ] paths = [tmpdir.join(fn + ".zip") for fn in names] path_strings = list(map(str, paths)) # Init files used for testing api.download(ids[0], str(tmpdir)) # File #1: complete and correct assert paths[0].check(exists=1, file=1) # File #2: complete but incorrect with paths[1].open("wb") as f: size = 130102 f.seek(size - 1) f.write(b'\0') # File #3: incomplete dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa' with paths[2].open("wb") as f: f.write(dummy_content) assert paths[2].check(exists=1, file=1) # Test expected = {str(paths[1]), str(paths[2])} result = api.check_files(ids=ids, directory=str(tmpdir)) assert set(result) == expected assert result[paths[1]][0]['id'] == ids[1] assert result[paths[2]][0]['id'] == ids[2] assert paths[0].check(exists=1, file=1) assert paths[1].check(exists=1, file=1) assert paths[2].check(exists=1, file=1) result = api.check_files(paths=path_strings) assert set(result) == expected assert result[paths[1]][0]['id'] == ids[1] assert result[paths[2]][0]['id'] == ids[2] assert paths[0].check(exists=1, file=1) assert paths[1].check(exists=1, file=1) assert paths[2].check(exists=1, file=1) result = api.check_files(paths=path_strings, delete=True) assert set(result) == expected assert result[paths[1]][0]['id'] == ids[1] assert result[paths[2]][0]['id'] == ids[2] assert paths[0].check(exists=1, file=1) assert not paths[1].check(exists=1, file=1) assert not paths[2].check(exists=1, file=1) missing_file = str( tmpdir.join( "S1A_EW_GRDH_1SDH_20141003T003840_20141003T003920_002658_002F54_4DD1.zip" )) result = api.check_files(paths=[missing_file]) assert set(result) == {missing_file} assert result[missing_file][0]['id'] with pytest.raises(ValueError): api.check_files(ids=ids) with pytest.raises(ValueError): api.check_files() tmpdir.remove()
def test_download_invalid_id(): api = SentinelAPI(**_api_auth) uuid = "1f62a176-c980-41dc-xxxx-c735d660c910" with pytest.raises(SentinelAPIError) as excinfo: api.download(uuid) assert 'Invalid key' in excinfo.value.msg
def test_download(tmpdir): api = SentinelAPI(**_api_auth) uuid = "1f62a176-c980-41dc-b3a1-c735d660c910" filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E" expected_path = tmpdir.join(filename + ".zip") tempfile_path = tmpdir.join(filename + ".zip.incomplete") # Download normally product_info = api.download(uuid, str(tmpdir), checksum=True) assert expected_path.samefile(product_info["path"]) assert not tempfile_path.check(exists=1) assert product_info["title"] == filename assert product_info["size"] == expected_path.size() assert product_info["downloaded_bytes"] == expected_path.size() hash = expected_path.computehash("md5") modification_time = expected_path.mtime() expected_product_info = product_info # File exists, expect nothing to happen product_info = api.download(uuid, str(tmpdir)) assert not tempfile_path.check(exists=1) assert expected_path.mtime() == modification_time expected_product_info["downloaded_bytes"] = 0 assert product_info == expected_product_info # Create invalid but full-sized tempfile, expect re-download expected_path.move(tempfile_path) with tempfile_path.open("wb") as f: f.seek(expected_product_info["size"] - 1) f.write(b'\0') assert tempfile_path.computehash("md5") != hash product_info = api.download(uuid, str(tmpdir)) assert expected_path.check(exists=1, file=1) assert expected_path.computehash("md5") == hash expected_product_info["downloaded_bytes"] = expected_product_info["size"] assert product_info == expected_product_info # Create invalid tempfile, without checksum check # Expect continued download and no exception dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa' with tempfile_path.open("wb") as f: f.write(dummy_content) expected_path.remove() product_info = api.download(uuid, str(tmpdir), checksum=False) assert not tempfile_path.check(exists=1) assert expected_path.check(exists=1, file=1) assert expected_path.computehash("md5") != hash expected_product_info["downloaded_bytes"] = expected_product_info[ "size"] - len(dummy_content) assert product_info == expected_product_info # Create invalid tempfile, with checksum check # Expect continued download and exception raised dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa' with tempfile_path.open("wb") as f: f.write(dummy_content) expected_path.remove() with pytest.raises(InvalidChecksumError): api.download(uuid, str(tmpdir), checksum=True) assert not tempfile_path.check(exists=1) assert not expected_path.check(exists=1, file=1) tmpdir.remove()
def test_check_existing(tmpdir): api = SentinelAPI(**_api_auth) ids = [ "5618ce1b-923b-4df2-81d9-50b53e5aded9", "d8340134-878f-4891-ba4f-4df54f1e3ab4", "1f62a176-c980-41dc-b3a1-c735d660c910" ] names = ["S1A_WV_OCN__2SSV_20150526T081641_20150526T082418_006090_007E3E_104C", "S1A_WV_OCN__2SSV_20150526T211029_20150526T211737_006097_007E78_134A", "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E"] paths = [tmpdir.join(fn + ".zip") for fn in names] path_strings = list(map(str, paths)) # Init files used for testing api.download(ids[0], str(tmpdir)) # File #1: complete and correct assert paths[0].check(exists=1, file=1) # File #2: complete but incorrect with paths[1].open("wb") as f: size = 130102 f.seek(size - 1) f.write(b'\0') # File #3: incomplete dummy_content = b'aaaaaaaaaaaaaaaaaaaaaaaaa' with paths[2].open("wb") as f: f.write(dummy_content) assert paths[2].check(exists=1, file=1) # Test expected = {str(paths[1]), str(paths[2])} result = api.check_files(ids=ids, directory=str(tmpdir)) assert set(result) == expected assert result[paths[1]][0]['id'] == ids[1] assert result[paths[2]][0]['id'] == ids[2] assert paths[0].check(exists=1, file=1) assert paths[1].check(exists=1, file=1) assert paths[2].check(exists=1, file=1) result = api.check_files(paths=path_strings) assert set(result) == expected assert result[paths[1]][0]['id'] == ids[1] assert result[paths[2]][0]['id'] == ids[2] assert paths[0].check(exists=1, file=1) assert paths[1].check(exists=1, file=1) assert paths[2].check(exists=1, file=1) result = api.check_files(paths=path_strings, delete=True) assert set(result) == expected assert result[paths[1]][0]['id'] == ids[1] assert result[paths[2]][0]['id'] == ids[2] assert paths[0].check(exists=1, file=1) assert not paths[1].check(exists=1, file=1) assert not paths[2].check(exists=1, file=1) missing_file = str(tmpdir.join( "S1A_EW_GRDH_1SDH_20141003T003840_20141003T003920_002658_002F54_4DD1.zip")) result = api.check_files(paths=[missing_file]) assert set(result) == {missing_file} assert result[missing_file][0]['id'] with pytest.raises(ValueError): api.check_files(ids=ids) with pytest.raises(ValueError): api.check_files() tmpdir.remove()