def downloadProduct(data): index, directory_path, username, password = data api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus') download_data = Downloads.objects.filter(product_id=index).update( start_time=datetime.datetime.now(), status='started') download_data = Downloads.objects.get(product_id=index) # download_data.start_time=datetime.datimetime.now() # download_data.status='started' # download_data.save() logger.info(f'Downloading product with product_id {index} Started') try: api.download(index, directory_path=directory_path, checksum=True) download_data.end_time = datetime.datetime.now() download_data.status = 'finished' download_data.save() except Exception as e: logger.exception( f'Exception occured {e} while downloading product with product_id {index}' ) download_data.end_time = datetime.datetime.now() download_data.status = 'error' download_data.save() except AttributeError as e: logger.exception( f'Exception occured {e} while downloading product with product_id {index}' ) download_data.end_time = datetime.datetime.now() download_data.status = 'error' download_data.save() logger.info(f'Downloading product with product_id {index} Finished')
def download_s1(user, password, dir_raw, dir_nc, start_date, end_date, footprint): api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus/') #footprint = "POLYGON((73 11, 74 11, 74 14, 73 14, 73 11))" products = api.query(footprint, date=(start_date, end_date), producttype='GRD') #print(products) for product in products: productInfo = api.get_product_odata(product) title = productInfo['title'] print(title) file_nc = os.path.join(dir_nc, "%s_VV.nc" % title) file_wkt = os.path.join(os.path.dirname(dir_nc), 'wkt', "%s.wkt" % title) if not os.path.exists(file_wkt): pFootPrint = productInfo['footprint'] file = open(file_wkt, "a") file.write(pFootPrint) file.close() if not os.path.exists(file_nc): api.download(product, dir_raw, checksum=True)
def download_scene(scene): """Download a scene and change its status """ from sentinelsat.sentinel import SentinelAPI from django.conf import settings path = join(settings.MEDIA_ROOT, scene.sat, scene.identifier) check_create_folder(path) try: api = SentinelAPI( settings.SENTINEL_USER, settings.SENTINEL_PASSWORD, settings.SENTINEL_API_URL ) except AttributeError: api = SentinelAPI(settings.SENTINEL_USER, settings.SENTINEL_PASSWORD) try: print('Changing status of scene %s to downloading' % scene.identifier) scene.change_status('downloading') print('Starting download of product %s on path %s' % (scene.product, path)) api.download(scene.product, path) print('Changing status of scene %s to downloaded' % scene.identifier) scene.change_status('downloaded') except Exception as exp: print('Unexpected error: %s' % exp) print('Changing status of scene %s to dl_failed' % scene.identifier) scene.change_status('dl_failed')
def download_s2(user, password, dir_raw, dir_nc, start_date, end_date, footprint, pr_status): api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus/') #footprint = "POLYGON((73 11, 74 11, 74 14, 73 14, 73 11))" #products = api.query(footprint, date=(start_date, end_date), producttype='S2MSI1C') products = api.query(footprint, date=(start_date, end_date), producttype='S2MSI1C',cloudcoverpercentage = (0,20)) #print(products) l = ['S2A_MSIL1C_20180601T051651_N0206_R062_T43PFN_20180601T082308', 'S2A_MSIL1C_20180621T051651_N0206_R062_T43PFN_20180621T081647', 'S2B_MSIL1C_20180613T050649_N0206_R019_T43PFN_20180613T084228', 'S2A_MSIL1C_20180601T051651_N0206_R062_T43PFP_20180601T082308', 'S2A_MSIL1C_20180621T051651_N0206_R062_T43PFP_20180621T081647', 'S2B_MSIL1C_20180613T050649_N0206_R019_T43PFP_20180613T084228', 'S2A_MSIL1C_20180608T050651_N0206_R019_T43PFN_20180608T084904', 'S2A_MSIL1C_20180628T050651_N0206_R019_T43PFN_20180628T081023', 'S2B_MSIL1C_20180616T051649_N0206_R062_T43PFN_20180616T090733', 'S2A_MSIL1C_20180608T050651_N0206_R019_T43PFP_20180608T084904', 'S2A_MSIL1C_20180628T050651_N0206_R019_T43PFP_20180628T081023', 'S2B_MSIL1C_20180616T051649_N0206_R062_T43PFP_20180616T090733', 'S2A_MSIL1C_20180611T051651_N0206_R062_T43PFN_20180611T081245', 'S2B_MSIL1C_20180603T050649_N0206_R019_T43PFN_20180603T084545', 'S2B_MSIL1C_20180623T050649_N0206_R019_T43PFN_20180623T084444', 'S2A_MSIL1C_20180611T051651_N0206_R062_T43PFP_20180611T081245', 'S2B_MSIL1C_20180603T050649_N0206_R019_T43PFP_20180603T084545', 'S2B_MSIL1C_20180623T050649_N0206_R019_T43PFP_20180623T084444', 'S2A_MSIL1C_20180618T050651_N02206_R019_T43PFN_20180618T085607', 'S2B_MSIL1C_20180606T051649_N0206_R062_T43PFN_20180606T104751', 'S2B_MSIL1C_20180626T051649_N0206_R062_T43PFN_20180626T090058', 'S2A_MSIL1C_20180618T050651_N0206_R019_T43PFP_20180618T085607', 'S2B_MSIL1C_20180606T051649_N0206_R062_T43PFP_20180606T104751', 'S2B_MSIL1C_20180626T051649_N0206_R062_T43PFP_20180626T090058'] for product in products: productInfo = api.get_product_odata(product) title = productInfo['title'] if title in l: continue tileNo_time = '%s_%s' % (title.split('_')[5], title.split('_')[2]) try: downloadFlag = not pr_status[tileNo_time] except KeyError: pr_status[tileNo_time] = False downloadFlag =True print "no error" #file_nc = os.path.join(dir_nc, "%s_VV.nc"%os.path.basename(title).split("_")[4]) #file_nc = os.path.join(dir_nc, "%s_VV.nc" % title[17:48]) file_wkt = os.path.join(os.path.dirname(dir_nc), "wkt/%s.wkt" % tileNo_time) if not os.path.exists(file_wkt): pFootPrint = productInfo['footprint'] file = open(file_wkt, "a") file.write(pFootPrint) file.close() if downloadFlag and not title in l: api.download(product, dir_raw, checksum=True) l.append(title) return pr_status
def download(user, password, productid, path, md5, url): """Download a Sentinel Product with your Copernicus Open Access Hub user and password and the id of the product you want to download. """ api = SentinelAPI(user, password, url) try: api.download(productid, path, md5) except SentinelAPIError as e: if 'Invalid key' in e.msg: logger.error('No product with ID \'%s\' exists on server', productid) else: raise
def download(user, password, productid, path, md5, url): """Download a Sentinel Product. It just needs your SciHub user and password and the id of the product you want to download. """ api = SentinelAPI(user, password, url) try: api.download(productid, path, md5) except SentinelAPIError as e: if 'Invalid key' in e.msg: logger.error('No product with ID \'%s\' exists on server', productid) else: raise
def downloadImages_s(self): S1_download_url = 'https://scihub.copernicus.eu/dhus' api = SentinelAPI('chenjinlv', 'cjl19890710', S1_download_url) self.downloadBar_s.setMinimum(0) self.downloadBar_s.setValue(0) self.downloadLabel_s.setText(u"等待下载") idfile = self.savecsv_s.text() outputdir = self.savedownload_s.text() if self.sate_s.currentText() == "s2a": csvf = open(idfile, 'r') csv_count = len(csvf.readlines()) - 1 csvf.close() print(csv_count) self.downloadBar_s.setMaximum(csv_count) with open(idfile, 'r') as f: idcsv = csv.reader(f) header = next(idcsv) for ck, row in enumerate(idcsv): self.downloadBar_s.setValue(ck) url = row[1] PRT_ID = row[0] # filename = url.split('/')[-1] if os.path.exists(outputdir + '/{}'.format(PRT_ID)): shutil.rmtree(outputdir + '/{}'.format(PRT_ID)) if not os.path.exists(outputdir + '/{}.zip'.format(PRT_ID)): dt = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) self.downloadLabel_s.setText( str(ck) + "/" + str(csv_count)) print(dt + "Downloading {} ...".format(PRT_ID)) self.get_image_s(PRT_ID, url, outputdir) elif self.sate_s.currentText() == "s1": csvf = open(idfile, 'r') csv_count = len(csvf.readlines()) - 1 csvf.close() print(csv_count) self.downloadBar_s.setMaximum(csv_count) with open(idfile, 'r') as f: idcsv = csv.reader(f) header = next(idcsv) for ck, row in enumerate(idcsv): self.downloadBar_s.setValue(ck) uuidprd = row[1] filename = row[0] self.downloadLabel_s.setText( str(ck) + "/" + str(csv_count)) api.download(uuidprd, directory_path=outputdir)
def download(product_id, output_dir, esa_sso_username, esa_sso_password, dhus_username, dhus_password): """Download product given its identifier. Parameters ---------- product_id : str Landsat, ERS, Envisat or Sentinel-1 product ID. output_dir : str Output directory. Returns ------- product_dir : str Product directory. """ platform = guess_platform(product_id) # Avoid if product is already downloaded product_path = find_product(product_id, output_dir) if product_path: return product_path if platform == 'Landsat': product = Product(product_id) product.download(output_dir, progressbar=False) elif platform in ('ERS', 'Envisat'): session = log_in(esa_sso_username, esa_sso_password) try: request_download(session, product_id, output_dir, progressbar=False) except FileExistsError: pass log_out(session) else: api = SentinelAPI(dhus_username, dhus_password, show_progressbars=False) meta = api.query(filename=product_id + '*') uuid = list(meta)[0] if _is_online(uuid, dhus_username, dhus_password): api.download(uuid, output_dir) unzip(os.path.join(output_dir, product_id + '.zip')) else: raise requests.exceptions.HTTPError( '503: Product offline. Re-upload requested.') return find_product(product_id, output_dir)
def run_download(product_id, out_dir, username, password): # start the downloading with the data id, output directory, and sentinelsat api #file_object = open(out_dir+'data_product_id',"w") #file_object.write(product_id) #file_object.close() url = 'https://scihub.copernicus.eu/dhus' d_api = SentinelAPI(username, password, url) try: download_info = d_api.download(product_id['uuid'], directory_path=out_dir) except: print('Server connection error') return 0 if os.path.exists(out_dir+download_info['title']+'.zip'): os.mknod(out_dir+download_info['title']+'_data_downloaded.ok') print('data_downloaded') retrievel_code = 1 #os.remove(out_dir+'data_product_id') elif download_info['Online']: # os.mknod(out_dir+"online_not_downloaded.ok") print('online_but_not_downloaded') elif not download_info['Online']: retrievel_code = d_api._trigger_offline_retrieval(download_info['url']) # check https://scihub.copernicus.eu/userguide/LongTermArchive#HTTP_Status_codes if retrievel_code == 202: # os.mknod(out_dir+"retrieval_accepted.ok") print("offline product retrieval accepted") elif retrievel_code == 403: # os.mknod(out_dir+"requests_exceed_quota.ok") print("offline product requests exceed quota") elif retrievel_code == 503: # os.mknod(out_dir+"retrieval_not_accepted.ok") print("offline product retrieval not accepted") return retrievel_code
def run_download_from_pool(product_id, out_dir, username, password): url = 'https://scihub.copernicus.eu/dhus' d_api = SentinelAPI(username, password, url) try: download_info = d_api.download(product_id, directory_path=out_dir) except: print('Server connection error') return 0 if os.path.exists(out_dir+'/'+download_info['title']+'.zip'): os.mknod(out_dir+'/'+download_info['title']+'.ok') print('data_downloaded') #os.remove(out_dir+'data_product_id') elif download_info['Online']: # os.mknod(out_dir+"online_not_downloaded.ok") print('online_but_not_downloaded') elif not download_info['Online']: retrievel_code = d_api._trigger_offline_retrieval(download_info['url']) # check https://scihub.copernicus.eu/userguide/LongTermArchive#HTTP_Status_codes if retrievel_code == 202: # os.mknod(out_dir+"retrieval_accepted.ok") print("offline product retrieval accepted") elif retrievel_code == 403: # os.mknod(out_dir+"requests_exceed_quota.ok") print("offline product requests exceed quota") elif retrievel_code == 503: # os.mknod(out_dir+"retrieval_not_accepted.ok") print("offline product retrieval not accepted") return download_info
def downloadSentinel(user, pw, aoi, start, stop): # For image before November 16th, 2015 curl = pycurl.Curl() curl.setopt(pycurl.CAINFO, certifi.where()) curl.setopt(pycurl.URL, 'https://scihub.copernicus.eu/dhus') curl.perform() # For image before November 16th, 2015 api = SentinelAPI(user, pw, 'https://scihub.copernicus.eu/dhus') AOI = KMLtoGeoJason.kml2geojson(aoi) api.query(get_coordinates(AOI), start, stop, producttype='GRD') # footprint generation of all found images: a=api.get_footprints() name = AOI[:-8]+"_S1footprint.geojson" foot = open(name, "w") foot.write(dumps(a, indent=2) + "\n") foot.close() ## ## with open(name) as f: ## contents = f.read() ## display(contents) # selected image download and unzip: imageId = raw_input("Insert Sentinel-1 image id: ") output_img = 'C:\Users\ithaca\Documents\Magda\Tool_MIE\SENTINEL-1_TOOL\Immagini_grandi' s1 = api.download(imageId, output_img) path = os.path.dirname(s1) with zipfile.ZipFile(s1, "r") as z: z.extractall(path)
def test_download(tmpdir): api = SentinelAPI(**_api_auth) uuid = "1f62a176-c980-41dc-b3a1-c735d660c910" filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E" expected_path = tmpdir.join(filename + ".zip") # Download normally path, product_info = api.download(uuid, str(tmpdir), checksum=True) assert expected_path.samefile(path) assert product_info["id"] == uuid assert product_info["title"] == filename assert product_info["size"] == expected_path.size() hash = expected_path.computehash() modification_time = expected_path.mtime() expected_product_info = product_info # File exists, test with checksum # Expect no modification path, product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.mtime() == modification_time assert product_info == expected_product_info # File exists, test without checksum # Expect no modification path, product_info = api.download(uuid, str(tmpdir), check_existing=False) assert expected_path.mtime() == modification_time assert product_info == expected_product_info # Create invalid file, expect re-download with expected_path.open("wb") as f: f.seek(expected_product_info["size"] - 1) f.write(b'\0') assert expected_path.computehash("md5") != hash path, product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.computehash("md5") == hash assert product_info == expected_product_info # Test continue with expected_path.open("rb") as f: content = f.read() with expected_path.open("wb") as f: f.write(content[:100]) assert expected_path.computehash("md5") != hash path, product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.computehash("md5") == hash assert product_info == expected_product_info # Test MD5 check with expected_path.open("wb") as f: f.write(b'abcd' * 100) assert expected_path.computehash("md5") != hash with pytest.raises(InvalidChecksumError): api.download(uuid, str(tmpdir), check_existing=True, checksum=True)
def test_download(tmpdir): api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD']) uuid = "1f62a176-c980-41dc-b3a1-c735d660c910" filename = "S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E" expected_path = tmpdir.join(filename + ".zip") # Download normally path, product_info = api.download(uuid, str(tmpdir), checksum=True) assert expected_path.samefile(path) assert product_info["id"] == uuid assert product_info["title"] == filename assert product_info["size"] == expected_path.size() hash = expected_path.computehash() modification_time = expected_path.mtime() expected_product_info = product_info # File exists, test with checksum # Expect no modification path, product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.mtime() == modification_time assert product_info == expected_product_info # File exists, test without checksum # Expect no modification path, product_info = api.download(uuid, str(tmpdir), check_existing=False) assert expected_path.mtime() == modification_time assert product_info == expected_product_info # Create invalid file, expect re-download with expected_path.open("wb") as f: f.seek(expected_product_info["size"] - 1) f.write(b'\0') assert expected_path.computehash("md5") != hash path, product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.computehash("md5") == hash assert product_info == expected_product_info # Test continue with expected_path.open("rb") as f: content = f.read() with expected_path.open("wb") as f: f.write(content[:100]) assert expected_path.computehash("md5") != hash path, product_info = api.download(uuid, str(tmpdir), check_existing=True) assert expected_path.computehash("md5") == hash assert product_info == expected_product_info # Test MD5 check with expected_path.open("wb") as f: f.write(b'abcd' * 100) assert expected_path.computehash("md5") != hash with pytest.raises(InvalidChecksumError): api.download(uuid, str(tmpdir), check_existing=True, checksum=True)
class Downloader(): def __init__(self, downloader_path, options): self.downloader_path = downloader_path self.options = options self.hyp3_archive_db = Hyp3_Archive_Sql(self.options.pg_db, self.options.find_granule_sql) self.get_sentinel_api() self.get_download_path() ensure_paths([self.download_path, self.options.final_dir]) def get_download_path(self): self.download_path = os.path.abspath( os.path.join(self.downloader_path, self.options.download_dir)) def get_sentinel_api(self): self.api = SentinelAPI(self.options.user, self.options.password, self.options.esa_host) def download_granule(self, product, granule): log.info( f"Downloading product: {product} corresponding to granule: {granule}" ) if not self.is_product_handled(product=product, granule=granule): self.api.download(product, directory_path=self.download_path) move(os.path.join(self.download_path, f"{granule}.zip"), self.options.final_dir) def is_product_handled(self, product, granule): if not self.hyp3_archive_db.is_granule_in_hyp3(product): if not os.path.exists( os.path.join(self.options.final_dir, f"{granule}*")): return False log.info( f"Product {product} correspond to granule {granule} has already been handled" ) return True
def download(request): if request.method == 'POST': data = json.loads(request.body) footprint = geojson_to_wkt(geojson.loads(data['geoJson'])) username = config.username # ask ITC for the username and password password = config.password api = SentinelAPI(username, password, "https://apihub.copernicus.eu/apihub/" ) # fill with SMARTSeeds user and password tanggal = '[{0} TO {1}]'.format(data['dateFrom'].replace('.000Z', 'Z'), data['dateTo'].replace('.000Z', 'Z')) print(tanggal) products = api.query(footprint, producttype=config.producttype, orbitdirection=config.orbitdirection, platformname='Sentinel-1', date=tanggal) #menyimpan di folder sentineldata dirpath = os.getcwd() + '/sentineldata' for product in products: try: api.download(product, directory_path=dirpath, checksum=True) except: continue for item in os.listdir(dirpath): if item.endswith(".incomplete"): os.remove(os.path.join(dirpath, item)) #fungsi notifikasi email = send_mail( #libary django untuk mengirim email 'Your Download was successful!', #subject email 'Terima kasih sudah menggunakan aplikasi webgis data yang anda unduh sudah masuk kedalam sistem website!', #isi email settings.EMAIL_HOST_USER, #email host pengirim notifikasi [request.user.email], #email penerima notifikasi fail_silently=False, ) return HttpResponse(request.body)
def download_s2(user, password, dir_raw, dir_nc, start_date, end_date, footprint, pr_status): api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus/') #footprint = "POLYGON((73 11, 74 11, 74 14, 73 14, 73 11))" products = api.query(footprint, date=(start_date, end_date), producttype='S2MSI1C') #print(products) for product in products: productInfo = api.get_product_odata(product) title = productInfo['title'] tileNo_time = '%s_%s' % (title.split('_')[5], title.split('_')[2]) try: downloadFlag = not pr_status[tileNo_time] except KeyError: pr_status[tileNo_time] = False downloadFlag = True #file_nc = os.path.join(dir_nc, "%s_VV.nc"%os.path.basename(title).split("_")[4]) #file_nc = os.path.join(dir_nc, "%s_VV.nc" % title[17:48]) file_wkt = os.path.join(os.path.dirname(dir_nc), "wkt/%s.wkt" % tileNo_time) if not os.path.exists(file_wkt): pFootPrint = productInfo['footprint'] file = open(file_wkt, "a") file.write(pFootPrint) file.close() if downloadFlag: api.download(product, dir_raw, checksum=True) return pr_status
class CopernicusAPI(BaseAPI): def __init__(self, user, password): super().__init__() self.__api = SentinelAPI(user=user, password=password) def search(self, start='NOW-7DAYS', end='NOW'): self._logger.info('Searching for new data sets') # loading search extend current_dir = os.path.dirname(__file__) extend_path = os.path.join(current_dir, "nrw.geojson") footprint = geojson_to_wkt(read_geojson(extend_path)) try: return self.__api.query(area=footprint, initial_date=start, end_date=end, platformname='Sentinel-1', producttype='GRD') except SentinelAPIError: self._logger.error('There was an error searching for data sets', exc_info=True) return {} def download(self, product_id): self._logger.info("Start downloading product: {}".format(product_id)) try: product_info = self.__api.download(product_id, checksum=True) self._logger.info( "Product was successfully downloaded to {}".format( product_info['path'])) return product_info['path'] except InvalidChecksumError: self._logger.error( 'The checksum of the download was invalid. Skipping product {}' ).format(product_id) except SentinelAPIError: self._logger.error( 'There was an error trying to download the product', exc_info=True) def remove(self, filename): self._logger.info('Removing product at {}'.format(filename)) try: os.remove(filename) except OSError: pass
def _download_from_hub(product, download_path, user, password, hub_address="https://scihub.copernicus.eu/apihub", overwrite=False): path = os.path.join(download_path, product["identifier"] + ".*") if glob.glob(path) and not overwrite: return glob.glob(path)[0] else: # Connect to the hub and download try: hub = SentinelAPI(user, password, hub_address) p = hub.download(product["uuid"], download_path) except SentinelAPIError as e: print(e) return "" with zipfile.ZipFile(p["path"], "r") as z: z.extractall(download_path) os.remove(p["path"]) return glob.glob(path)[0]
products_df = api.to_dataframe(products) # FINDING SMALLEST FILE smallestFile = None tempSize = 9999999999 for i in range(0, len(products)): if (api.get_product_odata(products_df.index[i])["size"] < tempSize): smallestFile = products_df.index[i] tempSize = api.get_product_odata(products_df.index[i])["size"] # ---------------------------------------------------------------- # SETTING MAX SIZE AND GETTING PRODUCT INFO maxSize = 500000000 # Set the max size for files to download (in bytes) if (tempSize < maxSize): api.download(smallestFile) smallestName = api.get_product_odata(smallestFile)["title"] smallestDate = api.get_product_odata(smallestFile)["date"].strftime("%d-%m-%Y_%H-%M") # ":" cause error in windowsOS and with KML links smallestLink = api.get_product_odata(smallestFile)["url"] print("Downloading " + smallestName + ", Size: " + str(tempSize) + " bytes.") else: print("No file small enough to download") quit() fileName = 'sentinel-image(a)_' + smallestDate cleanFileName = 'sentinel-image(C)_' + smallestDate saveName = 'sentinel_images/' + fileName + '.png' # file name cleanSaveName = 'sentinel_images_clean/' + cleanFileName + '.png' # ----------------------------------------------------------------
# GeoJSON FeatureCollection containing footprints and metadata of the scenes fp = api.get_footprints(products) with open('footprints.json', 'w') as outfile: #json.dump({'numbers':n, 'strings':s, 'x':x, 'y':y}, outfile, indent=4) json.dump(fp, outfile, indent=4) with open('product.json', 'w') as outfile: #json.dump({'numbers':n, 'strings':s, 'x':x, 'y':y}, outfile, indent=4) json.dump(products, outfile, indent=4) #print(fp["features"][0]["properties"]["identifier"]) i = 0 #2017-02-07T10:42:00.746Z datetime_old = format_date("20170101") print datetime_old for entry in fp["features"]: datetime = entry["properties"]["date_beginposition"] if (datetime >= datetime_old): datetime_old = datetime product_id = entry["properties"]["product_id"] print "###########" + datetime print entry["properties"]["identifier"] print entry["properties"]["product_id"] print entry["properties"]["date_beginposition"] api.download(product_id)
def execute(self, context): log.info("###################") log.info("## DHUS Download ##") log.info('API URL: %s', self.dhus_url) log.info('API User: %s', self.dhus_user) #log.info('API Password: %s', self.dhus_pass) log.info('Max Downloads: %s', self.download_max) log.info('Download Directory: %s', self.download_dir) log.info('Input from: %s', self.get_inputs_from) log.info('Product IDs: %s', self.product_ids) log.info("Execute DHUS Download.. ") # create the download directory if it does not exists if not os.path.exists(self.download_dir): log.info("Creating directory for download: {}".format( self.download_dir)) os.makedirs(self.download_dir) # generate a dict of products from # 1) the list of product_ids # 2) in case 1) is None then check the XCOM key in 'get_inputs_from' self.products = dict() if self.product_ids != None and len(self.product_ids) != 0: for product_id in self.product_ids: self.products[product_id] = "" print("Download request for {} products via IDs:\n{}".format( len(self.product_ids), self.products)) elif self.get_inputs_from != None: self.products = context['task_instance'].xcom_pull( task_ids=self.get_inputs_from, key=XCOM_RETURN_KEY) print("Downloading request for {} products via XCOM:\n{}".format( len(self.products), self.products)) else: # exit gracefully if no products are found log.info('no products to process') return None # log warning in case the amount of products exceed the limit if len(self.products) > self.download_max: log.warn("Found products ({}) exceeds download limit ({})".format( len(self.products), self.download_max)) # download all files via it's ID log.info('Starting downloading..') product_downloaded = {} api = SentinelAPI(self.dhus_user, self.dhus_pass, self.dhus_url) for product_id in self.products.keys(): # If download limit reached, stopp and break out # Else if the file already exists, then try next from search product_filename = os.path.join( self.download_dir, self.products[product_id].get("title") + ".zip") if len(product_downloaded) >= self.download_max: log.info("Limit exceeded, stopping download..") break elif os.path.exists(product_filename): log.info("Product already downloaded. Continuing..") continue log.info('Downloading Product..\nuuid: {}\ntitle: {}\nsize: {}'. format(product_id, self.products[product_id].get("title"), self.products[product_id].get("size"))) downloaded = api.download(product_id, directory_path=self.download_dir) path = downloaded['path'] # TODO check if file in 'path' is binary. # It might is an XML file containing an error such as # "Maximum number of 2 concurrent flows achieved by the user "xyz"" # Check MD5 # If file already downloaded move on to next one? product_downloaded[path] = downloaded # print summary and push products to XCOM log.debug("Downloaded {} products:\n{}".format( len(product_downloaded), pp.pprint(product_downloaded))) context['task_instance'].xcom_push(key='downloaded_products', value=product_downloaded) context['task_instance'].xcom_push(key='downloaded_products_paths', value=' '.join( product_downloaded.keys())) return product_downloaded
def download(user, password, productid, path, md5, url): """Download a Sentinel Product. It just needs your SciHub user and password and the id of the product you want to download. """ api = SentinelAPI(user, password, url) api.download(productid, path, md5)
def download_images(save_imgs, save_rgb, save_tiles, unet_weights, unet_clouds, class_path, class_clouds, poly_path, percentiles_forest, percentiles_clouds, boundsdata): # connect to the API user = '******' password = '******' api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') # search by polygon footprint = geojson_to_wkt(read_geojson(boundsdata)) # search for the images products = api.query( footprint, date=(["NOW-30DAYS", "NOW"]), area_relation='IsWithin', platformname='Sentinel-2', processinglevel='Level-2A', #cloudcoverpercentage = (0, 20) ) print(len(products)) table_names = api.to_geodataframe(products) uuid_names = table_names['uuid'] # names to download name_zip = table_names['title'] #title of image extracted_name = table_names['filename'] # name of folder .SAFE k = 0 # download images for fname in uuid_names: file_dir = save_imgs + '/' + extracted_name[k] if os.path.isdir(file_dir) is False: retval = os.getcwd() os.chdir(save_imgs) print("Downloading data...") api.get_product_odata(fname) api.download(fname) os.chdir(retval) # return to previous directory path_zip_name = save_imgs + '/' + name_zip[k] + '.zip' while not os.path.exists(path_zip_name): time.sleep(1) if os.path.isfile(path_zip_name): # extract files zip_ref = zipfile.ZipFile(path_zip_name, 'r') zip_ref.extractall(save_imgs) zip_ref.close() os.remove(path_zip_name) # remove .zip file print("%s has been removed successfully" % name_zip[k]) path_to_folder = save_imgs + '/' + extracted_name[ k] + '/GRANULE/' # calls the rgb_tiles function dir_save_tiles = save_tiles + '/' + name_zip[k] if os.path.isdir(dir_save_tiles) is False: print('Creating RGB tiles') os.mkdir(dir_save_tiles) rgb_tiles(path_to_folder, save_rgb, dir_save_tiles, name_zip[k]) # calls the application() Unet function save_class_path = class_path + '/' + name_zip[k] if os.path.isdir(save_class_path) is False: print('Applying UNet') os.mkdir(save_class_path) application(dir_save_tiles, unet_weights, save_class_path, percentiles_forest, clouds=0) # merge predicted tiles into one GeoTiff join_tiles(save_class_path, class_path, path_to_folder) print("Tiles merged!") save_class_clouds = class_clouds + '/' + name_zip[k] if os.path.isdir(save_class_clouds) is False: print('Applying UNet clouds') os.mkdir(save_class_clouds) application(dir_save_tiles, unet_clouds, save_class_clouds, percentiles_clouds, clouds=1) # merge the clouds predicted tiles into one GeoTiff join_tiles(save_class_clouds, class_clouds, path_to_folder) print("Clouds tiles merged!") # polygons evalutation print("Polygons evaluation") polygons(name_zip[k], class_path, class_clouds, path_to_folder, save_class_path, save_imgs, poly_path, time_spaced=None) k = k + 1 else: raise ValueError("%s isn't a file!" % path_zip_name) else: path_to_folder = save_imgs + '/' + extracted_name[k] + '/GRANULE/' # calls the rgb_tiles function dir_save_tiles = save_tiles + '/' + name_zip[k] if os.path.isdir(dir_save_tiles) is False: print('Creating RGB tiles') os.mkdir(dir_save_tiles) rgb_tiles(path_to_folder, save_rgb, dir_save_tiles, name_zip[k]) # calls the application() Unet function save_class_path = class_path + '/' + name_zip[k] if os.path.isdir(save_class_path) is False: print('Applying UNet') os.mkdir(save_class_path) application(dir_save_tiles, unet_weights, save_class_path, percentiles_forest, clouds=0) # merge predicted tiles into one GeoTiff join_tiles(save_class_path, class_path, path_to_folder) print("Tiles merged!") save_class_clouds = class_clouds + '/' + name_zip[k] if os.path.isdir(save_class_clouds) is False: print('Applying UNet clouds') os.mkdir(save_class_clouds) application(dir_save_tiles, unet_clouds, save_class_clouds, percentiles_clouds, clouds=1) # merge the clouds predicted tiles into one GeoTiff join_tiles(save_class_clouds, class_clouds, path_to_folder) print("Clouds tiles merged!") # polygons evalutation print("Polygons evaluation") polygons(name_zip[k], class_path, class_clouds, path_to_folder, save_class_path, save_imgs, poly_path, time_spaced=None) k = k + 1 return
class Sentinelsat_products: def __init__(self, date_start, date_finish, footprint=settings.footprint, platformname="Sentinel-3"): self.date_start = date_start self.date_finish = date_finish self.api = SentinelAPI(settings.sentinel_api_user, settings.sentinel_api_key, 'https://scihub.copernicus.eu/dhus') self.wkt_footprint = footprint self.products = self.query_products(self.date_start, self.date_finish) def query_products(self, date_start, date_finish, platformname="Sentinel-3"): # connect to the API api = SentinelAPI(settings.sentinel_api_user, settings.sentinel_api_key, 'https://scihub.copernicus.eu/dhus') # search by polygon, time, and Hub query keywords products = api.query(self.wkt_footprint, area_relation='Contains', date=(self.date_start, self.date_finish), platformname='Sentinel-3') return products def filter_products(self, instrument, level, p_type, timeliness): removed_products = [] for product_key in self.products: odata = self.api.get_product_odata(product_key, full=True) product_instrument = odata["Instrument"] product_level = odata["Product level"] product_type = odata["Product type"] #mission_type = odata["Mission type"] product_timeliness = odata["Timeliness Category"] #filter only from Level 1 OLCI instrument with NTC full resolution conditions = ((product_instrument == instrument) and (p_type in product_type) and product_timeliness == timeliness and product_level == level) if conditions: pass #print(instrument, product_level, product_type) else: removed_products.append(product_key) for key in removed_products: del self.products[key] def download_products(self, make_subset=True): print("----------") for key in self.products: file_name = self.products[key]["filename"] file_date = self.products[key]["summary"][:16].split("Date: ")[1] download_path = os.path.join(settings.data_path, file_date) if not os.path.exists(download_path): os.makedirs(download_path) # if it was downloaded before it won't download again download_info = self.api.download(key, directory_path=download_path) #print(download_info) zip_path = download_info["path"] with zipfile.ZipFile(zip_path, 'r') as zip_ref: zip_ref.extractall(download_path) if make_subset: extracted_directory = os.path.join(download_path, file_name) product = snappy_utils.read_product(extracted_directory) subset = snappy_utils.make_subset(product, settings.footprint) snappy_utils.write_product( subset, os.path.join(download_path, "laguna.tif"), "GeoTiff") snappy_utils.write_product( subset, os.path.join(download_path, "laguna.dim"), "BEAM-DIMAP")
api = SentinelAPI(user, password, 'https://scihub.copernicus.eu/dhus') #Setting the search footprint using a Geojson file footprint = geojson_to_wkt(read_geojson(AOI)) #Setting the search query parameters products = api.query(footprint, date = ('20200101', '20200331'), platformname = 'Sentinel-2', processinglevel = 'Level-2A', cloudcoverpercentage = (0, 20)) #Printing the number of products found print("The number of products found is: {} " .format (len(products))) #Creating a table with all the product search results products_table = api.to_geodataframe(products) #This part downloads the product(s) in the same folder where your code is located ## OPTION 1: Download single product api.download('df132697-676e-43ce-b7bd-45211696119f') ## OPTION 2:Download all products download_list = [] for index, row in products_table.iterrows(): download_list.append (row['title']) print ("The following products will be downloaded: {}" . format (download_list)) api.download (row['uuid'])
# Start Download for entry in range(0, len(df)): # The uuid element allows to create the path to the file uuid_element = df['uuid'][entry] id_sar = df['identifier'][entry] sentinel_link = df['link'][entry] # Destinationpath with filename where download to be stored destinationpath = dirpath + id_sar + '.zip' if os.path.exists(destinationpath): logger.info(id_sar + ' already downloaded') print(id_sar + ' already downloaded') else: # Download file and read try: api.download(df['uuid'][entry], directory_path=dirpath, checksum=True) logger.info("Successfully downloaded" + id_sar + 'in to' + destinationpath) except: logger.warning("error connection!.... Download Interrupted!") print("error connection!.... Download Interrupted!") time.sleep(1) # delete all incomplete file for item in os.listdir(dirpath): if item.endswith(".incomplete"): os.remove(os.path.join(dirpath, item))
def getSentinelFiles(DATE, COLHUB_UNAME, COLHUB_PW, TMPDIR, bbox, max_files=1, polarization='hh', platform='s1', time_window=1): print('Arguments -> Box: %s, Max downloads: %s, Polarization: %s, Platform: %s' \ %(bbox, max_files, polarization, platform)) # api = SentinelAPI(COLHUB_UNAME, COLHUB_PW, 'https://colhub.met.no/#/home') api = SentinelAPI(COLHUB_UNAME, COLHUB_PW, 'https://scihub.copernicus.eu/dhus/#/home') date = DATE.strftime('%Y%m%d') yestdate = (DATE - timedelta(time_window)).strftime('%Y%m%d') footprint = geojson_to_wkt(read_geojson(bbox)) try: if platform == 's1': products = api.query(footprint, (yestdate, date), platformname='Sentinel-1', producttype='GRD', sensoroperationalmode='EW') elif platform == 's2': products = api.query( footprint, (yestdate, date), platformname='Sentinel-2', cloudcoverpercentage=(0, 80) # TODO: find reasonable threshold ) else: print('Not a valid platform!') return [False] except SentinelAPIError as e: print(e) return [False] except: print( "Unknown error occurred while accessing Copernicus Open Access Hub" ) return [False] if len(products) == 0: print("No files found at date: " + date) return [False] print("Found", len(products), "Sentinel images.") if platform == 's2': products_df = api.to_dataframe(products).sort_values( ['cloudcoverpercentage', 'beginposition'], ascending=[True, True]) products_df = products_df.head(1) else: products_df = api.to_dataframe(products).sort_values('beginposition', ascending=True) downloadNames = [] for i in range(len(products_df)): print("Image %s / %s" % (i, len(products_df))) if i == max_files: # Prevents too large mosaic file break product_size = float(products_df['size'].values[i].split(' ')[0]) product_name = products_df['filename'].values[i][:-5] product_date = products_df['beginposition'].values[i] product_clouds = '' if platform == 's2': product_clouds = ', Cloudcover: ' + str( products_df['cloudcoverpercentage'].values[i]) print("Name: %s, size: %s MB%s" % (product_name, product_size, product_clouds)) # if max_files == 1: # No point with ingestion date in mosaics with several images # self.returns[platform + 'c'] = products_df['ingestiondate'].values[i] api.download(products_df['uuid'][i], TMPDIR) if platform == 's1': # UNZIPPING DOWNLOADED FILE print("Unzipping product...") zip_ref = zipfile.ZipFile(TMPDIR + product_name + '.zip') zip_ref.extractall(TMPDIR) zip_ref.close() geofiles = glob.glob(TMPDIR + product_name + '.SAFE/measurement/*') # FINDING RIGHT POLARIZATION FILE (now using HH-polarization) if '-' + polarization + '-' in geofiles[0]: downloadNames.append(geofiles[0]) else: downloadNames.append(geofiles[1]) elif platform == 's2': downloadNames.append(product_name) return downloadNames
def download(user, password, productid, path): """Download a Sentinel-1 Product. It just needs your SciHub user and password and the id of the product you want to download. """ api = SentinelAPI(user, password) api.download(productid, path)
ShapefileName1 = JsonFileName.split('.')[0] + '.shp' gdal.Warp('aRaster_ABC_cliped.tif', 'aRaster_ABC.tif', cutlineDSName=ShapefileName1, dstAlpha=True) plt.imshow(gdal.Open('aRaster_ABC_cliped.tif').ReadAsArray()[0]) plt.show() plt.imshow(gdal.Open('aRaster_ABC.tif').ReadAsArray()[0]) plt.show() Identifier = ProductID['Identifier'][int(ID)] Decision = raw_input('Do you think this is correct [Y or N]: ') os.remove('abc.tif') os.remove('aRaster_ABC.tif') os.remove('aRaster_ABC_cliped.tif') api.download(ProductID['Pr_Id'][int(ID)], Path_to_Download) elif Kind == 'A': print 'You all Data is being downloading\n' for item in ProductID['Pr_Id']: api.download(item, Path_to_Download) elif Kind == 'M': print 'You are downloading Multipel Scene ID' ID = raw_input( 'Type Multipel Scene ID number separated by , eg: 1,2,5\nTypeNow: ' ) ID = ID.split(',') for i in ID: i = int(i) api.download(ProductID['Pr_Id'][i], Path_to_Download)