def retrieve_mata_data(filename,username,password): url = 'https://scihub.copernicus.eu/dhus' info = filename.split('_') satellite = info[0] mode = info[1] product = info[2] orbitnumber = np.int(info[7]) time_start = np.int(info[5].split('T')[0])-1 time_end = str(np.int(time_start+2)) time_start = str(time_start) api = SentinelAPI(username, password, url) products = api.query( beginposition=(time_start,time_end), platformname='Sentinel-1', producttype=product, sensoroperationalmode=mode, polarisationmode='VV VH', orbitnumber=orbitnumber ) products_df = api.to_dataframe(products) index = -1 for i in range(len(products_df)): if products_df['title'][i] in filename: index = i return products_df.iloc[index]
def test_to_pandas(): api = SentinelAPI(**_api_auth) products = api.query(get_coordinates('tests/map.geojson'), "20151219", "20151228", platformname="Sentinel-2") df = api.to_dataframe(products) assert 'S2A_OPER_PRD_MSIL1C_PDMC_20151228T112701_R110_V20151227T142229_20151227T142229' in df.index
# set query parameters query_kwargs = { 'area': footprint, 'platformname': 'Sentinel-1', 'producttype': 'GRD', # orbitdirection='ASCENDING'), 'date': (datefrom, dateto), # 'processinglevel': 'Level-1C', # 'cloudcoverpercentage': clouds } # search the Sentinel data hub API products = api.query(**query_kwargs) # convert list of products to Pandas DataFrame products_df = api.to_dataframe(products) print('Search resulted in ' + str(products_df.shape[0]) + ' satellite images with ' + str(products_df.shape[1]) + ' attributes.') # sort the search results products_df_sorted = products_df.sort_values(['ingestiondate'], ascending=[True]) #print(products_df_sorted) outfile = 'searchresults_full.csv' products_df_sorted.to_csv(outfile) # limit to first Ndown products sorted by lowest cloud cover and earliest acquisition date products_df_n = products_df_sorted.head(ndown) outfile = 'searchresults4download.csv' products_df_n.to_csv(outfile)
class Processor(): def __init__(self, sentinel_user, sentinel_pass, start_date, end_date, dl_dir, input_file, debug=False): self.SENTINEL_USER = sentinel_user self.SENTINEL_PASS = sentinel_pass self.DL_DIR = dl_dir self.INPUT_FILE = input_file self.START_DATE = start_date self.END_DATE = end_date self.DEBUG = debug if not os.path.exists(self.DL_DIR): os.mkdir(self.DL_DIR) def phase_1(self): self.api = SentinelAPI(self.SENTINEL_USER, self.SENTINEL_PASS) self.aoi_footprint = geojson_to_wkt(read_geojson(self.INPUT_FILE)) def phase_2(self): self.api_products = self.api.query( self.aoi_footprint, date=(self.START_DATE, self.END_DATE), area_relation='Intersects', platformname='Sentinel-2', cloudcoverpercentage=(0, 30), ) def phase_3(self): """ We're doing the conversion from a GeoDataFrame to a list of dictionaries. After the conversion we intend to use the "footprint" and the "index" columns. This step is required because there are multiple products with the same footprint and later on we need the index in order to download the images from SentinelAPI. """ self.product_df = self.api.to_dataframe(self.api_products) if len(self.product_df.index) == 0: raise Exception("No images for selected period") self.product_df = self.product_df.sort_values( ['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]) self.tile_footprints = [] for x in self.product_df[[ "size", "tileid", "processinglevel", "footprint" ]].T.to_dict().items(): self.tile_footprints.append({**x[1], "index": x[0]}) if self.DEBUG: pprint(self.tile_footprints[:3]) def phase_4(self): L1 = min_cover_1(self.tile_footprints) if self.DEBUG: print("{} tiles after the 1st reduction".format(len(L1))) L2 = min_cover_2(L1) if self.DEBUG: print("{} tiles after the 2nd reduction".format(len(L2))) self.reduced_footprints = L2 def phase_5(self): dl_indexes = [x["index"] for x in self.reduced_footprints] self.api.download_all(dl_indexes, directory_path=self.DL_DIR) if self.DEBUG: pprint(dl_indexes) def phase_6(self): """ We're decompressing the archives unless they're already decompressed. """ for p in pathlib.Path(self.DL_DIR).iterdir(): p_dir = re.sub('\.zip$', '.SAFE', str(p)) if os.path.isfile(p) and not os.path.exists(p_dir): extract_path = os.path.dirname(p) print("Dezarhivare " + str(p)) with zipfile.ZipFile(p, 'r') as zip_ref: zip_ref.extractall(extract_path) def phase_7(self): """ Converting the .jp2 images to .tiff """ def select_files(path, pattern): L = [] for root, dirs, files in os.walk(path): if len(dirs) == 0: for f in files: if re.match(pattern, f): L.append(os.path.join(root, f)) return L def convert_to_tiff(paths): tiff_paths = [] for p in paths: print("Converting " + p) with rasterio.open(p, mode="r") as src: profile = src.meta.copy() profile.update(driver="GTiff") outfile = re.sub(".jp2", ".tiff", p) with rasterio.open(outfile, 'w', **profile) as dst: dst.write(src.read()) tiff_paths.append(outfile) return tiff_paths self.jp2_paths = select_files(self.DL_DIR, ".*_TCI.jp2$") self.tiff_paths = convert_to_tiff(self.jp2_paths) def phase_8(self): """ We're mergin the raster images. """ raster_list = [ rasterio.open(f, mode='r', driver="GTiff") for f in self.tiff_paths ] merged_data, out_trans = rasterio.merge.merge(raster_list) if self.DEBUG: fig, ax = plt.subplots(figsize=(14, 14)) show(merged_data, cmap='terrain', ax=ax) merged_meta = raster_list[0].meta.copy() merged_meta.update({ "driver": "GTiff", "height": merged_data.shape[1], "width": merged_data.shape[2], "transform": out_trans, "crs": raster_list[0].crs, "count": 3, }) if self.DEBUG: for x in [x.meta for x in raster_list] + [merged_meta]: pprint(x) self.MERGED_RAW = os.path.join(self.DL_DIR, "merged1.tiff") with rasterio.open(self.MERGED_RAW, mode="w", **merged_meta) as dest: dest.write(merged_data) def phase_9(self): """ Reprojecting the images to EPSG:4326 """ dst_crs = 'EPSG:4326' with rasterio.open(self.MERGED_RAW) as src: transform, width, height = calculate_default_transform( src.crs, dst_crs, src.width, src.height, *src.bounds) kwargs = src.meta.copy() kwargs.update({ 'crs': dst_crs, 'transform': transform, 'width': width, 'height': height }) self.MERGED_4326 = os.path.join(self.DL_DIR, "merged1_4326.tiff") with rasterio.open(self.MERGED_4326, mode="w", **kwargs) as dst: for i in range(1, src.count + 1): reproject(source=rasterio.band(src, i), destination=rasterio.band(dst, i), src_transform=src.transform, src_crs=src.crs, dst_transform=transform, dst_crs=dst_crs, resampling=Resampling.nearest) def phase_10(self): """ We're clipping the area of interest. """ with rasterio.open(self.MERGED_4326) as src: out_image, out_transform = rasterio.mask.mask( src, [shapely.wkt.loads(self.aoi_footprint)], crop=True) out_meta = src.meta out_meta.update({ "driver": "GTiff", "height": out_image.shape[1], "width": out_image.shape[2], "transform": out_transform, }) self.MERGED_REGION = os.path.join(self.DL_DIR, "merged1_region.tiff") with rasterio.open(self.MERGED_REGION, "w", **out_meta) as dest: dest.write(out_image) if self.DEBUG: import matplotlib.pyplot as plt fig, ax = plt.subplots(figsize=(14, 14)) from rasterio.plot import show show(out_image, cmap='terrain', ax=ax) def reset(self): """ We're resetting object state to allow for a subsequent run. """ self.aoi_footprint = None self.api_products = None self.product_df = None self.tile_footprints = None self.reduced_footprints = None self.jp2_paths = None self.tiff_paths = None self.MERGED_RAW = None self.MERGED_4326 = None self.MERGED_REGION = None
def get_image( footprint, date, search_window, destination, bands=["B03", "B11"], choose=False ): """ Retrieves sentinel-2 scenes. Scenes must entirely contain <footprint> within date +/- search_window. Downloads <bands> to <dir>/sentinel2. Requires SCIHUB_USERNAME and SCIHUB_PASSWORD environment variables. """ print("Searching for Sentinel-2 Imagery...") workdir = path.join(destination, "sentinel-2") makedirs(workdir, exist_ok=True) api = SentinelAPI( environ["SCIHUB_USERNAME"], environ["SCIHUB_PASSWORD"], "https://scihub.copernicus.eu/dhus", ) footprint_wkt = geojson_to_wkt(read_geojson(path.join(destination, footprint))) footprint_geom = wkt.loads(footprint_wkt) date_window = timedelta(days=search_window) date_range = (date - date_window, date + date_window) q = api.query(footprint_wkt, date=date_range, platformname="Sentinel-2") results = api.to_dataframe(q) # filter results does_overlap = [ wkt.loads(i_fp).contains(footprint_geom) for i_fp in results.footprint ] results = results[does_overlap] print("Overlapping scenes: {}".format(len(results))) results.to_csv(path.join(workdir, "s2-collects.csv")) # choose result that's closest in time results["timedeltas"] = (date - results.datatakesensingstart).abs() results = results.sort_values(by="timedeltas", ascending=True) image_iloc = 0 if choose: print( tabulate( results[ ["datatakesensingstart", "cloudcoverpercentage", "timedeltas"] ].reset_index(drop=True), headers="keys", ) ) image_iloc = int(input("Choose image ID [0-{}]: ".format(len(results) - 1))) # build request for AWS data. tile_name, time, aws_index = AwsTile.tile_id_to_tile( results.iloc[image_iloc].level1cpdiidentifier ) metafiles = ["tileInfo", "preview"] request = AwsTileRequest( tile=tile_name, time=time, aws_index=aws_index, bands=bands, metafiles=metafiles, data_folder=workdir, data_source=DataSource.SENTINEL2_L1C, ) if input("Download? (y/n): ").lower() == "y": request.save_data() else: print("Aborted.") return None dateparts = time.split("-") zero_pad_date = "{:d}-{:02d}-{:d}".format( int(dateparts[0]), int(dateparts[1]), int(dateparts[2]) ) imgpath = path.join( workdir, ",".join([str(tile_name), zero_pad_date, str(aws_index)]) ) print(imgpath) return imgpath
def getSentinelFiles(DATE, COLHUB_UNAME, COLHUB_PW, TMPDIR, bbox, max_files=1, polarization='hh', platform='s1', time_window=1): print('Arguments -> Box: %s, Max downloads: %s, Polarization: %s, Platform: %s' \ %(bbox, max_files, polarization, platform)) # api = SentinelAPI(COLHUB_UNAME, COLHUB_PW, 'https://colhub.met.no/#/home') api = SentinelAPI(COLHUB_UNAME, COLHUB_PW, 'https://scihub.copernicus.eu/dhus/#/home') date = DATE.strftime('%Y%m%d') yestdate = (DATE - timedelta(time_window)).strftime('%Y%m%d') footprint = geojson_to_wkt(read_geojson(bbox)) try: if platform == 's1': products = api.query(footprint, (yestdate, date), platformname='Sentinel-1', producttype='GRD', sensoroperationalmode='EW') elif platform == 's2': products = api.query( footprint, (yestdate, date), platformname='Sentinel-2', cloudcoverpercentage=(0, 80) # TODO: find reasonable threshold ) else: print('Not a valid platform!') return [False] except SentinelAPIError as e: print(e) return [False] except: print( "Unknown error occurred while accessing Copernicus Open Access Hub" ) return [False] if len(products) == 0: print("No files found at date: " + date) return [False] print("Found", len(products), "Sentinel images.") if platform == 's2': products_df = api.to_dataframe(products).sort_values( ['cloudcoverpercentage', 'beginposition'], ascending=[True, True]) products_df = products_df.head(1) else: products_df = api.to_dataframe(products).sort_values('beginposition', ascending=True) downloadNames = [] for i in range(len(products_df)): print("Image %s / %s" % (i, len(products_df))) if i == max_files: # Prevents too large mosaic file break product_size = float(products_df['size'].values[i].split(' ')[0]) product_name = products_df['filename'].values[i][:-5] product_date = products_df['beginposition'].values[i] product_clouds = '' if platform == 's2': product_clouds = ', Cloudcover: ' + str( products_df['cloudcoverpercentage'].values[i]) print("Name: %s, size: %s MB%s" % (product_name, product_size, product_clouds)) # if max_files == 1: # No point with ingestion date in mosaics with several images # self.returns[platform + 'c'] = products_df['ingestiondate'].values[i] api.download(products_df['uuid'][i], TMPDIR) if platform == 's1': # UNZIPPING DOWNLOADED FILE print("Unzipping product...") zip_ref = zipfile.ZipFile(TMPDIR + product_name + '.zip') zip_ref.extractall(TMPDIR) zip_ref.close() geofiles = glob.glob(TMPDIR + product_name + '.SAFE/measurement/*') # FINDING RIGHT POLARIZATION FILE (now using HH-polarization) if '-' + polarization + '-' in geofiles[0]: downloadNames.append(geofiles[0]) else: downloadNames.append(geofiles[1]) elif platform == 's2': downloadNames.append(product_name) return downloadNames
class S1Download(object): """ This module makes searching, downloading and retrieving metadata of Sentinel-1 satellite images from the Copernicus Open Access Hub easy. Parameters ---------- username : str Username for Copernicus Open Access Hub password : str Password for Copernicus Open Access Hub region : str A geojson file. timestart : str Start time like "YYYY-MM-DD" timeend : str End time like "YYYY-MM-DD". outdir : str Output directory. producttype : {'SLC', 'GRD', 'OCN', ''RAW} Product type. If None, all types will be recognized. polarisationmode tuple or str A combination of V and H like ('VH', 'HV') or simple 'VH'. sensoroperationalmode : {'SM', 'IW', 'EW', 'WV'} Sensor operational mode. If None, all types will be recognized. orbitnumber : int Orbit number orbitdirection : {DESCENDING, ASCENDING} Orbit direction. If None, all types will be recognized. Attributes ---------- api : object Sentinelsat API object. outdir : str region : wkt A geojson to WKT object. kwargs : dict Dictionary with setted attributes. files : DataFrame Pandas DataFrame with detected files. Methods ------- download() Download all files. print_products() Print all detected files. Examples -------- The general usage is :: $ ds1.download [-p] username=string password=string region=string timestart=string timeend=string outdir=sting [*attributes=string] [--verbose] [--quiet] For *attributes the following parameters can be used :: >>> ["producttype", "polarisationmode", "sensoroperationalmode", "orbitnumber", "orbitdirection"] Print all Sentinel 1 data with product type GRD between 2015-01-02 and 2015-01-12:: $ ds1.download -p username=USER password=PASSWORD region=myGEoJsOnFile.geojson timestart=2015-01-02 timeend=2015-01-12 outdir='home/usr/data' producttype=SLC Download the last query :: $ ds1.download username=USER password=PASSWORD region=myGEoJsOnFile.geojson timestart=2015-01-02 timeend=2015-01-12 outdir='home/usr/data' producttype=SLC Notes ----- **Flags:** * p : Print the detected files and exit. """ def __init__(self, username, password, region, timestart, timeend, outdir, producttype=None, polarisationmode=None, sensoroperationalmode=None, orbitnumber=None, orbitdirection=None): # Inititalise Sentinel Python API ------------------------------------------------------------------------------ self.api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus') # Initialize Directory ----------------------------------------------------------------------------------------- if not os.path.exists(outdir): os.makedirs(outdir) else: self.outdir = outdir # Initialise Mandatory Parameter ------------------------------------------------------------------------------ self.region = geojson_to_wkt(read_geojson(region)) # < Reformat Time > ------------ timestart_split = timestart.split('-') timestart_temp = '' for item in timestart_split: timestart_temp += item timeend_split = timeend.split('-') timeend_temp = '' for item in timeend_split: timeend_temp += item self.date = (timestart_temp, timeend_temp) # Initialize Attributes ---------------------------------------------------------------------------------------- input_parameter = [ producttype, polarisationmode, sensoroperationalmode, orbitnumber, orbitdirection ] __KEYS__ = [ "producttype", "polarisationmode", "sensoroperationalmode", "orbitnumber", "orbitdirection" ] self.kwargs = {} for i, item in enumerate(input_parameter): if item is not None: self.kwargs[__KEYS__[i]] = item # Inititalise Sentinel Producs with API ------------------------------------------------------------------------ self.products = self.api.query(self.region, date=self.date, platformname='Sentinel-1', **self.kwargs) self.files = self.api.to_dataframe(self.products) def download(self): self.api.download_all(self.products, directory_path=self.outdir) return 0 def print_products(self): """ Print all detected files. Returns ------- None """ df = self.files.to_string() sys.stdout.write(df)