def retrieve_mata_data(filename,username,password):
url = 'https://scihub.copernicus.eu/dhus'
info = filename.split('_')
satellite = info[0]
mode = info[1]
product = info[2]
orbitnumber = np.int(info[7])
time_start = np.int(info[5].split('T')[0])-1
time_end = str(np.int(time_start+2))
time_start = str(time_start)
api = SentinelAPI(username, password, url)
products = api.query(
beginposition=(time_start,time_end),
platformname='Sentinel-1',
producttype=product,
sensoroperationalmode=mode,
polarisationmode='VV VH',
orbitnumber=orbitnumber
)
products_df = api.to_dataframe(products)
index = -1
for i in range(len(products_df)):
if products_df['title'][i] in filename:
index = i
return products_df.iloc[index]
def test_to_pandas():
api = SentinelAPI(**_api_auth)
products = api.query(get_coordinates('tests/map.geojson'),
"20151219",
"20151228",
platformname="Sentinel-2")
df = api.to_dataframe(products)
assert 'S2A_OPER_PRD_MSIL1C_PDMC_20151228T112701_R110_V20151227T142229_20151227T142229' in df.index
# set query parameters
query_kwargs = {
'area': footprint,
'platformname': 'Sentinel-1',
'producttype': 'GRD',
# orbitdirection='ASCENDING'),
'date': (datefrom, dateto),
# 'processinglevel': 'Level-1C',
# 'cloudcoverpercentage': clouds
}
# search the Sentinel data hub API
products = api.query(**query_kwargs)
# convert list of products to Pandas DataFrame
products_df = api.to_dataframe(products)
print('Search resulted in ' + str(products_df.shape[0]) +
' satellite images with ' + str(products_df.shape[1]) + ' attributes.')
# sort the search results
products_df_sorted = products_df.sort_values(['ingestiondate'],
ascending=[True])
#print(products_df_sorted)
outfile = 'searchresults_full.csv'
products_df_sorted.to_csv(outfile)
# limit to first Ndown products sorted by lowest cloud cover and earliest acquisition date
products_df_n = products_df_sorted.head(ndown)
outfile = 'searchresults4download.csv'
products_df_n.to_csv(outfile)
class Processor():
def __init__(self,
sentinel_user,
sentinel_pass,
start_date,
end_date,
dl_dir,
input_file,
debug=False):
self.SENTINEL_USER = sentinel_user
self.SENTINEL_PASS = sentinel_pass
self.DL_DIR = dl_dir
self.INPUT_FILE = input_file
self.START_DATE = start_date
self.END_DATE = end_date
self.DEBUG = debug
if not os.path.exists(self.DL_DIR):
os.mkdir(self.DL_DIR)
def phase_1(self):
self.api = SentinelAPI(self.SENTINEL_USER, self.SENTINEL_PASS)
self.aoi_footprint = geojson_to_wkt(read_geojson(self.INPUT_FILE))
def phase_2(self):
self.api_products = self.api.query(
self.aoi_footprint,
date=(self.START_DATE, self.END_DATE),
area_relation='Intersects',
platformname='Sentinel-2',
cloudcoverpercentage=(0, 30),
)
def phase_3(self):
"""
We're doing the conversion from a GeoDataFrame to a list of dictionaries.
After the conversion we intend to use the "footprint" and the "index" columns.
This step is required because there are multiple products with the same footprint
and later on we need the index in order to download the images from SentinelAPI.
"""
self.product_df = self.api.to_dataframe(self.api_products)
if len(self.product_df.index) == 0:
raise Exception("No images for selected period")
self.product_df = self.product_df.sort_values(
['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True])
self.tile_footprints = []
for x in self.product_df[[
"size", "tileid", "processinglevel", "footprint"
]].T.to_dict().items():
self.tile_footprints.append({**x[1], "index": x[0]})
if self.DEBUG:
pprint(self.tile_footprints[:3])
def phase_4(self):
L1 = min_cover_1(self.tile_footprints)
if self.DEBUG:
print("{} tiles after the 1st reduction".format(len(L1)))
L2 = min_cover_2(L1)
if self.DEBUG:
print("{} tiles after the 2nd reduction".format(len(L2)))
self.reduced_footprints = L2
def phase_5(self):
dl_indexes = [x["index"] for x in self.reduced_footprints]
self.api.download_all(dl_indexes, directory_path=self.DL_DIR)
if self.DEBUG:
pprint(dl_indexes)
def phase_6(self):
"""
We're decompressing the archives unless they're already decompressed.
"""
for p in pathlib.Path(self.DL_DIR).iterdir():
p_dir = re.sub('\.zip$', '.SAFE', str(p))
if os.path.isfile(p) and not os.path.exists(p_dir):
extract_path = os.path.dirname(p)
print("Dezarhivare " + str(p))
with zipfile.ZipFile(p, 'r') as zip_ref:
zip_ref.extractall(extract_path)
def phase_7(self):
"""
Converting the .jp2 images to .tiff
"""
def select_files(path, pattern):
L = []
for root, dirs, files in os.walk(path):
if len(dirs) == 0:
for f in files:
if re.match(pattern, f):
L.append(os.path.join(root, f))
return L
def convert_to_tiff(paths):
tiff_paths = []
for p in paths:
print("Converting " + p)
with rasterio.open(p, mode="r") as src:
profile = src.meta.copy()
profile.update(driver="GTiff")
outfile = re.sub(".jp2", ".tiff", p)
with rasterio.open(outfile, 'w', **profile) as dst:
dst.write(src.read())
tiff_paths.append(outfile)
return tiff_paths
self.jp2_paths = select_files(self.DL_DIR, ".*_TCI.jp2$")
self.tiff_paths = convert_to_tiff(self.jp2_paths)
def phase_8(self):
"""
We're mergin the raster images.
"""
raster_list = [
rasterio.open(f, mode='r', driver="GTiff") for f in self.tiff_paths
]
merged_data, out_trans = rasterio.merge.merge(raster_list)
if self.DEBUG:
fig, ax = plt.subplots(figsize=(14, 14))
show(merged_data, cmap='terrain', ax=ax)
merged_meta = raster_list[0].meta.copy()
merged_meta.update({
"driver": "GTiff",
"height": merged_data.shape[1],
"width": merged_data.shape[2],
"transform": out_trans,
"crs": raster_list[0].crs,
"count": 3,
})
if self.DEBUG:
for x in [x.meta for x in raster_list] + [merged_meta]:
pprint(x)
self.MERGED_RAW = os.path.join(self.DL_DIR, "merged1.tiff")
with rasterio.open(self.MERGED_RAW, mode="w", **merged_meta) as dest:
dest.write(merged_data)
def phase_9(self):
"""
Reprojecting the images to EPSG:4326
"""
dst_crs = 'EPSG:4326'
with rasterio.open(self.MERGED_RAW) as src:
transform, width, height = calculate_default_transform(
src.crs, dst_crs, src.width, src.height, *src.bounds)
kwargs = src.meta.copy()
kwargs.update({
'crs': dst_crs,
'transform': transform,
'width': width,
'height': height
})
self.MERGED_4326 = os.path.join(self.DL_DIR, "merged1_4326.tiff")
with rasterio.open(self.MERGED_4326, mode="w", **kwargs) as dst:
for i in range(1, src.count + 1):
reproject(source=rasterio.band(src, i),
destination=rasterio.band(dst, i),
src_transform=src.transform,
src_crs=src.crs,
dst_transform=transform,
dst_crs=dst_crs,
resampling=Resampling.nearest)
def phase_10(self):
"""
We're clipping the area of interest.
"""
with rasterio.open(self.MERGED_4326) as src:
out_image, out_transform = rasterio.mask.mask(
src, [shapely.wkt.loads(self.aoi_footprint)], crop=True)
out_meta = src.meta
out_meta.update({
"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform,
})
self.MERGED_REGION = os.path.join(self.DL_DIR,
"merged1_region.tiff")
with rasterio.open(self.MERGED_REGION, "w", **out_meta) as dest:
dest.write(out_image)
if self.DEBUG:
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(14, 14))
from rasterio.plot import show
show(out_image, cmap='terrain', ax=ax)
def reset(self):
"""
We're resetting object state to allow for a subsequent run.
"""
self.aoi_footprint = None
self.api_products = None
self.product_df = None
self.tile_footprints = None
self.reduced_footprints = None
self.jp2_paths = None
self.tiff_paths = None
self.MERGED_RAW = None
self.MERGED_4326 = None
self.MERGED_REGION = None
def get_image(
footprint, date, search_window, destination, bands=["B03", "B11"], choose=False
):
"""
Retrieves sentinel-2 scenes.
Scenes must entirely contain <footprint>
within date +/- search_window.
Downloads <bands> to
<dir>/sentinel2.
Requires SCIHUB_USERNAME and SCIHUB_PASSWORD environment variables.
"""
print("Searching for Sentinel-2 Imagery...")
workdir = path.join(destination, "sentinel-2")
makedirs(workdir, exist_ok=True)
api = SentinelAPI(
environ["SCIHUB_USERNAME"],
environ["SCIHUB_PASSWORD"],
"https://scihub.copernicus.eu/dhus",
)
footprint_wkt = geojson_to_wkt(read_geojson(path.join(destination, footprint)))
footprint_geom = wkt.loads(footprint_wkt)
date_window = timedelta(days=search_window)
date_range = (date - date_window, date + date_window)
q = api.query(footprint_wkt, date=date_range, platformname="Sentinel-2")
results = api.to_dataframe(q)
# filter results
does_overlap = [
wkt.loads(i_fp).contains(footprint_geom) for i_fp in results.footprint
]
results = results[does_overlap]
print("Overlapping scenes: {}".format(len(results)))
results.to_csv(path.join(workdir, "s2-collects.csv"))
# choose result that's closest in time
results["timedeltas"] = (date - results.datatakesensingstart).abs()
results = results.sort_values(by="timedeltas", ascending=True)
image_iloc = 0
if choose:
print(
tabulate(
results[
["datatakesensingstart", "cloudcoverpercentage", "timedeltas"]
].reset_index(drop=True),
headers="keys",
)
)
image_iloc = int(input("Choose image ID [0-{}]: ".format(len(results) - 1)))
# build request for AWS data.
tile_name, time, aws_index = AwsTile.tile_id_to_tile(
results.iloc[image_iloc].level1cpdiidentifier
)
metafiles = ["tileInfo", "preview"]
request = AwsTileRequest(
tile=tile_name,
time=time,
aws_index=aws_index,
bands=bands,
metafiles=metafiles,
data_folder=workdir,
data_source=DataSource.SENTINEL2_L1C,
)
if input("Download? (y/n): ").lower() == "y":
request.save_data()
else:
print("Aborted.")
return None
dateparts = time.split("-")
zero_pad_date = "{:d}-{:02d}-{:d}".format(
int(dateparts[0]), int(dateparts[1]), int(dateparts[2])
)
imgpath = path.join(
workdir, ",".join([str(tile_name), zero_pad_date, str(aws_index)])
)
print(imgpath)
return imgpath
def getSentinelFiles(DATE,
COLHUB_UNAME,
COLHUB_PW,
TMPDIR,
bbox,
max_files=1,
polarization='hh',
platform='s1',
time_window=1):
print('Arguments -> Box: %s, Max downloads: %s, Polarization: %s, Platform: %s' \
%(bbox, max_files, polarization, platform))
# api = SentinelAPI(COLHUB_UNAME, COLHUB_PW, 'https://colhub.met.no/#/home')
api = SentinelAPI(COLHUB_UNAME, COLHUB_PW,
'https://scihub.copernicus.eu/dhus/#/home')
date = DATE.strftime('%Y%m%d')
yestdate = (DATE - timedelta(time_window)).strftime('%Y%m%d')
footprint = geojson_to_wkt(read_geojson(bbox))
try:
if platform == 's1':
products = api.query(footprint, (yestdate, date),
platformname='Sentinel-1',
producttype='GRD',
sensoroperationalmode='EW')
elif platform == 's2':
products = api.query(
footprint,
(yestdate, date),
platformname='Sentinel-2',
cloudcoverpercentage=(0, 80) # TODO: find reasonable threshold
)
else:
print('Not a valid platform!')
return [False]
except SentinelAPIError as e:
print(e)
return [False]
except:
print(
"Unknown error occurred while accessing Copernicus Open Access Hub"
)
return [False]
if len(products) == 0:
print("No files found at date: " + date)
return [False]
print("Found", len(products), "Sentinel images.")
if platform == 's2':
products_df = api.to_dataframe(products).sort_values(
['cloudcoverpercentage', 'beginposition'], ascending=[True, True])
products_df = products_df.head(1)
else:
products_df = api.to_dataframe(products).sort_values('beginposition',
ascending=True)
downloadNames = []
for i in range(len(products_df)):
print("Image %s / %s" % (i, len(products_df)))
if i == max_files: # Prevents too large mosaic file
break
product_size = float(products_df['size'].values[i].split(' ')[0])
product_name = products_df['filename'].values[i][:-5]
product_date = products_df['beginposition'].values[i]
product_clouds = ''
if platform == 's2':
product_clouds = ', Cloudcover: ' + str(
products_df['cloudcoverpercentage'].values[i])
print("Name: %s, size: %s MB%s" %
(product_name, product_size, product_clouds))
# if max_files == 1: # No point with ingestion date in mosaics with several images
# self.returns[platform + 'c'] = products_df['ingestiondate'].values[i]
api.download(products_df['uuid'][i], TMPDIR)
if platform == 's1':
# UNZIPPING DOWNLOADED FILE
print("Unzipping product...")
zip_ref = zipfile.ZipFile(TMPDIR + product_name + '.zip')
zip_ref.extractall(TMPDIR)
zip_ref.close()
geofiles = glob.glob(TMPDIR + product_name + '.SAFE/measurement/*')
# FINDING RIGHT POLARIZATION FILE (now using HH-polarization)
if '-' + polarization + '-' in geofiles[0]:
downloadNames.append(geofiles[0])
else:
downloadNames.append(geofiles[1])
elif platform == 's2':
downloadNames.append(product_name)
return downloadNames
class S1Download(object):
"""
This module makes searching, downloading and retrieving metadata of Sentinel-1 satellite images from the
Copernicus Open Access Hub easy.
Parameters
----------
username : str
Username for Copernicus Open Access Hub
password : str
Password for Copernicus Open Access Hub
region : str
A geojson file.
timestart : str
Start time like "YYYY-MM-DD"
timeend : str
End time like "YYYY-MM-DD".
outdir : str
Output directory.
producttype : {'SLC', 'GRD', 'OCN', ''RAW}
Product type. If None, all types will be recognized.
polarisationmode tuple or str
A combination of V and H like ('VH', 'HV') or simple 'VH'.
sensoroperationalmode : {'SM', 'IW', 'EW', 'WV'}
Sensor operational mode. If None, all types will be recognized.
orbitnumber : int
Orbit number
orbitdirection : {DESCENDING, ASCENDING}
Orbit direction. If None, all types will be recognized.
Attributes
----------
api : object
Sentinelsat API object.
outdir : str
region : wkt
A geojson to WKT object.
kwargs : dict
Dictionary with setted attributes.
files : DataFrame
Pandas DataFrame with detected files.
Methods
-------
download()
Download all files.
print_products()
Print all detected files.
Examples
--------
The general usage is
::
$ ds1.download [-p] username=string password=string region=string timestart=string timeend=string outdir=sting
[*attributes=string] [--verbose] [--quiet]
For *attributes the following parameters can be used
::
>>> ["producttype", "polarisationmode", "sensoroperationalmode", "orbitnumber", "orbitdirection"]
Print all Sentinel 1 data with product type GRD between 2015-01-02 and 2015-01-12::
$ ds1.download -p username=USER password=PASSWORD region=myGEoJsOnFile.geojson timestart=2015-01-02
timeend=2015-01-12 outdir='home/usr/data' producttype=SLC
Download the last query
::
$ ds1.download username=USER password=PASSWORD region=myGEoJsOnFile.geojson timestart=2015-01-02
timeend=2015-01-12 outdir='home/usr/data' producttype=SLC
Notes
-----
**Flags:**
* p : Print the detected files and exit.
"""
def __init__(self,
username,
password,
region,
timestart,
timeend,
outdir,
producttype=None,
polarisationmode=None,
sensoroperationalmode=None,
orbitnumber=None,
orbitdirection=None):
# Inititalise Sentinel Python API ------------------------------------------------------------------------------
self.api = SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus')
# Initialize Directory -----------------------------------------------------------------------------------------
if not os.path.exists(outdir):
os.makedirs(outdir)
else:
self.outdir = outdir
# Initialise Mandatory Parameter ------------------------------------------------------------------------------
self.region = geojson_to_wkt(read_geojson(region))
# < Reformat Time > ------------
timestart_split = timestart.split('-')
timestart_temp = ''
for item in timestart_split:
timestart_temp += item
timeend_split = timeend.split('-')
timeend_temp = ''
for item in timeend_split:
timeend_temp += item
self.date = (timestart_temp, timeend_temp)
# Initialize Attributes ----------------------------------------------------------------------------------------
input_parameter = [
producttype, polarisationmode, sensoroperationalmode, orbitnumber,
orbitdirection
]
__KEYS__ = [
"producttype", "polarisationmode", "sensoroperationalmode",
"orbitnumber", "orbitdirection"
]
self.kwargs = {}
for i, item in enumerate(input_parameter):
if item is not None:
self.kwargs[__KEYS__[i]] = item
# Inititalise Sentinel Producs with API ------------------------------------------------------------------------
self.products = self.api.query(self.region,
date=self.date,
platformname='Sentinel-1',
**self.kwargs)
self.files = self.api.to_dataframe(self.products)
def download(self):
self.api.download_all(self.products, directory_path=self.outdir)
return 0
def print_products(self):
"""
Print all detected files.
Returns
-------
None
"""
df = self.files.to_string()
sys.stdout.write(df)
