def get_products(login_json, coordinates, date_start, date_end, download_dir):
with open(login_json, 'r') as fp:
LOGIN_INFO = json.load(fp)
USER_NAME, PASSWORD = list(LOGIN_INFO.values())
# connect to the API
api = SentinelAPI(USER_NAME, PASSWORD, 'https://scihub.copernicus.eu/dhus')
# define a map polygon
geojson = Polygon(coordinates=coordinates)
# search by polygon, time, and Hub query keywords
footprint = geojson_to_wkt(geojson)
dates = (date_start, date_end) # (date(2018, 4, 1), date(2018, 4, 11))
# June to July maps
products = api.query(
footprint,
date=dates,
platformname='Sentinel-2',
# producttype='S2MSI2A',
area_relation='Intersects', # area of interest is inside footprint
cloudcoverpercentage=(0, 40))
# download all results from the search
api.download_all(products, directory_path=download_dir)
# product_id = list(products.keys())[0]
# api.download(id=product_id, directory_path=download_dir)
# GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries
return api.to_geodataframe(products)
def test_download_all(tmpdir):
api = SentinelAPI(environ['SENTINEL_USER'], environ['SENTINEL_PASSWORD'])
# From https://scihub.copernicus.eu/apihub/odata/v1/Products?$top=5&$orderby=ContentLength
filenames = [
"S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E",
"S1A_WV_OCN__2SSV_20150526T211029_20150526T211737_006097_007E78_134A",
"S1A_WV_OCN__2SSV_20150526T081641_20150526T082418_006090_007E3E_104C"
]
api.query_raw(" OR ".join(filenames))
assert len(api.get_products()) == len(filenames)
# Download normally
result = api.download_all(str(tmpdir))
assert len(result) == len(filenames)
for path, product_info in result.items():
pypath = py.path.local(path)
assert pypath.purebasename in filenames
assert pypath.check(exists=1, file=1)
assert pypath.size() == product_info["size"]
# Force one download to fail
path, product_info = list(result.items())[0]
py.path.local(path).remove()
with requests_mock.mock(real_http=True) as rqst:
url = "https://scihub.copernicus.eu/apihub/odata/v1/Products('%s')/?$format=json" % product_info[
"id"]
json = api.session.get(url).json()
json["d"]["Checksum"]["Value"] = "00000000000000000000000000000000"
rqst.get(url, json=json)
result = api.download_all(str(tmpdir), max_attempts=1, checksum=True)
assert len(result) == len(filenames)
assert result[path] is None
def test_download_all(tmpdir):
api = SentinelAPI(**_api_auth)
# From https://scihub.copernicus.eu/apihub/odata/v1/Products?$top=5&$orderby=ContentLength
filenames = ["S1A_WV_OCN__2SSH_20150603T092625_20150603T093332_006207_008194_521E",
"S1A_WV_OCN__2SSV_20150526T211029_20150526T211737_006097_007E78_134A",
"S1A_WV_OCN__2SSV_20150526T081641_20150526T082418_006090_007E3E_104C"]
api.load_query(" OR ".join(filenames))
assert len(api.get_products()) == len(filenames)
# Download normally
result = api.download_all(str(tmpdir))
assert len(result) == len(filenames)
for path, product_info in result.items():
pypath = py.path.local(path)
assert pypath.purebasename in filenames
assert pypath.check(exists=1, file=1)
assert pypath.size() == product_info["size"]
# Force one download to fail
path, product_info = list(result.items())[0]
py.path.local(path).remove()
with requests_mock.mock(real_http=True) as rqst:
url = "https://scihub.copernicus.eu/apihub/odata/v1/Products('%s')/?$format=json" % product_info["id"]
json = api.session.get(url).json()
json["d"]["Checksum"]["Value"] = "00000000000000000000000000000000"
rqst.get(url, json=json)
result = api.download_all(str(tmpdir), max_attempts=1, checksum=True)
assert len(result) == len(filenames)
assert result[path] is None
def sentinel1_tile_download(file,username,password,tilepath):
api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
td = timedelta(days=60)
endDate = datetime.now()
startDate = endDate - td
footprint = geojson_to_wkt(read_geojson(file))
print(footprint)
#products = api.query(footprint,
# date=(startDate, endDate),platformname='Sentinel-1')
products = api.query(footprint,
producttype='SLC',
orbitdirection='ASCENDING')
# download all results from the search
api.download_all(products,directorypath=tilepath)
return
def download_scenes(period):
date_from = period.date_from
date_to = period.date_to
# Check if result has already been done
scene_filename = 's1_{dfrom}_{dto}.tif'.format(
dfrom=period.date_from.strftime('%Y%m'),
dto=period.date_to.strftime('%Y%m'))
scene_path = os.path.join(RESULTS_PATH, scene_filename)
if os.path.exists(scene_path):
print(
"Sentinel-1 mosaic for period {}-{} already done:".format(
date_from, date_to), scene_path)
return
# Prepare API client for download
api = SentinelAPI(settings.SCIHUB_USER, settings.SCIHUB_PASS,
settings.SCIHUB_URL)
# Query scenes
footprint = geojson_to_wkt(read_geojson(AOI_PATH))
products = api.query(footprint,
date=(date_from, date_to),
platformname='Sentinel-1',
producttype='GRD',
polarisationmode='VV VH',
orbitdirection='ASCENDING')
for k, p in products.items():
print((k, p['summary']))
os.makedirs(S1_RAW_PATH, exist_ok=True)
# Filter already downloaded products
products_to_download = {
k: v
for k, v in products.items() if not os.path.exists(
os.path.join(S1_RAW_PATH, '{}.zip'.format(v['title'])))
}
# Download products
results = api.download_all(products_to_download,
directory_path=S1_RAW_PATH)
products = list(products.values())
# Process the images of each product
with mp.Pool(settings.S1_PROC_NUM_JOBS) as pool:
pool.map(process_product, products)
# Create a median composite from all images of each band, generate extra
# bands and concatenate results into a single multiband imafge.
superimpose(products)
median(products, period)
generate_vvvh(period)
concatenate_results(period)
clip_result(period)
clean_temp_files(period)
def s5down(datum=str(dt.date.today()), product_type='no2'):
# this function will download s5p data of given product_type for given date
# e.g. s5down('2019-08-15','no2')
if product_type == 'no2':
strtype = 'L2__NO2___'
if product_type == 'aerosols':
strtype = 'L2__AER_AI'
datum = dt.datetime.strptime(datum, '%Y-%m-%d').date()
time_in = dt.datetime.combine(datum, dt.time(0, 0))
time_out = dt.datetime.combine(datum, dt.time(23, 59))
api = SentinelAPI('s5pguest', 's5pguest',
'https://s5phub.copernicus.eu/dhus')
#coordinates for CZ:
footprint = 'POLYGON((12.278971773041526 48.69059060056844,18.98957262575027 48.69059060056844,18.98957262575027 51.081759060281655,12.278971773041526 51.081759060281655,12.278971773041526 48.69059060056844))'
products = api.query(footprint,
date=(time_in, time_out),
platformname='Sentinel-5',
producttype=strtype)
print('there are ' + str(len(products)) + ' products found')
a = api.download_all(products)
geotiffs = []
for product_ID in iter(products):
product_path = a[0][product_ID]['path']
print('converting ' + product_path + ' to geotiff')
geotiffs.append(s5p2geotiff(product_path, product_type))
if not geotiffs:
print('some error happened, no geotiffs generated')
clean_downloaded(products, a)
return None
tifstring = ''
for tif in geotiffs:
tifstring = tifstring + ' ' + tif
print('merging geotiffs to ' + str(datum) +
'.tif and cropping for CZ extents')
outfile = str(datum) + '.' + product_type + '.tif'
tmpfile = 'tmp.tif'
os.system(
'gdal_merge.py -o merged.tif -of GTiff -ul_lr 11.3867 51.4847 19.943 47.7933 -a_nodata 9999 '
+ tifstring)
if product_type == 'no2':
#need to compute 1000x
gdal_calc = 'gdal_calc.py -A merged.tif --outfile=' + tmpfile + ' --calc="(A*1000 > 0)*(A * 1000 < 0.7)*(A * 1000)" --overwrite'
print(gdal_calc)
os.system(gdal_calc)
else:
tmpfile = 'merged.tif'
#now oversample using cubic..
gdalwarp = 'gdalwarp -tr 0.015 0.015 -r cubicspline -dstnodata 9999 -srcnodata 9999 ' + tmpfile + ' ' + outfile
#gdalwarp = 'gdalwarp -s_srs EPSG:4326 -t_srs EPSG:4326 -tr 0.015 0.015 -r cubicspline -dstnodata 9999 -srcnodata 9999 temp1000.tif '+outfile
print(gdalwarp)
os.system(gdalwarp)
print('(the file will be also saved as {}.tif)'.format(product_type))
copyfile(outfile, '../data/' + product_type + '.tif')
#cleaning
clean_downloaded(products, a)
return geotiffs
def search(
user, password, tile, geojson, start, end, download, md5,
sentinel1, sentinel2, cloud, footprints, path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if cloud:
search_kwargs.update(
{"platformname": "Sentinel-2",
"cloudcoverpercentage": "[0 TO %s]" % cloud})
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
if tile:
api.query(point = get_coordinates(tile = tile), initial_date = start, end_date = end, **search_kwargs)
elif geojson:
api.query(area = get_coordinates(geojson_file = geojson), initial_date = start, end_date = end, **search_kwargs)
else:
raise ValueError("Either a --geojson or --tile arguments must be given.")
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
result = api.download_all(path, checksum=md5)
if md5 is True:
corrupt_scenes = [(path, info["id"]) for path, info in result.items() if info is not None]
if len(corrupt_scenes) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for corrupt_tuple in corrupt_scenes:
outfile.write("%s : %s\n" % corrupt_tuple)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
print('---')
print(
'%s scenes found with a total size of %.2f GB' %
(len(api.get_products()), api.get_products_size()))
def download_sentinel(platform, prod_type, scihub_user, scihub_pass, start_date, end_date, region=None, filename=None, down_dir=None):
from sentinelsat.sentinel import SentinelAPI
import os
# change the working directory to the location of files
if down_dir!=None:
os.chdir(down_dir)
print(region)
# connect to the API
api = SentinelAPI(scihub_user, scihub_pass, 'https://scihub.copernicus.eu/dhus')
# search by polygon, time, and Hub query keywords
if region is not None and filename is not None:
products = api.query(region, date = (start_date, end_date), filename = filename, producttype = prod_type, platformname = platform)
elif region is not None:
products = api.query(region, date = (start_date, end_date), producttype = prod_type, platformname = platform)
elif filename is not None:
products = api.query(date = (start_date, end_date), filename = filename, producttype = prod_type, platformname = platform)
else:
products = api.query(date = (start_date, end_date), producttype = prod_type, platformname = platform)
# download all results from the search
print("Files will be downloaded to {}".format(os.getcwd()))
api.download_all(products)
def search(user, password, geojson, start, end, download, footprints, path, query):
"""Search for Sentinel-1 products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password)
if query is not None:
query = dict([i.split('=') for i in query.split(',')])
api.query(get_coordinates(geojson), start, end, **query)
else:
api.query(get_coordinates(geojson), start, end)
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
api.download_all(path)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
def search(
user, password, geojson, start, end, download, md5,
sentinel1, sentinel2, cloud, footprints, path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if cloud:
search_kwargs.update(
{"platformname": "Sentinel-2",
"cloudcoverpercentage": "[0 TO %s]" % cloud})
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
api.query(get_coordinates(geojson), start, end, **search_kwargs)
if footprints is True:
footprints_geojson = api.get_footprints()
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
result = api.download_all(path, checksum=md5)
if md5 is True:
corrupt_scenes = [(path, info["id"]) for path, info in result.items() if info is not None]
if len(corrupt_scenes) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for corrupt_tuple in corrupt_scenes:
outfile.write("%s : %s\n" % corrupt_tuple)
else:
for product in api.get_products():
print('Product %s - %s' % (product['id'], product['summary']))
print('---')
print(
'%s scenes found with a total size of %.2f GB' %
(len(api.get_products()), api.get_products_size()))
args = parser.parse_args()
print("loading id...")
data = json.load(open(args.data))
startdate = date(data["startdate"][0], data["startdate"][1], data["startdate"][2])
enddate = date(data["enddate"][0], data["enddate"][1], data["enddate"][2])
print("connecting to sentinel API...")
api = SentinelAPI(data["login"], data["password"], 'https://scihub.copernicus.eu/dhus')
# search by polygon, time, and SciHub query keywords
print("searching...")
footprint = geojson_to_wkt(read_geojson(args.geojson))
if args.sentinel == 1:
products = api.query(footprint,
date=(startdate,enddate),
platformname = 'Sentinel-1',
producttype = "GRD"
)
elif args.sentinel == 2:
products = api.query(footprint,
date=(startdate,enddate),
platformname = 'Sentinel-2'
)
print(" product number: ",len(products))
# download all results from the search
print("downloading...")
api.download_all(products)
def search(
user, password, geojson, start, end, download, md5, sentinel, producttype,
instrument, sentinel1, sentinel2, cloud, footprints, path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if sentinel and not (producttype or instrument):
search_kwargs.update({"platformname": "Sentinel-" + sentinel})
if instrument and not producttype:
search_kwargs.update({"instrumentshortname": instrument})
if producttype:
search_kwargs.update({"producttype": producttype})
if cloud:
if sentinel not in ['2', '3']:
logger.error('Cloud cover is only supported for Sentinel 2 and 3.')
raise ValueError('Cloud cover is only supported for Sentinel 2 and 3.')
search_kwargs.update({"cloudcoverpercentage": "[0 TO %s]" % cloud})
# DEPRECATED: to be removed with next major release
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
logger.info('DEPRECATED: Please use --sentinel instead')
# DEPRECATED: to be removed with next major release
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
logger.info('DEPRECATED: Please use --sentinel instead')
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
wkt = geojson_to_wkt(read_geojson(geojson))
products = api.query(wkt, start, end, **search_kwargs)
if footprints is True:
footprints_geojson = api.to_geojson(products)
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
product_infos, failed_downloads = api.download_all(products, path, checksum=md5)
if md5 is True:
if len(failed_downloads) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for failed_id in failed_downloads:
outfile.write("%s : %s\n" % (failed_id, products[failed_id]['title']))
else:
for product_id, props in products.items():
logger.info('Product %s - %s' % (product_id, props['summary']))
logger.info('---')
logger.info(
'%s scenes found with a total size of %.2f GB' %
(len(products), api.get_products_size(products)))
def download_scenes(period):
date_from = period.date_from
date_to = period.date_to
period_s = '{dfrom}_{dto}'.format(dfrom=period.date_from.strftime("%Y%m"),
dto=period.date_to.strftime("%Y%m"))
# Check if result has already been done
scene_filename = f's2_{period_s}*.tif'
scene_path = os.path.join(RESULTS_PATH, scene_filename)
if len(glob(scene_path)) == 2:
print(
"Sentinel-2 mosaic for period {}-{} already done:".format(
date_from, date_to), scene_path)
return
if not settings.SCIHUB_USER or not settings.SCIHUB_PASS:
raise "SCIHUB_USER and/or SCIHUB_PASS are not set. " + \
"Please read the Configuration section on README."
api = SentinelAPI(settings.SCIHUB_USER, settings.SCIHUB_PASS,
settings.SCIHUB_URL)
# Search by polygon, time, and Hub query keywords
footprint = geojson_to_wkt(
read_geojson(os.path.join(APPDIR, 'data', 'extent.geojson')))
products = api.query(footprint,
date=(date_from, date_to),
platformname='Sentinel-2',
cloudcoverpercentage=(0, 20))
# Skip L2A products
l2 = []
for p in products:
if 'MSIL2A' in products[p]['title']:
l2.append(p)
for p in l2:
products.pop(p)
for p in products:
print(products[p]['title'])
# Filter already downloaded products
l1c_path = os.path.join(S2_L1C_PATH, period_s)
os.makedirs(l1c_path, exist_ok=True)
products_to_download = {
k: v
for k, v in products.items() if
not os.path.exists(os.path.join(l1c_path, '{}.zip'.format(v['title'])))
}
# Download products
api.download_all(products_to_download, directory_path=l1c_path)
products = list(products.values())
# Unzip
for p in products:
unzip_product(p, period_s)
# Get the list of L1C products still to be processed to L2A
l2a_path = os.path.join(S2_L2A_PATH, period_s)
os.makedirs(l2a_path, exist_ok=True)
l1c_can_prods = get_canonical_names(glob(os.path.join(l1c_path, '*.SAFE')))
l2a_can_prods = get_canonical_names(glob(os.path.join(l2a_path, '*.SAFE')))
missing_l1c_prods = [
l1c_can_prods[k]
for k in set(l1c_can_prods.keys()) - set(l2a_can_prods.keys())
]
# Run s2m preprocess (sen2cor) on raw directory
for p in missing_l1c_prods:
sen2_preprocess(p, period_s)
# Build mosaic
mosaic_path = os.path.join(settings.IMAGES_PATH, 'mosaic', period_s)
os.makedirs(mosaic_path, exist_ok=True)
xmin, ymin, xmax, ymax = [
260572.3994411753083114, 8620358.0515629947185516,
324439.4877797830849886, 8720597.2414500378072262
]
mosaic_name = 's2_{}{}_{}{}_mosaic'.format(date_from.year, date_from.month,
date_to.year, date_to.month)
for res in [10, 20]:
cmd = "python3 {}/mosaic.py -te {} {} {} {} -e 32718 -res {} -n {} -v -o {} {}".format(
settings.S2M_CLI_PATH, xmin, ymin, xmax, ymax, res, mosaic_name,
mosaic_path, l2a_path)
rv = os.system(cmd)
if rv != 0:
raise ValueError('s2m mosaic failed')
generate_vegetation_indexes(mosaic_name, period_s)
concatenate_results(mosaic_name, period_s)
clip_results(period_s)
clean_temp_files(period_s)
def cli(
user,
password,
geometry,
start,
end,
uuid,
name,
download,
sentinel,
producttype,
instrument,
cloud,
footprints,
path,
query,
url,
order_by,
limit,
):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your Copernicus Open Access Hub user and password, you must pass a geojson file
containing the geometry of the area you want to search for or the UUIDs of the products. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
_set_logger_handler()
if user is None or password is None:
try:
user, password = requests.utils.get_netrc_auth(url)
except TypeError:
pass
if user is None or password is None:
raise click.UsageError(
"Missing --user and --password. Please see docs "
"for environment variables and .netrc support.")
api = SentinelAPI(user, password, url)
search_kwargs = {}
if sentinel and not (producttype or instrument):
search_kwargs["platformname"] = "Sentinel-" + sentinel
if instrument and not producttype:
search_kwargs["instrumentshortname"] = instrument
if producttype:
search_kwargs["producttype"] = producttype
if cloud:
if sentinel not in ["2", "3"]:
logger.error("Cloud cover is only supported for Sentinel 2 and 3.")
exit(1)
search_kwargs["cloudcoverpercentage"] = (0, cloud)
if query is not None:
search_kwargs.update((x.split("=") for x in query))
if geometry is not None:
search_kwargs["area"] = geojson_to_wkt(read_geojson(geometry))
if uuid is not None:
uuid_list = [x.strip() for x in uuid]
products = {}
for productid in uuid_list:
try:
products[productid] = api.get_product_odata(productid)
except SentinelAPIError as e:
if "Invalid key" in e.msg:
logger.error("No product with ID '%s' exists on server",
productid)
exit(1)
else:
raise
elif name is not None:
search_kwargs["identifier"] = name[0] if len(
name) == 1 else "(" + " OR ".join(name) + ")"
products = api.query(order_by=order_by, limit=limit, **search_kwargs)
else:
start = start or "19000101"
end = end or "NOW"
products = api.query(date=(start, end),
order_by=order_by,
limit=limit,
**search_kwargs)
if footprints is True:
footprints_geojson = api.to_geojson(products)
if os.path.isdir(footprints):
foot_path = os.path.join(footprints, "search_footprints.geojson")
else:
foot_path = "search_footprints.geojson"
if path == ".":
dump_path = os.path.join(os.getcwd(), foot_path)
else:
dump_path = os.path.join(path, foot_path)
with open(dump_path, "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
product_infos, triggered, failed_downloads = api.download_all(
products, path)
if len(failed_downloads) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"),
"w") as outfile:
for failed_id in failed_downloads:
outfile.write("%s : %s\n" %
(failed_id, products[failed_id]["title"]))
else:
for product_id, props in products.items():
if uuid is None:
logger.info("Product %s - %s", product_id, props["summary"])
else: # querying uuids has no summary key
logger.info(
"Product %s - %s - %s MB",
product_id,
props["title"],
round(int(props["size"]) / (1024.0 * 1024.0), 2),
)
if uuid is None:
logger.info("---")
logger.info(
"%s scenes found with a total size of %.2f GB",
len(products),
api.get_products_size(products),
)
def download(product, credentials):
api = SentinelAPI(credentials[0], credentials[1], 'https://scihub.copernicus.eu/dhus')
api.download_all(product)
for elem in product:
os.system('unzip ' + product[elem]['title'] + '.zip')
os.remove(product[elem]['title'] + '.zip')
def search(user, password, geojson, start, end, download, md5, sentinel,
producttype, instrument, sentinel1, sentinel2, cloud, footprints,
path, query, url):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your SciHub user and password, you must pass a geojson file
containing the polygon of the area you want to search for. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
api = SentinelAPI(user, password, url)
search_kwargs = {}
if sentinel and not (producttype or instrument):
search_kwargs.update({"platformname": "Sentinel-" + sentinel})
if instrument and not producttype:
search_kwargs.update({"instrumentshortname": instrument})
if producttype:
search_kwargs.update({"producttype": producttype})
if cloud:
if sentinel not in ['2', '3']:
logger.error('Cloud cover is only supported for Sentinel 2 and 3.')
raise ValueError(
'Cloud cover is only supported for Sentinel 2 and 3.')
search_kwargs.update({"cloudcoverpercentage": "[0 TO %s]" % cloud})
# DEPRECATED: to be removed with next major release
elif sentinel2:
search_kwargs.update({"platformname": "Sentinel-2"})
logger.info('DEPRECATED: Please use --sentinel instead')
# DEPRECATED: to be removed with next major release
elif sentinel1:
search_kwargs.update({"platformname": "Sentinel-1"})
logger.info('DEPRECATED: Please use --sentinel instead')
if query is not None:
search_kwargs.update(dict([i.split('=') for i in query.split(',')]))
wkt = geojson_to_wkt(read_geojson(geojson))
products = api.query(wkt, start, end, **search_kwargs)
if footprints is True:
footprints_geojson = api.to_geojson(products)
with open(os.path.join(path, "search_footprints.geojson"),
"w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
product_infos, failed_downloads = api.download_all(products,
path,
checksum=md5)
if md5 is True:
if len(failed_downloads) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"),
"w") as outfile:
for failed_id in failed_downloads:
outfile.write(
"%s : %s\n" %
(failed_id, products[failed_id]['title']))
else:
for product_id, props in products.items():
logger.info('Product %s - %s' % (product_id, props['summary']))
logger.info('---')
logger.info('%s scenes found with a total size of %.2f GB' %
(len(products), api.get_products_size(products)))
def sen2_download(products, conf):
# TODO: Specify download location, file resuming.
api = SentinelAPI(conf["sen2"]["user"], conf["sen2"]["pass"],
'https://scihub.copernicus.eu/dhus')
api.download_all(products, conf["data"]["out_folder"])
class S1Download(object):
"""
This module makes searching, downloading and retrieving metadata of Sentinel-1 satellite images from the
Copernicus Open Access Hub easy.
Parameters
----------
username : str
Username for Copernicus Open Access Hub
password : str
Password for Copernicus Open Access Hub
region : str
A geojson file.
timestart : str
Start time like "YYYY-MM-DD"
timeend : str
End time like "YYYY-MM-DD".
outdir : str
Output directory.
producttype : {'SLC', 'GRD', 'OCN', ''RAW}
Product type. If None, all types will be recognized.
polarisationmode tuple or str
A combination of V and H like ('VH', 'HV') or simple 'VH'.
sensoroperationalmode : {'SM', 'IW', 'EW', 'WV'}
Sensor operational mode. If None, all types will be recognized.
orbitnumber : int
Orbit number
orbitdirection : {DESCENDING, ASCENDING}
Orbit direction. If None, all types will be recognized.
Attributes
----------
api : object
Sentinelsat API object.
outdir : str
region : wkt
A geojson to WKT object.
kwargs : dict
Dictionary with setted attributes.
files : DataFrame
Pandas DataFrame with detected files.
Methods
-------
download()
Download all files.
print_products()
Print all detected files.
Examples
--------
The general usage is
::
$ ds1.download [-p] username=string password=string region=string timestart=string timeend=string outdir=sting
[*attributes=string] [--verbose] [--quiet]
For *attributes the following parameters can be used
::
>>> ["producttype", "polarisationmode", "sensoroperationalmode", "orbitnumber", "orbitdirection"]
Print all Sentinel 1 data with product type GRD between 2015-01-02 and 2015-01-12::
$ ds1.download -p username=USER password=PASSWORD region=myGEoJsOnFile.geojson timestart=2015-01-02
timeend=2015-01-12 outdir='home/usr/data' producttype=SLC
Download the last query
::
$ ds1.download username=USER password=PASSWORD region=myGEoJsOnFile.geojson timestart=2015-01-02
timeend=2015-01-12 outdir='home/usr/data' producttype=SLC
Notes
-----
**Flags:**
* p : Print the detected files and exit.
"""
def __init__(self,
username,
password,
region,
timestart,
timeend,
outdir,
producttype=None,
polarisationmode=None,
sensoroperationalmode=None,
orbitnumber=None,
orbitdirection=None):
# Inititalise Sentinel Python API ------------------------------------------------------------------------------
self.api = SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus')
# Initialize Directory -----------------------------------------------------------------------------------------
if not os.path.exists(outdir):
os.makedirs(outdir)
else:
self.outdir = outdir
# Initialise Mandatory Parameter ------------------------------------------------------------------------------
self.region = geojson_to_wkt(read_geojson(region))
# < Reformat Time > ------------
timestart_split = timestart.split('-')
timestart_temp = ''
for item in timestart_split:
timestart_temp += item
timeend_split = timeend.split('-')
timeend_temp = ''
for item in timeend_split:
timeend_temp += item
self.date = (timestart_temp, timeend_temp)
# Initialize Attributes ----------------------------------------------------------------------------------------
input_parameter = [
producttype, polarisationmode, sensoroperationalmode, orbitnumber,
orbitdirection
]
__KEYS__ = [
"producttype", "polarisationmode", "sensoroperationalmode",
"orbitnumber", "orbitdirection"
]
self.kwargs = {}
for i, item in enumerate(input_parameter):
if item is not None:
self.kwargs[__KEYS__[i]] = item
# Inititalise Sentinel Producs with API ------------------------------------------------------------------------
self.products = self.api.query(self.region,
date=self.date,
platformname='Sentinel-1',
**self.kwargs)
self.files = self.api.to_dataframe(self.products)
def download(self):
self.api.download_all(self.products, directory_path=self.outdir)
return 0
def print_products(self):
"""
Print all detected files.
Returns
-------
None
"""
df = self.files.to_string()
sys.stdout.write(df)
def cli(
user,
password,
geometry,
start,
end,
uuid,
name,
download,
quicklook,
sentinel,
producttype,
instrument,
cloud,
footprints,
path,
query,
url,
order_by,
location,
limit,
info,
):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your Copernicus Open Access Hub user and password, you must pass a geojson file
containing the geometry of the area you want to search for or the UUIDs of the products. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
_set_logger_handler()
if user is None or password is None:
try:
user, password = requests.utils.get_netrc_auth(url)
except TypeError:
pass
if user is None or password is None:
raise click.UsageError(
"Missing --user and --password. Please see docs "
"for environment variables and .netrc support.")
api = SentinelAPI(user, password, url)
if info:
ctx = click.get_current_context()
click.echo("DHuS version: " + api.dhus_version)
ctx.exit()
search_kwargs = {}
if sentinel and not (producttype or instrument):
search_kwargs["platformname"] = "Sentinel-" + sentinel
if instrument and not producttype:
search_kwargs["instrumentshortname"] = instrument
if producttype:
search_kwargs["producttype"] = producttype
if cloud:
if sentinel not in ["2", "3"]:
logger.error("Cloud cover is only supported for Sentinel 2 and 3.")
exit(1)
search_kwargs["cloudcoverpercentage"] = (0, cloud)
if query is not None:
search_kwargs.update((x.split("=") for x in query))
if location is not None:
wkt, info = placename_to_wkt(location)
minX, minY, maxX, maxY = info["bbox"]
r = 6371 # average radius, km
extent_east = r * math.radians(maxX - minX) * math.cos(
math.radians((minY + maxY) / 2))
extent_north = r * math.radians(maxY - minY)
logger.info(
"Querying location: '%s' with %.1f x %.1f km, %f, %f to %f, %f bounding box",
info["display_name"],
extent_north,
extent_east,
minY,
minX,
maxY,
maxX,
)
search_kwargs["area"] = wkt
if geometry is not None:
# check if the value is an existing path
if os.path.exists(geometry):
search_kwargs["area"] = geojson_to_wkt(read_geojson(geometry))
# check if the value is a GeoJSON
else:
if geometry.startswith("{"):
try:
geometry = json.loads(geometry)
search_kwargs["area"] = geojson_to_wkt(geometry)
except json_parse_exception:
raise click.UsageError(
"geometry string starts with '{' but is not a valid GeoJSON."
)
# check if the value is a WKT
elif is_wkt(geometry):
search_kwargs["area"] = geometry
else:
raise click.UsageError(
"The geometry input is neither a GeoJSON file with a valid path, "
"a GeoJSON String nor a WKT string.")
if uuid is not None:
uuid_list = [x.strip() for x in uuid]
products = {}
for productid in uuid_list:
try:
products[productid] = api.get_product_odata(productid)
except InvalidKeyError:
logger.error("No product with ID '%s' exists on server",
productid)
exit(1)
elif name is not None:
search_kwargs["identifier"] = name[0] if len(
name) == 1 else "(" + " OR ".join(name) + ")"
products = api.query(order_by=order_by, limit=limit, **search_kwargs)
else:
start = start or "19000101"
end = end or "NOW"
products = api.query(date=(start, end),
order_by=order_by,
limit=limit,
**search_kwargs)
if footprints is True:
footprints_geojson = api.to_geojson(products)
with open(os.path.join(path, "search_footprints.geojson"),
"w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if quicklook:
downloaded_quicklooks, failed_quicklooks = api.download_all_quicklooks(
products, path)
if failed_quicklooks:
api.logger.warning("Some quicklooks failed: %s out of %s",
len(failed_quicklooks), len(products))
if download is True:
product_infos, triggered, failed_downloads = api.download_all(
products, path)
if len(failed_downloads) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"),
"w") as outfile:
for failed_id in failed_downloads:
outfile.write("%s : %s\n" %
(failed_id, products[failed_id]["title"]))
else:
for product_id, props in products.items():
if uuid is None:
logger.info("Product %s - %s", product_id, props["summary"])
else: # querying uuids has no summary key
logger.info(
"Product %s - %s - %s MB",
product_id,
props["title"],
round(int(props["size"]) / (1024.0 * 1024.0), 2),
)
if uuid is None:
logger.info("---")
logger.info(
"%s scenes found with a total size of %.2f GB",
len(products),
api.get_products_size(products),
)
class Processor():
def __init__(self,
sentinel_user,
sentinel_pass,
start_date,
end_date,
dl_dir,
input_file,
debug=False):
self.SENTINEL_USER = sentinel_user
self.SENTINEL_PASS = sentinel_pass
self.DL_DIR = dl_dir
self.INPUT_FILE = input_file
self.START_DATE = start_date
self.END_DATE = end_date
self.DEBUG = debug
if not os.path.exists(self.DL_DIR):
os.mkdir(self.DL_DIR)
def phase_1(self):
self.api = SentinelAPI(self.SENTINEL_USER, self.SENTINEL_PASS)
self.aoi_footprint = geojson_to_wkt(read_geojson(self.INPUT_FILE))
def phase_2(self):
self.api_products = self.api.query(
self.aoi_footprint,
date=(self.START_DATE, self.END_DATE),
area_relation='Intersects',
platformname='Sentinel-2',
cloudcoverpercentage=(0, 30),
)
def phase_3(self):
"""
We're doing the conversion from a GeoDataFrame to a list of dictionaries.
After the conversion we intend to use the "footprint" and the "index" columns.
This step is required because there are multiple products with the same footprint
and later on we need the index in order to download the images from SentinelAPI.
"""
self.product_df = self.api.to_dataframe(self.api_products)
if len(self.product_df.index) == 0:
raise Exception("No images for selected period")
self.product_df = self.product_df.sort_values(
['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True])
self.tile_footprints = []
for x in self.product_df[[
"size", "tileid", "processinglevel", "footprint"
]].T.to_dict().items():
self.tile_footprints.append({**x[1], "index": x[0]})
if self.DEBUG:
pprint(self.tile_footprints[:3])
def phase_4(self):
L1 = min_cover_1(self.tile_footprints)
if self.DEBUG:
print("{} tiles after the 1st reduction".format(len(L1)))
L2 = min_cover_2(L1)
if self.DEBUG:
print("{} tiles after the 2nd reduction".format(len(L2)))
self.reduced_footprints = L2
def phase_5(self):
dl_indexes = [x["index"] for x in self.reduced_footprints]
self.api.download_all(dl_indexes, directory_path=self.DL_DIR)
if self.DEBUG:
pprint(dl_indexes)
def phase_6(self):
"""
We're decompressing the archives unless they're already decompressed.
"""
for p in pathlib.Path(self.DL_DIR).iterdir():
p_dir = re.sub('\.zip$', '.SAFE', str(p))
if os.path.isfile(p) and not os.path.exists(p_dir):
extract_path = os.path.dirname(p)
print("Dezarhivare " + str(p))
with zipfile.ZipFile(p, 'r') as zip_ref:
zip_ref.extractall(extract_path)
def phase_7(self):
"""
Converting the .jp2 images to .tiff
"""
def select_files(path, pattern):
L = []
for root, dirs, files in os.walk(path):
if len(dirs) == 0:
for f in files:
if re.match(pattern, f):
L.append(os.path.join(root, f))
return L
def convert_to_tiff(paths):
tiff_paths = []
for p in paths:
print("Converting " + p)
with rasterio.open(p, mode="r") as src:
profile = src.meta.copy()
profile.update(driver="GTiff")
outfile = re.sub(".jp2", ".tiff", p)
with rasterio.open(outfile, 'w', **profile) as dst:
dst.write(src.read())
tiff_paths.append(outfile)
return tiff_paths
self.jp2_paths = select_files(self.DL_DIR, ".*_TCI.jp2$")
self.tiff_paths = convert_to_tiff(self.jp2_paths)
def phase_8(self):
"""
We're mergin the raster images.
"""
raster_list = [
rasterio.open(f, mode='r', driver="GTiff") for f in self.tiff_paths
]
merged_data, out_trans = rasterio.merge.merge(raster_list)
if self.DEBUG:
fig, ax = plt.subplots(figsize=(14, 14))
show(merged_data, cmap='terrain', ax=ax)
merged_meta = raster_list[0].meta.copy()
merged_meta.update({
"driver": "GTiff",
"height": merged_data.shape[1],
"width": merged_data.shape[2],
"transform": out_trans,
"crs": raster_list[0].crs,
"count": 3,
})
if self.DEBUG:
for x in [x.meta for x in raster_list] + [merged_meta]:
pprint(x)
self.MERGED_RAW = os.path.join(self.DL_DIR, "merged1.tiff")
with rasterio.open(self.MERGED_RAW, mode="w", **merged_meta) as dest:
dest.write(merged_data)
def phase_9(self):
"""
Reprojecting the images to EPSG:4326
"""
dst_crs = 'EPSG:4326'
with rasterio.open(self.MERGED_RAW) as src:
transform, width, height = calculate_default_transform(
src.crs, dst_crs, src.width, src.height, *src.bounds)
kwargs = src.meta.copy()
kwargs.update({
'crs': dst_crs,
'transform': transform,
'width': width,
'height': height
})
self.MERGED_4326 = os.path.join(self.DL_DIR, "merged1_4326.tiff")
with rasterio.open(self.MERGED_4326, mode="w", **kwargs) as dst:
for i in range(1, src.count + 1):
reproject(source=rasterio.band(src, i),
destination=rasterio.band(dst, i),
src_transform=src.transform,
src_crs=src.crs,
dst_transform=transform,
dst_crs=dst_crs,
resampling=Resampling.nearest)
def phase_10(self):
"""
We're clipping the area of interest.
"""
with rasterio.open(self.MERGED_4326) as src:
out_image, out_transform = rasterio.mask.mask(
src, [shapely.wkt.loads(self.aoi_footprint)], crop=True)
out_meta = src.meta
out_meta.update({
"driver": "GTiff",
"height": out_image.shape[1],
"width": out_image.shape[2],
"transform": out_transform,
})
self.MERGED_REGION = os.path.join(self.DL_DIR,
"merged1_region.tiff")
with rasterio.open(self.MERGED_REGION, "w", **out_meta) as dest:
dest.write(out_image)
if self.DEBUG:
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(14, 14))
from rasterio.plot import show
show(out_image, cmap='terrain', ax=ax)
def reset(self):
"""
We're resetting object state to allow for a subsequent run.
"""
self.aoi_footprint = None
self.api_products = None
self.product_df = None
self.tile_footprints = None
self.reduced_footprints = None
self.jp2_paths = None
self.tiff_paths = None
self.MERGED_RAW = None
self.MERGED_4326 = None
self.MERGED_REGION = None
end_date='20170730',
platformname='Sentinel-1',
producttype='SLC')
print(api._last_query)
print('%s product results for your query. The products need %s Gb disk space'
) % (len(products), api.get_products_size(products))
### convert to Pandas DataFrame
products_df = api.to_dataframe(products)
print(products_df.index.values)
### download all query products
path = '/media/nas_data/Thomas/Wallerfing/Sentinel_1_data'
result = api.download_all(products,
directory_path=path,
max_attempts=10,
checksum=True,
check_existing=True,
show_progress=False)
print('Downloaded files:')
print(result.viewkeys())
"""
Change and/or sort query results
"""
# ### sort and limit to first 5 sorted products
# products_df_sorted = products_df.sort_values(['ingestiondate', 'producttype'], ascending=[True, True])
# products_df_sorted = products_df_sorted.head(1)
# ### download (sorted and reduced) products in order
# path = '/media/tweiss/Daten'
# result={}
# for product_id in products_df_sorted["id"]:
# convert result to Pandas Dataframe
products_df = api.to_dataframe(products)
# create dataframe only for overview
df_overview = products_df[["title"]]
df_string = df_overview.to_string(index=False).split()[1:]
df_new = pd.DataFrame(df_string, columns=["Product"])
# make dates column
dates = df_new["Product"].str[11:19]
dt_series = pd.to_datetime(dates)
df_new["dates"] = dt_series
df_new.sort_values(by=["dates"], inplace=True)
# make tiles column
tiles = df_new["Product"].str[39:44]
df_new["tile"] = tiles
print(df_new)
# request confirmation for download of filtered scenes
proceed = input(
'Do you want to proceed with the download of the selected scenes? [y/n] ')
if proceed == 'y':
# path to folder where downloaded scenes will be stored defined by user on the command line
download_folder = sys.argv[2]
# actual download of filtered scenes
api.download_all(products, directory_path=download_folder)
else:
sys.exit()
s1footprints.to_csv(outfile)
###############################################
# download the selected scenes
###############################################
# make a 'data' directory (if it does not exist yet) to where the images will be downloaded
datadir = wd + "data/"
if not os.path.exists(datadir):
print("Creating directory: ", datadir)
os.mkdir(datadir)
# change to the 'data' directory
os.chdir(datadir)
# download sorted and reduced products in order
api.download_all(products_df_n['uuid'])
# save the footprints of the scenes marked for download together with their
# metadata in a Geojson file
# first, run a new query to get the metadata for the selected scenes
products_n = OrderedDict()
for uuid in products_df_n['uuid']:
kw = query_kwargs.copy()
kw['uuid'] = uuid
pp = api.query(**kw)
products_n.update(pp)
# then, write the footprints and metadata to a geojson file
os.chdir(wd) # change to the working directory
outfile = 'footprints.geojson'
with open(outfile, 'w') as f:
products = api.query(footprint,
date=(date(y, m, day), date(y, m, day + 1)),
producttype='S2MSI1C',
platformname='Sentinel-2')
print(type(products))
products_df = api.to_dataframe(products)
cond1 = len(products_df.index) == 2
cond2 = np.all(
products_df.loc[:,
'relativeorbitnumber'].values == np.asarray([ro, ro]))
if cond1 and cond2:
print('There are two scenes of relative Orbit 65 available:')
print(products_df)
print('Scenes will be downloaded...')
api.download_all(products, directory_path=out_dir)
else:
print('For date ' + str(d) +
' conditions are not met. Following scenes are available:')
print(products_df)
print('Will continue with next date...')
continue
# extract all downloaded data
l1cs = os.listdir(out_dir)
print(l1cs)
for s in l1cs:
file = os.path.join(out_dir, s)
with zipfile.ZipFile(file, 'r') as f:
f.extractall(path=out_dir)
print(s + ' was successfully extracted.')
def download_and_build_composite(date_from, date_to):
from eo_sensors.utils import clip, rescale_byte, unzip
from sentinelsat.sentinel import SentinelAPI, geojson_to_wkt, read_geojson
period_s = '{dfrom}_{dto}'.format(dfrom=date_from.strftime("%Y%m%d"),
dto=date_to.strftime("%Y%m%d"))
proc_scene_dir = os.path.join(PROC_DIR, period_s)
tci_path = os.path.join(proc_scene_dir, 'tci.tif')
if os.path.exists(tci_path):
logger.info("TCI file already generated at %s", tci_path)
return tci_path
if not settings.SCIHUB_USER or not settings.SCIHUB_PASS:
raise "SCIHUB_USER and/or SCIHUB_PASS are not set. " + \
"Please read the Configuration section on README."
api = SentinelAPI(settings.SCIHUB_USER, settings.SCIHUB_PASS,
settings.SCIHUB_URL)
extent = read_geojson(EXTENT_PATH)
footprint = geojson_to_wkt(extent)
logger.info(
"Query S2MSI2A products with up to %d%% cloud cover from %s to %s",
MAX_CLOUD_PERC, date_from, date_to)
products = api.query(footprint,
date=(date_from, date_to),
platformname='Sentinel-2',
cloudcoverpercentage=(0, MAX_CLOUD_PERC),
producttype='S2MSI2A')
logger.info("Found %d products", len(products))
raw_dir = os.path.join(RAW_DIR, period_s)
os.makedirs(raw_dir, exist_ok=True)
# Filter already downloaded products
products_to_download = {
k: v
for k, v in products.items()
if not (os.path.exists(
os.path.join(raw_dir, '{}.zip'.format(v['title']))) or os.path.
exists(os.path.join(raw_dir, '{}.SAFE'.format(v['title']))))
}
# Download products
if products_to_download:
logger.info("Download all products (%d)", len(products_to_download))
api.download_all(products_to_download, directory_path=raw_dir)
# Unzip compressed files, if there are any
for p in glob(os.path.join(raw_dir, '*.zip')):
name, _ = os.path.splitext(os.path.basename(p))
p_dir = os.path.join(raw_dir, f'{name}.SAFE')
if not os.path.exists(p_dir):
logger.info("Unzip %s", p)
unzip(p, delete_zip=False)
# Build mosaic
mosaic_dir = os.path.join(proc_scene_dir, 'mosaic')
os.makedirs(mosaic_dir, exist_ok=True)
# FIXME: Read bounds from EXTENT_UTM_PATH
xmin, ymin, xmax, ymax = [
261215.0000000000000000, 8620583.0000000000000000,
323691.8790999995544553, 8719912.0846999995410442
]
cmd = f"python3 {settings.S2M_CLI_PATH}/mosaic.py " \
f"-te {xmin} {ymin} {xmax} {ymax} " \
f"-e 32718 -res 10 -v " \
f"-p {settings.S2M_NUM_JOBS} " \
f"-o {mosaic_dir} {raw_dir}"
run_subprocess(cmd)
# Get mosaic band rasters
mosaic_rgb_paths = [
glob(os.path.join(mosaic_dir, f'*_{band}.tif'))
for band in ['B04', 'B03', 'B02']
]
mosaic_rgb_paths = [p[0] for p in mosaic_rgb_paths if p]
logger.info("RGB paths: %s", mosaic_rgb_paths)
# Use gdalbuildvrt to concatenate RGB bands from mosaic
vrt_path = os.path.join(mosaic_dir, 'tci.vrt')
cmd = f"gdalbuildvrt -separate {vrt_path} {' '.join(mosaic_rgb_paths)}"
run_subprocess(cmd)
# Clip to extent and rescale virtual raster
clipped_tci_path = os.path.join(mosaic_dir, 'tci.tif')
clip(src=vrt_path, dst=clipped_tci_path, aoi=EXTENT_UTM_PATH)
# Rescale image
rescale_byte(src=clipped_tci_path, dst=tci_path, in_range=(100, 3000))
return tci_path
'https://scihub.copernicus.eu/dhus')
# download single scene by known product id
#api.download(<product_id>)
# search by polygon, time, and Hub query keywords
footprint = geojson_to_wkt(read_geojson(aoi_path))
products = api.query(footprint,
date=fechas,
platformname='Sentinel-2',
cloudcoverpercentage=(0, 100))
print(products)
# download all results from the search
result = api.download_all(products)
print(result)
# GeoJSON FeatureCollection containing footprints and metadata of the scenes
#api.to_geojson(products)
# GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries
#api.to_geodataframe(products)
# Get basic information about the product: its title, file size, MD5 sum, date, footprint and
# its download url
#api.get_product_odata(<product_id>)
# Get the product's full metadata available on the server
#api.get_product_odata(<product_id>, full=True)
class Command(BaseCommand):
help = 'Download and process data'
def add_arguments(self, parser):
parser.description = """Download and process satellite data for given
time period."""
parser.description = """
User has to setup username and password for scihub.copernicus.eu
service.
Either --year and --week or --date has to be specified.
"""
parser.add_argument('--user',
required=True,
type=str,
help="Copernicus SciHub user name")
parser.add_argument('--password',
required=True,
type=str,
help="Copernicus SciHub user password")
parser.add_argument('--date',
required=False,
type=str,
help="""Any day within required week in format
YYYYMMDD""")
parser.add_argument('--week',
required=False,
type=str,
help="Number of week in required year")
parser.add_argument('--year', required=False, type=str)
parser.add_argument('--area',
required=True,
type=str,
help="Area name or id")
parser.add_argument('--clouds',
required=True,
type=float,
help="Cloud coverage")
def handle(self, *args, **options):
"""
main method
"""
user = options['user']
passw = options['password']
required_date = required_week = required_year = required_date = 0
if options['date']:
required_date = int(options['date'])
if options['week']:
required_week = int(options['week'])
if options['year']:
required_year = int(options['year'])
area_name = options['area']
clouds = options['clouds']
self.api = SentinelAPI(user, passw,
'https://scihub.copernicus.eu/dhus')
try:
self.area = Area.objects.get(id=int(area_name))
except:
try:
self.area = Area.objects.get(name=area_name)
except:
self.stdout.write(
self.style.ERROR(
'Given area <{}> does not exist'.format(area_name)))
sys.exit(1)
(starting_date, end_date,
week_nr) = self._get_dates(required_year, required_week,
required_date)
products = self.get_products(starting_date,
end_date,
self.area,
clouds=clouds)
if not len(products.items()):
# TODO save empty week maybe?
self.stdout.write(
self.style.WARNING(
'There is no data for given time period ' +
'<{start}, {end}>, '.format(start=starting_date,
end=end_date) +
'maximal cloud cover <{cloud}%> and area <{area}>'.format(
area=area_name, cloud=clouds)))
return
#self.tempdir = tempfile.mkdtemp(dir="/home/jachym/data/opengeolabs/lifemonitor/")
#!self.tempdir = "/home/jachym/data/opengeolabs/lifemonitor/tmpq8_15z8f/"
self.tempdir = tempfile.mkdtemp()
_TO_BE_CLEANED.append(self.tempdir)
self.api.download_all(products, self.tempdir)
products_data = self.get_bands(products)
patched_bands = self._patch_rasters(products_data)
analysed_data = self._analyse(patched_bands)
if Week.objects.filter(date=starting_date,
area=self.area).count() == 0:
week = Week(
date=starting_date,
area=self.area,
)
else:
week = Week.objects.get(date=starting_date, area=self.area)
week.cutline = self.cutline_geom.wkt
for band in patched_bands:
band_key = band.lower()
eval("week.{}".format(band_key)).save(
os.path.basename(patched_bands[band]),
File(open(patched_bands[band], "rb")),
save=True)
week.save()
for an in analysed_data:
at = AnalysisType.objects.get(name=an)
if Analysis.objects.filter(week=week, type=at).count() == 0:
analysis = Analysis(week=week, type=at)
else:
analysis = Analysis.objects.get(week=week, type=at)
if analysed_data[an]["raster"]:
analysis.raster.save(
os.path.basename(analysed_data[an]["raster"]),
File(open(analysed_data[an]["raster"], "rb")),
save=True)
if analysed_data[an]["image"]:
analysis.image.save(
os.path.basename(analysed_data[an]["image"]),
File(open(analysed_data[an]["image"], "rb")),
save=True)
if analysed_data[an]["vector"]:
analysis.vector.save(
os.path.basename(analysed_data[an]["vector"]),
File(open(analysed_data[an]["vector"], "rb")),
save=True)
analysis.save()
self.stdout.write(
self.style.SUCCESS('Successfully create data for week {}'.format(
week.week)))
def save_satellite_image(self, product):
kwargs = copy.deepcopy(product)
kwargs.pop("highprobacloudspercentage")
kwargs.pop("notvegetatedpercentage")
kwargs.pop("snowicepercentage")
kwargs.pop("unclassifiedpercentage")
kwargs.pop("vegetationpercentage")
kwargs.pop("waterpercentage")
kwargs.pop("gmlfootprint")
if kwargs["footprint"].find("POLYGON") == 0:
geom = shapely.wkt.loads(kwargs["footprint"])
geom = MultiPolygon([geom])
kwargs["footprint"] = geom.wkt
kwargs["frmt"] = kwargs.pop("format")
kwargs["cloudcoverpercentage"] = kwargs.pop(
"mediumprobacloudspercentage")
if SatelliteImage.objects.filter(title=kwargs["title"]).count() == 0:
img = SatelliteImage(**kwargs)
img.save()
else:
img = SatelliteImage.objects.get(title=kwargs["title"])
return img
def _analyse(self, bands):
"""Perform required analysis for given area
:param bands: dict will all available raster bands
:return: dict with resulting analysis each analysis should have raster,
image, vector keys
"""
data = {}
for analysis in ["ndvi", "ndwi"]: # self.area.characteristics:
mod_analysis = importlib.import_module(
"varanus.methods.{}".format(analysis))
data[analysis] = mod_analysis.analyse(bands, self.tempdir)
return data
def _patch_rasters(self, products_data):
"""Patch bands together from more products
:param products_data: dict {product: {band: file_name}}
:return: dict {band: file_name}
"""
target_dir = os.path.join(self.tempdir, self.area.name, "merged")
if not os.path.isdir(target_dir):
os.makedirs(target_dir)
products = products_data.keys()
data = {}
for band in products_data[list(products)[0]].keys():
input_files = []
for product in products_data:
input_files.append(products_data[product][band])
output = os.path.join(target_dir, "{}.tif".format(band))
merge_command = [
"-n", "0", "-a_nodata", "0", "-o", output, "-co",
"COMPRESS=DEFLATE"
] + input_files
gm.main(merge_command)
data[band] = output
return data
def _get_final_cutline(self, target, cutline, crs):
"""
Get final cutline based on input cutline and cloud mask
:param target: name of target working directory
:param cultine: geojson file name with input cutline
:param crs: EPSG:CODE cutline's coordinate reference system
:return: file name with resulting cutline
"""
granule = os.path.join(target, "GRANULE")
qi_data = os.path.join(granule, os.listdir(granule)[0], "QI_DATA")
clouds_file = os.path.join(qi_data, "MSK_CLOUDS_B00.gml")
clouds_vectors = []
clouds_ds = ogr.Open(clouds_file)
layer = clouds_ds.GetLayer()
if not layer:
return cutline
feature = layer.GetNextFeature()
while feature is not None:
geom = feature.GetGeometryRef()
json_geom = geom.ExportToJson()
cloud_vectors = shape(json.loads(json_geom))
feature = layer.GetNextFeature()
cloud_vectors = unary_union(cloud_vectors)
cloud_vectors = fiona.transform.transform_geom(crs, "EPSG:4326",
mapping(cloud_vectors))
cutline_features = []
with fiona.open(cutline) as cutline:
for f in cutline:
cutline_features.append(shape(f["geometry"]))
cutline_features = unary_union(cutline_features)
final_cutline = cutline_features.difference(shape(cloud_vectors))
if final_cutline.type == "Polygon":
final_cutline = MultiPolygon([final_cutline])
self.cutline_geom = final_cutline
target_file = "{}.geojson".format(os.path.join(target,
"cloud_cutline"))
with open(target_file, "w") as out:
data = {
"type": "FeatureCollection",
"features": [{
"geometry": mapping(final_cutline)
}]
}
json.dump(data, out)
return target_file
def _get_all_band_files(self, target):
"""Create list of all available band files for given product target
directory
resulting data structure:
```
{
"res10m": {
"B01": "/path/to/file.jp2",
"B02": ...
},
"res20m": {
...
},
...
}
```
:return: data structure
"""
data = {}
granule = os.path.join(target, "GRANULE")
granule_name = os.listdir(granule)[0]
resolutions = os.listdir(
os.path.join(granule, granule_name, "IMG_DATA"))
for res in resolutions:
data[res] = {}
images = os.listdir(
os.path.join(granule, granule_name, "IMG_DATA", res))
for image in images:
band_name = image.split("_")[2]
data[res][band_name] = os.path.join(granule, granule_name,
"IMG_DATA", res, image)
return data
def _cut_bands(self, bands, cutline, target):
"""Cut area of interest based on given cutline
:param bands: data structure of all bands as returned by
_get_all_band_files
:param cutline: filename of required cutline
:param target: directory, where the resulting data should be uploaded
to
:return: same structure as input `bands`, but with cut raster files
"""
area_dir = os.path.join(target, self.area.name)
if not os.path.isdir(area_dir):
os.mkdir(area_dir)
data = {}
for res in bands:
for band in bands[res]:
new_file = os.path.join(area_dir, "{}.jp2".format(band))
gdal.Warp(new_file,
os.path.join(bands[res][band]),
dstSRS="+init=epsg:4326",
cropToCutline=True,
resampleAlg="near",
format="GTiff",
cutlineDSName=cutline,
dstNodata=0,
creationOptions=["COMPRESS=DEFLATE"])
data[band] = new_file
return data
def _get_dates(self, year=None, week=None, date=None):
"""
:param year: required year
:param week: required week
:param date: date as integer
:return: (start_date, end_date, week_number)
"""
week_number = None
start_date = None
end_date = None
if date:
date = int(date)
if date < 10000000:
self.stdout.write(
self.style.ERROR(
'Date <{}> is not in required format YYYYMMDD'.format(
date)))
sys.exit(1)
year = date // 10000
month = (date - (year * 10000)) // 100
day = (date - year * 10000 - month * 100)
date = datetime.datetime(year, month, day)
year, week, weekday = date.isocalendar()
firstday = datetime.timedelta(days=weekday - 1)
lastday = datetime.timedelta(days=PERIOD - weekday)
start_date = date - firstday
end_date = date + lastday
else:
first_day_in_year = datetime.date(year, 1, 1)
first_day_in_week = datetime.timedelta(days=(week - 1) * 7)
last_day_in_week = datetime.timedelta(days=PERIOD)
start_date = first_day_in_year + first_day_in_week
end_date = start_date + last_day_in_week
return (start_date, end_date, week)
def get_bands(self, products):
"""
Get dict with raster bands from downloaded Sentinel products
:param products: list of products
:return: dict of raster files for each band
"""
data = {}
for pid in products:
product = products[pid]
self.save_satellite_image(product)
title = product["title"]
filename = product["filename"]
zipfname = os.path.join(self.tempdir, "{}.zip".format(title))
with ZipFile(zipfname, 'r') as zipObj:
zipObj.extractall(path=self.tempdir)
cutline = self.area.to_geojson_file(self.tempdir)
product_dir = os.path.join(self.tempdir, filename)
all_bands = self._get_all_band_files(product_dir)
crs = self._get_crs_from_band(all_bands["R10m"]["B02"])
clouds_cutline = self._get_final_cutline(product_dir, cutline, crs)
bands = self._cut_bands(all_bands, clouds_cutline, product_dir)
data[product_dir] = bands
return data
def _get_crs_from_band(self, raster):
"""Get EPSG:CODE text string based on input raster file
:param raster: full file name
:return: crs "EPSG:<CODE>" text string
"""
with rio.open(raster) as r:
return "EPSG:{code}".format(code=r.read_crs().to_epsg())
def get_products(self, start_date, end_date, area, clouds=100):
"""
:param start_date: starting date object
:param end_date: end date object
:param area: required area object
"""
products = self.api.query(area.area.wkt,
date=(start_date, end_date),
platformname='Sentinel-2',
cloudcoverpercentage=(0, clouds),
producttype="S2MSI2A")
return products
def cli(user, password, geometry, start, end, uuid, name, download, sentinel, producttype,
instrument, cloud, footprints, path, query, url, order_by, limit):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your Copernicus Open Access Hub user and password, you must pass a geojson file
containing the geometry of the area you want to search for or the UUIDs of the products. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
_set_logger_handler()
api = SentinelAPI(user, password, url)
search_kwargs = {}
if sentinel and not (producttype or instrument):
search_kwargs["platformname"] = "Sentinel-" + sentinel
if instrument and not producttype:
search_kwargs["instrumentshortname"] = instrument
if producttype:
search_kwargs["producttype"] = producttype
if cloud:
if sentinel not in ['2', '3']:
logger.error('Cloud cover is only supported for Sentinel 2 and 3.')
raise ValueError('Cloud cover is only supported for Sentinel 2 and 3.')
search_kwargs["cloudcoverpercentage"] = (0, cloud)
if query is not None:
search_kwargs.update((x.split('=') for x in query.split(',')))
if geometry is not None:
search_kwargs['area'] = geojson_to_wkt(read_geojson(geometry))
if uuid is not None:
uuid_list = [x.strip() for x in uuid.split(',')]
products = {}
for productid in uuid_list:
try:
products[productid] = api.get_product_odata(productid)
except SentinelAPIError as e:
if 'Invalid key' in e.msg:
logger.error('No product with ID \'%s\' exists on server', productid)
elif name is not None:
search_kwargs["identifier"] = name
products = api.query(order_by=order_by, limit=limit, **search_kwargs)
else:
start = start or "19000101"
end = end or "NOW"
products = api.query(date=(start, end),
order_by=order_by, limit=limit, **search_kwargs)
if footprints is True:
footprints_geojson = api.to_geojson(products)
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
product_infos, failed_downloads = api.download_all(products, path)
if len(failed_downloads) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for failed_id in failed_downloads:
outfile.write("%s : %s\n" % (failed_id, products[failed_id]['title']))
else:
for product_id, props in products.items():
if uuid is None:
logger.info('Product %s - %s', product_id, props['summary'])
else: # querying uuids has no summary key
logger.info('Product %s - %s - %s MB', product_id, props['title'],
round(int(props['size']) / (1024. * 1024.), 2))
if uuid is None:
logger.info('---')
logger.info('%s scenes found with a total size of %.2f GB',
len(products), api.get_products_size(products))
def cli(user, password, geometry, start, end, uuid, name, download, md5, sentinel, producttype,
instrument, cloud, footprints, path, query, url, order_by, limit):
"""Search for Sentinel products and, optionally, download all the results
and/or create a geojson file with the search result footprints.
Beyond your Copernicus Open Access Hub user and password, you must pass a geojson file
containing the geometry of the area you want to search for or the UUIDs of the products. If you
don't specify the start and end dates, it will search in the last 24 hours.
"""
_set_logger_handler()
api = SentinelAPI(user, password, url)
search_kwargs = {}
if sentinel and not (producttype or instrument):
search_kwargs["platformname"] = "Sentinel-" + sentinel
if instrument and not producttype:
search_kwargs["instrumentshortname"] = instrument
if producttype:
search_kwargs["producttype"] = producttype
if cloud:
if sentinel not in ['2', '3']:
logger.error('Cloud cover is only supported for Sentinel 2 and 3.')
raise ValueError('Cloud cover is only supported for Sentinel 2 and 3.')
search_kwargs["cloudcoverpercentage"] = (0, cloud)
if query is not None:
search_kwargs.update((x.split('=') for x in query.split(',')))
if geometry is not None:
search_kwargs['area'] = geojson_to_wkt(read_geojson(geometry))
if uuid is not None:
uuid_list = [x.strip() for x in uuid.split(',')]
products = {}
for productid in uuid_list:
try:
products[productid] = api.get_product_odata(productid)
except SentinelAPIError as e:
if 'Invalid key' in e.msg:
logger.error('No product with ID \'%s\' exists on server', productid)
elif name is not None:
search_kwargs["identifier"] = name
products = api.query(order_by=order_by, limit=limit, **search_kwargs)
else:
start = start or "19000101"
end = end or "NOW"
products = api.query(date=(start, end),
order_by=order_by, limit=limit, **search_kwargs)
if footprints is True:
footprints_geojson = api.to_geojson(products)
with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile:
outfile.write(gj.dumps(footprints_geojson))
if download is True:
product_infos, failed_downloads = api.download_all(products, path, checksum=md5)
if md5 is True:
if len(failed_downloads) > 0:
with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile:
for failed_id in failed_downloads:
outfile.write("%s : %s\n" % (failed_id, products[failed_id]['title']))
else:
for product_id, props in products.items():
if uuid is None:
logger.info('Product %s - %s', product_id, props['summary'])
else: # querying uuids has no summary key
logger.info('Product %s - %s - %s MB', product_id, props['title'],
round(int(props['size']) / (1024. * 1024.), 2))
if uuid is None:
logger.info('---')
logger.info('%s scenes found with a total size of %.2f GB',
len(products), api.get_products_size(products))
#!/usr/bin/env python3
from sentinelsat.sentinel import SentinelAPI, read_geojson, geojson_to_wkt
from datetime import date
import os
from_date = date(2018, 11, 1)
to_date = date(2019, 1, 1)
root_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')
aoi_path = os.path.join(root_path, 'data', 'aoi_4326.geojson')
download_path = os.path.join(root_path, 'data', 'images', 's1', '_real')
api = SentinelAPI(os.getenv("SCIHUB_USER"), os.getenv("SCIHUB_PASS"), 'https://scihub.copernicus.eu/dhus')
footprint = geojson_to_wkt(read_geojson(aoi_path))
products = api.query(footprint,
date=(from_date, to_date),
platformname='Sentinel-1',
producttype='GRD',
polarisationmode='VV VH',
orbitdirection='ASCENDING')
for k, p in products.items():
print((k, p['summary']))
os.makedirs(download_path, exist_ok=True)
results = api.download_all(products, directory_path=download_path)
print(results)
