def search_api_sat(self, sname, bounds, time, collection=None):
"""
API search of the different satellite granules and return metadata dictionary
:param sname: short name satellite product, ex: 'MOD03'
:param bounds: bounding box as (lon_min,lon_max,lat_min,lat_max)
:param time: time interval (init_time_iso,final_time_iso)
:param collection: id of the collection to specify
:return metas: a dictionary with all the metadata for the API search
"""
maxg = 1000
# creating polygon with the search bounding box
lonmin, lonmax, latmin, latmax = bounds
bbox = [(lonmin, latmax), (lonmin, latmin), (lonmax, latmin),
(lonmax, latmax), (lonmin, latmax)]
time_utcf = (utc_to_utcf(time[0]), utc_to_utcf(time[1]))
api = GranuleQuery()
search = api.parameters(short_name=sname,
downloadable=True,
polygon=bbox,
temporal=time_utcf)
sh = search.hits()
if sh > maxg:
logging.warning(
"The number of hits %s is larger than the limit %s." %
(sh, maxg))
logging.warning("Any satellite data with prefix %s used." %
self.prefix)
logging.warning(
"Use a reduced bounding box or a reduced time interval.")
metas = []
else:
metas = api.get(sh)
if collection:
metas = [
m for m in metas if m['collection_concept_id'] == collection
]
logging.info(
'search_api_sat - CMR API gives {0} hits for {1} of collection {2}'
.format(len(metas), sname, collection))
return metas
def query_cmr(self, params=None):
"""
Queries CMR for one or more data sets short-names using the spatio-temporal
constraints defined in params. Returns a json list of CMR records.
"""
if params is None:
return None
self.granules = {}
datasets = self._expand_datasets(params)
for d in datasets:
cmr_api = GranuleQuery()
g = cmr_api.parameters(
short_name=d['name'],
version=d['version'],
temporal=d['temporal'],
bounding_box=d['bounding_box']).get_all()
self.granules[d['name']] = g
self.cmr_download_size(self.granules)
return self.granules
def fetch_datasets(self, download_type='ROI_polygon',
roi_polygon=None,
startdate=None,
enddate=None,
cloudcover_max=5,
product_shortname="SPL3SMP",
max_products=-1,
inverse_polygon_order=False):
"""Query NASA API for products
See: https://modis.ornl.gov/data/modis_webservice.html
"""
from cmr import GranuleQuery
self.product_shortname = product_shortname
self.roi_polygon = roi_polygon
if download_type == 'ROI_polygon':
if roi_polygon.split('.')[-1] == 'geojson':
list_coords = from_geojson_to_list_coords(self.roi_polygon)
#print(list_coords)
else:
raise RuntimeError("Unknown download type")
if inverse_polygon_order:
list_coords = list_coords[::-1]
api = GranuleQuery().polygon(list_coords).short_name(self.product_shortname).temporal(startdate, enddate)
n_produtes = api.hits()
#print(f"{n_produtes} products found for these parameters")
if max_products == -1:
max_products = n_produtes
self.list_products = api.get(limit=max_products)
self.list_products_id = [ f["producer_granule_id"] for f in self.list_products]
def search_api_sat(self, sname, bbox, time, version=None):
"""
API search of the different satellite granules and return metadata dictionary
:param sname: short name satellite product, ex: 'MOD03'
:param bbox: polygon with the search bounding box
:param time: time interval (init_time_iso,final_time_iso)
:return metas: a dictionary with all the metadata for the API search
"""
maxg=100
time_utcf=(utc_to_utcf(time[0]),utc_to_utcf(time[1]))
api = GranuleQuery()
if not version:
search = api.parameters(
short_name=sname,
downloadable=True,
polygon=bbox,
temporal=time_utcf
)
else:
search = api.parameters(
short_name=sname,
downloadable=True,
polygon=bbox,
temporal=time_utcf,
version=version
)
sh=search.hits()
logging.info("CMR API: %s gets %s hits in this range" % (sname,sh))
if sh>maxg:
logging.warning("The number of hits %s is larger than the limit %s." % (sh,maxg))
logging.warning("Any satellite data with prefix %s used." % self.prefix)
logging.warning("Use a reduced bounding box or a reduced time interval.")
metas = []
else:
metas = api.get(sh)
return metas
def query(start_date, end_date, product='MCD43A4.006', provider='LPDAAC_ECS'):
""" Search CMR database for spectral MODIS tiles matching a temporal range,
defined by a start date and end date. Returns metadata containing the URL of
each image.
"""
q = GranuleQuery()
prod, ver = product.split('.')
q.short_name(prod).version(ver)
q.temporal('%sT00:00:00Z' % str(start_date),
'%sT23:59:00Z' % str(end_date))
_granules = q.get_all()
# filter dates
day_offset = products[product]['day_offset']
granules = []
for gran in _granules:
# CMR uses day 1 of window - correct this to be middle of window
date = (dateparser(gran['time_start'].split('T')[0]) +
datetime.timedelta(days=day_offset)).date()
if (start_date <= date <= end_date):
granules.append(gran)
logger.info("%s granules found within %s - %s" %
(len(granules), start_date, end_date))
return granules
def get_modis(destination, date, search_window, footprint, choose, clip,
reproject):
"""
Retrieves MODIS scenes.
Scenes must entirely contain <footprint> within date +/- search_window.
"""
workdir = path.join(destination, "modis")
makedirs(workdir, exist_ok=True)
footprint_path = path.join(destination, footprint)
searchFootprint = shape(fiona.open(footprint_path)[0]["geometry"])
date_window = timedelta(days=search_window)
date_range = (date - date_window, date + date_window)
cmrAPI = GranuleQuery()
results = (cmrAPI.short_name(MODIS_SNOW_CMR_SHORT_NAME).bounding_box(
*searchFootprint.bounds).temporal(*date_range).get())
results = json_normalize(results)
print(results["time_start"].iloc[0])
print(date)
results["time_start"] = to_datetime(
results["time_start"]).dt.tz_localize(None)
results["timedeltas"] = (date - results.time_start).abs()
results = results.sort_values(by="timedeltas", ascending=True)
results["browse"] = [[
_i["href"] for _i in image_links if _i.get("title", "") == "(BROWSE)"
][0] for image_links in results.links]
print(results.iloc[0].links)
image_iloc = 0
if choose:
print(
tabulate(
results[[
"cloud_cover", "day_night_flag", "timedeltas", "browse"
]].reset_index(drop=True),
headers="keys",
))
image_iloc = int(
input("Choose image ID [0-{}]: ".format(len(results) - 1)))
image_url = results.iloc[image_iloc].links[0]["href"]
MODIS_DL_COMMAND = ("wget "
"--http-user=$EARTHDATA_USERNAME "
"--http-password=$EARTHDATA_PASSWORD "
"--no-check-certificate --auth-no-challenge "
'-r --reject "index.html*" -np -e robots=off -nH -nd '
"--directory-prefix={destination} "
"{image_url}".format(destination=workdir,
image_url=image_url))
_sh = Popen(MODIS_DL_COMMAND, shell=True).communicate()
## have to do some gdal magic with MODIS data.
## 1) turn it into a TIFF with the right projection.
MODISfile = glob(path.join(workdir, "MOD10A1*.hdf*"))[0]
output_file = path.join(workdir, "MODIS_reproj.tif")
MODIS_CONVERT_COMMAND = (
"gdalwarp "
"HDF4_EOS:EOS_GRID:{path}:MOD_Grid_Snow_500m:NDSI_Snow_Cover "
"-cutline {footprint} -crop_to_cutline -dstnodata 9999 "
"-t_srs {projection} -r cubic "
"-s_srs '+proj=sinu +R=6371007.181 +nadgrids=@null +wktext' "
"{output_tif} ".format(
path=MODISfile,
footprint=footprint_path,
projection=reproject,
output_tif=output_file,
))
_sh = Popen(MODIS_CONVERT_COMMAND, shell=True).communicate()
print(MODISfile, output_file)
## Search/Download - sandbox
from cmr import CollectionQuery, GranuleQuery
import json
import requests
import urlparse
import os
import sys
api = GranuleQuery()
fire = GranuleQuery()
#MOD14: bounding box = Colorado (gps coord sorted lat,lon; counterclockwise)
MOD14granules = api.parameters(
short_name="MOD14",
platform="Terra",
downloadable=True,
polygon=[(-109.0507527,40.99898), (-109.0698568,37.0124375), (-102.0868788,36.9799819),(-102.0560592,40.999126), (-109.0507527,40.99898)],
temporal=("2017-01-01T00:00:00Z", "2017-01-07T00:00:00Z") #time start,end
)
print "MOD14 gets %s hits in this range" % MOD14granules.hits()
MOD14granules = api.get(10)
#MOD03: geoloc data for MOD14
MOD03granules = api.parameters(
short_name="MOD03",
platform="Terra",
downloadable=True,
polygon=[(-109.0507527,40.99898), (-109.0698568,37.0124375), (-102.0868788,36.9799819),(-102.0560592,40.999126), (-109.0507527,40.99898)],
temporal=("2017-01-01T00:00:00Z", "2017-01-07T00:00:00Z") #time start,end
)
print "MOD03 gets %s hits in this range" % MOD03granules.hits()
from cmr import CollectionQuery, GranuleQuery
import json
api = GranuleQuery()
#MOD14: data - auto-sorted by start_time
MOD14granules = api.parameters(
short_name="MOD14",
platform="Terra",
downloadable=True)
print MOD14granules.hits()
#MOD14granules = api.get_all()
MOD14granules = api.get(10)
for granule in MOD14granules:
print granule["title"], granule["time_start"], granule["time_end"], granule["polygons"]
#MOD03: geolocation data - auto-sorted by start_time
MOD03granules = api.parameters(
short_name="MOD03",
platform="Terra",
downloadable=True)
print MOD03granules.hits()
MOD03granules = api.get(10)
for granule in MOD03granules:
print json.dumps(granule, indent=4, separators=(',', ': '))
#print granule["title"], granule["time_start"], granule["time_end"], granule["longitude"]
#"polygons" gives long/lat data in pairs-first/last pair are the same
# note - it appears that all granules are quadrilaterals