def get_ntl_file_urls(district_gdf: gpd.geodataframe.GeoDataFrame, products: str, startTime: str, endTime: str, collection: int, district_json_file: str):
(w, s, e, n) = ode.district_extents(district_gdf)
# Create a ModapsClient object
a = modapsclient.ModapsClient()
a.headers["Authorization"] = "Bearer " + os.environ.get("LADS_TOKEN")
collections = a.getCollections('VNP46A2')
fileIDs = a.searchForFiles(products=products, startTime=startTime, endTime=endTime,
north=n, south=s, east=e, west=w, coordsOrTiles='coords', dayNightBoth='N', collection=collection)
file_url_list = []
file_name_list = []
for id in fileIDs:
file_url = a.getFileUrls(id)[0]
file_url_list.append(file_url)
file_name_start_pos = file_url.rfind("/") + 1
file_name = file_url[file_name_start_pos:]
file_name_list.append(file_name)
import json
#file_name_list_json = json.dumps(file_name_list)
district_json = {"district_id": int(district_gdf.index[0]),
"start_time": startTime,
"end_time": endTime,
"file_list": file_name_list}
# write json to file
with open(district_json_file + "-" + startTime + "-" + endTime + ".json", 'w', encoding='utf-8') as district_json_dumped:
json.dump(json.dumps(district_json), district_json_dumped)
return file_url_list
def get_LAADS_filepaths(product,
start_time,
end_time,
collection,
bbox=None,
connection_type="https"):
#-------------------------------------------------------------------------.
# Check bbox
if (bbox is None):
bbox = check_bbox(bbox)
coordsOrTiles = "global"
else:
coordsOrTiles = "coords"
#-------------------------------------------------------------------------.
# Process start_time and end_time
# TODO: enhance modaps to accept also hour and minute
# start_time_str = start_time.strftime('%Y-%m-%d %H:%M:%S')
# end_time_str = end_time.strftime('%Y-%m-%d %H:%M:%S')
start_time_str = start_time.strftime('%Y-%m-%d') # %H:%M:%S')
end_time_str = end_time.strftime('%Y-%m-%d') # %H:%M:%S')
#-------------------------------------------------------------------------.
# Retrieve fileIDs
proxy = m.ModapsClient()
fileIDs = proxy.searchForFiles(products=product,
startTime=start_time_str,
endTime=end_time_str,
west=bbox[0],
south=bbox[1],
east=bbox[2],
north=bbox[3],
coordsOrTiles=coordsOrTiles,
collection=collection)
#-------------------------------------------------------------------------.
if ((len(fileIDs) == 1) and (fileIDs[0] == 'No results')):
return
#-------------------------------------------------------------------------.
# Check file is online for direct download
# TODO: Log which fail is not online !
fileIDs_formatted = ",".join(fileIDs)
l_properties = proxy.getFileProperties(fileIDs_formatted)
idx_online = [
properties['online'] == "true" for properties in l_properties
]
fileIDs = list(np.array(fileIDs)[idx_online])
#-------------------------------------------------------------------------.
# Retrieve file urls
fileIDs_formatted = ",".join(fileIDs)
filepaths = sorted(proxy.getFileUrls(fileIDs_formatted))
#-------------------------------------------------------------------------.
# Change to opendap if specified
if (connection_type == "opendap"):
filepaths = convert_to_OPeNDAP_filepaths(filepaths)
#-------------------------------------------------------------------------.
# Return file paths
return (filepaths)
def download_MOD44B():
product='MOD44B'
print("Downloading -> ", product)
years= np.unique(np.array([startdate_obj.year,enddate_obj.year]))
for year in years:
try:
''' The MOD44B data product layers include percent tree cover, percent non-tree cover, percent non-vegetated,
cloud cover, and quality indicators. The start date of the annual period for this product begins with
day of year (DOY) 65 (March 5). '''
temp = datetime.strptime( str(year) +'-01-01', '%Y-%m-%d')
temp= temp + timedelta(days=64)
print(temp)
doy = int(datetime(temp.year,temp.month,temp.day).timetuple().tm_yday)
print(doy)
path_= os.path.join( path,product, str(temp.year))
if not os.path.exists(path_):
print(path_)
os.makedirs(path_)
a = modapsclient.ModapsClient()
items=a.searchForFiles(products=product, startTime=temp,
endTime=temp, north=north,south=south,
west=west,east=east, collection=collection)
print("Products count = > ",len(items),items[0])
for p in items:
if(p.strip() =='No results'.strip()): continue
url=a.getFileUrls(p)[0]
print('=====>',p,url)
# cmd=('wget --user hiik324 --password Ecology123 {0} --header "Authorization: Bearer C88B2F44-881A-11E9-B4DB-D7883D88392C" -P {1} '.format( url, path_))
cmd=('wget --user hiik324 --password Ecology123 {0} -P {1} '.format( url, path_))
os.system(cmd)
except:
print("Unexpected error:", sys.exc_info()[0])
continue
def download_modis_products():
products = [
"MCD12C1", "MCD15A3H", "MCD43D59", "MCD43D60", "MCD43D61", "MCD43D62",
"MCD43D63", "MOD15A2H", "MOD44B"
]
for product in products:
print("Downloading -> ", product)
try:
for day in range(int((enddate_obj - startdate_obj).days)):
temp = startdate_obj + timedelta(days=day)
doy = int(
datetime(temp.year, temp.month,
temp.day).timetuple().tm_yday)
if (product == 'MOD44B'):
path_ = os.path.join(path, product, str(temp.year))
else:
path_ = os.path.join(path, product, str(temp.year),
str(doy))
if not os.path.exists(path_):
print(path_)
os.makedirs(path_)
a = modapsclient.ModapsClient()
items = a.searchForFiles(products=product,
startTime=temp,
endTime=temp,
north=north,
south=south,
west=west,
east=east,
collection=collection)
print("Products count = > ", len(items))
# for p in items:
# url=a.getFileUrls(p)[0]
# print(p,url)
# # cmd=('wget --user hiik324 --password Ecology123 {0} --header "Authorization: Bearer C88B2F44-881A-11E9-B4DB-D7883D88392C" -P {1} '.format( url, path_))
# cmd=('wget --user hiik324 --password Ecology123 {0} -P {1} '.format( url, path_))
# os.system(cmd)
except:
print("Error")
print("Unexpected error:", sys.exc_info()[0])
def download_Aqua_Terra_L2():
products=["MYD04_L2","MYD06_L2","MYD07_L2","MYD11_L2","MOD04_L2","MOD06_L2","MOD07_L2","MOD11_L2"]
for product in products:
print("Downloading ->", product)
try:
for day in range (int((enddate_obj - startdate_obj).days)):
temp= startdate_obj + timedelta(days=day)
doy = int(datetime(temp.year,temp.month,temp.day).timetuple().tm_yday)
if(product[:3]=='MYD'):
path_= os.path.join( path,"Aqua_L2", str(temp.year),str(doy),product)
elif (product[:3]=='MOD'):
path_= os.path.join( path,"Terra_L2", str(temp.year),str(doy),product)
if not os.path.exists(path_):
print(path_)
os.makedirs(path_)
a = modapsclient.ModapsClient()
if(product=='MYD011_L2' or product =="MOD11_L2"):
collection="6"
else:
collection="61"
items=a.searchForFiles(products=product, startTime=startdate,
endTime=enddate, north=north,south=south,
west=west,east=east, collection=collection)
print("Products count = > ",len(items))
# download_items(a,items,path_)
except Exception as e:
print("Error: ",e)
print("Unexpected error:", sys.exc_info()[0])
continue
def get_products(satellite, instrument):
modaps_inst = _modaps_sat_acronym(satellite, instrument)
# Retrieve products
proxy = m.ModapsClient()
products = sorted(proxy.listProductsByInstrument(modaps_inst))
return products
def get_collection_most_recent(product):
proxy = m.ModapsClient()
collections = proxy.getCollections(product)
collections = [int(v) for v in collections.keys()]
most_recent_collection = str(max(collections))
return most_recent_collection
def get_collections(product):
proxy = m.ModapsClient()
collections = proxy.getCollections(product)
return collections
def find_modis_data(location,
date=datetime.utcnow(),
ndays=1,
products=['MODATML2', 'MYDATML2'],
collection='61',
dbg=False):
"""Finds near-realtime MODIS products from the Aqua/Terra satellites for a
specific location <location>, starting at a specifc ([date]; defaults to
'now') for a number of days ([ndays]; defaults to 1). By default the MODIS
Joint Atmosphere products, MODATML2/MYDATML2 in Collection 61:
https://ladsweb.modaps.eosdis.nasa.gov/missions-and-measurements/products/l2-joint-atmosphere/MYDATML2/
are searched but this can be overridden by passing in [products] and
[collection].
A dictionary with the name of HDF file products as keys is returned; the
values for each HDF file is a dictionary of the following form:
url: URL link to retrieve the HDF file,
checksum: CRC checksum of the file (`cksum` can be used to generate
the checksum and size,
size: size of the HDF file in bytes,
name: name of the HDF file,
"""
import modapsclient as m
import logging
logging.basicConfig(level=logging.WARNING)
a = m.ModapsClient()
start_date = date.date()
end_date = start_date + timedelta(days=ndays)
west_val, south_val, east_val, north_val = make_bounding_box(location)
dl_files = {}
for product in products:
if dbg: print(product)
collections = a.getCollections(product)
if collection not in collections:
print("Did not find specified collection (%s) in %s".format(
collection, collections))
return None
files = a.searchForFiles(product,
start_date,
end_date,
north=north_val,
west=west_val,
east=east_val,
south=south_val,
collection=collection)
if dbg: print(files)
for hdf_file in files:
url = a.getFileUrls(hdf_file)
props = a.getFileProperties(hdf_file)
if len(props) == 1:
props = props[0]
else:
print("Unexpected number of file properties found")
continue
if props.get('online', 'false') == 'true':
dl_files[hdf_file] = {
'url': url,
'checksum': props.get('checksum', ''),
'size': props.get('fileSizeBytes', ''),
'name': props.get('fileName', '')
}
else:
print("File id %d (%s) is not online".format(
hdf_file, props.get('fileName', 'UNKNOWN')))
return dl_files
def realmodaps():
a = modapsclient.ModapsClient()
return a
def mockmodaps():
a = modapsclient.ModapsClient()
a._makeurl = fakeurl
a._rawresponse = mockedresponse
return a
def __init__(self, config_name="ModisAPI"):
self._config = load_config(Modis.CONFIG_PATH, config_name)
self._defaults = self._config['defaults']
self._modis_tiles = self._load_modis_tiles()
self._api_key = get_api_key(self._config)
self._m = m.ModapsClient()