def get_spec_from_api(api_base_url):
"""Get the spec and version at the given source."""
spec_url = URL(api_base_url) / "swagger/v2/swagger.json"
spec_response = spec_url.get()
spec = spec_response.json()
version = fetch_orbit_spec_version_no(api_base_url)
return spec, version
def fetch_orbit_spec_version_no(api_base_url):
"""Get the Orbit API spec version number from the given source."""
spec_version_url = URL(api_base_url) / "About/Version"
spec_version_response = spec_version_url.get()
spec_version = spec_version_response.json().get("version")
if spec_version is None:
raise LookupError("Failed to fetch OrbitAPI spec version number.")
else:
return spec_version
def modisFile(year=2020, month=1, day=1,tile='h08v06',\
product='MCD15A3H',timeout=None,\
version='006',no_cache=False,cache=None,\
verbose=False,force=False,altcache='/shared/groups/jrole001/geog0111'):
'''
Get the filename associated with a MODIS product file
for a certain date and version.
modisFile : return the filename for a MODIS product
E.g.
/Users/plewis/.modis_cache/e4ftl01.cr.usgs.gov/\
MOTA/MCD15A3H.006/2020.01.05/\
MCD15A3H.A2020005.h08v06.006.2020010210940.hdf
N.B. You need to have a username and password to access the data.
These are available at https://urs.earthdata.nasa.gov
Example of use:
from geog0111.modisUtils import modisURL, modisFile
modinfo = {
'product' : 'MCD15A3H',
'year' : 2020,
'month' : 1,
'day' : 5,
'tile' : 'h08v06'
}
url = modisURL(**modinfo,verbose=False)
print(f'-> {url}')
filename = modisFile(**modinfo,verbose=False)
print(f'-> {filename}')
-> https://e4ftl01.cr.usgs.gov/MOTA/MCD15A3H.006/2020.01.05/MCD15A3H.A2020005.h08v06.006.2020010210940.hdf
-> /Users/plewis/.modis_cache/e4ftl01.cr.usgs.gov/\
MOTA/MCD15A3H.006/2020.01.05/MCD15A3H.A2020005.h08v06.006.2020010210940.hdf
Returns:
filename : Path object of MODIS data product file
e.g. what you would find on
https://e4ftl01.cr.usgs.gov/MOTA/MCD15A3H.006/2020.01.05/MCD15A3H.A2020005.h08v06.006.2020010210940.hdf
Downloaded to some cache location e.g.
/Users/plewis/.modis_cache/e4ftl01.cr.usgs.gov/MOTA/MCD15A3H.006/2020.01.05/MCD15A3H.A2020005.h08v06.006.2020010210940.hdf
Control options:
year : int of year (2000+ for Terra, 2002+ for Aqua products)
(year=2020)
month: int of month (1-12) (month=1)
day : int of day (1-31, as appropriate) (day=1)
tile : string of tile (tile='h08v06')
product : string of MODIS product name (product='MCD15A3H')
version : int or string of version (version='006')
timeout : timeout in seconds
verbose : verbosity (verbose=False)
Cache options:
no_cache : Set True if you don't want to use the cache
(no_cache=False)
This is common for most functions, but
modisFile() will use a cache in any case,
as it has to store the file somewhere.
If you don;'t want to keep that, then
you can delete after use.
cache : Use cache='/home/somewhere/else' to specify
a personal cache location with write permission
(ie somewhere in your filespace)
Specify personal cache root. By default,
this will be ~, and the cache will go into
~/.modis_cache. You can change that to
somewhere else
here. It will still use the sub-directory
.modis_cache.
Use cache='/home/somewhere/else' to specify a
personal cache location with write permission
(ie somewhere in your filespace)
altcache : Specify system cache root.
Use altcache='/home/notme/somewhere' to specify a
system cache location with read permission
(ie somewhere not necessarily in your filespace)
force : Bool : Use force=True to override information in the cache
Get the URL associated with a MODIS product
for a certain date and version. Since this can
involve an expensive call to get the html to access the file URL
The html data used can be cached unless no_cache = True
(See modisHTML())
This function returns the URL for the product/date page listing
The caching is done to avoid repeated calls to expensive URL downloads.
The idea is that there will be a system cache, where shared files will
be set up (where you have read permission), and a personal cache
where you can read and write your own files. Unless you
use force=True or disble cache with no_cache=True, then the code
will look in (i) personal; (ii) system cache before attempting
to download any file from a URL.
The cached files are stored in the same structure as the URL, i.e
https://e4ftl01.cr.usgs.gov/MOTA/MCD15A3H.006/2020.01.05/\
MCD15A3H.A2020005.h08v06.006.2020010210940.hdf
will be stored (personal cache) as:
~/.modis_cache/e4ftl01.cr.usgs.gov/MOTA/MCD15A3H.006/2020.01.05/\
MCD15A3H.A2020005.h08v06.006.2020010210940.hdf
The html cache is what is returned from e.g.
https://e4ftl01.cr.usgs.gov/MOTA/MCD15A3H.006/2020.01.05
and is stored as eg
~/.modis_cache/e4ftl01.cr.usgs.gov/MOTA/MCD15A3H.006/2020.01.05/index.html
'''
if verbose:
print('modisFile')
print(f'year : {year}\nmonth : {month}\nday : {day}\ntile : {tile}')
print(f'product : {product}\nversion : {version}\nforce : {force}')
print(f'altcache : {altcache}')
print(f'cache : {cache}')
# get the URL for the file of interest
url = modisURL(year, month, day,tile,product=product,version=version,\
no_cache=no_cache,cache=cache,altcache=altcache,\
force=force,verbose=verbose)
if url == None:
print(f'No dataset URL found for conditions requested.')
print(
f'Check the date and tile for the dataset URL that you have requested: {modisServer(product=product,version=version)}'
)
print('modisFile')
print(f'year : {year}\nmonth : {month}\nday : {day}\ntile : {tile}')
print(f'product : {product}\nversion : {version}\nforce : {force}')
print(f'altcache : {altcache}')
print(f'cache : {cache}')
print(f'If you think that is ok, then first try to rerun')
print(f'If you still have problems, perhaps use: force=True')
print(f'and try setting timout, e.g. timout=200 (it is in seconds)')
return None
if cache == None:
cache = Path().cwd()
cache = cache / ".modis_cache"
cache.mkdir(parents=True, exist_ok=True)
if not no_cache:
if verbose:
print(f'cache {cache}')
#import pdb;pdb.set_trace()
# generate the Path where the local cache would go
cache_part = Path(url.hostinfo, '/'.join(url.parts[1:]))
# check to see if we have it
if (not no_cache) and (not force) and Path(cache, cache_part).exists():
if verbose:
print('getting from cache')
return Path(cache, cache_part)
# get from cache.store for backwards compatibility
if (not no_cache) and (not force) and cache and Path(
cache,
cache_part.as_posix() + '.store').exists():
if verbose:
print('getting from cache in backward compatibility mode')
return Path(cache, cache_part.as_posix() + '.store')
# get from altcache
if (not no_cache) and (not force) and altcache and Path(
altcache, cache_part).exists():
if verbose:
print('getting from altcache')
return Path(altcache, cache_part)
# get from altcache.store
if (not no_cache) and (not force) and altcache and Path(
altcache,
cache_part.as_posix() + '.store').exists():
if verbose:
print('getting from altcache in backward compatibility mode')
return Path(altcache, cache_part.as_posix() + '.store')
# else pull the file
# first try a get : we have to do this twice bacause of
# auth redirect
if verbose:
print(f'logging in to {url.anchor}')
url2 = url.with_userinfo(*Cylog(url.anchor).login())
if verbose:
print(f'get info from {url.anchor}')
r = url2.get(timeout=timeout)
if verbose:
print(f'get data from {url.anchor}')
url3 = URL(r.url).with_userinfo(*Cylog(url.anchor).login())
r2 = url3.get(timeout=timeout)
if verbose:
print(f'done - status code {r2.status_code}')
if r2.status_code == 200:
data = r2.content
if verbose:
print(f"received {len(data)} bytes")
# write to cache file
cache_file = cache / cache_part
cache_file.parent.mkdir(parents=True, exist_ok=True)
nbytes_written = cache_file.write_bytes(data)
if not (nbytes_written == len(data)):
if verbose:
print(
f'error writing cache file {cache_file}: {len(data)} bytes expected but {nbytes_written} bytes written'
)
return None
else:
print(f'cached data to file {cache_file}: {len(data)}')
return cache_file
else:
if verbose:
print(f'failed to pull data from {url.anchor}')
return None
