def fetch(cls, asset, tile, date):
"""Fetch to the stage.
Returns a list with one item, the full path to the staged asset.
"""
utils.verbose_out('{}: fetch tile {} for {}'.format(asset, tile, date),
3)
fn_prefix = cls._assets[asset]['fn-prefix']
stage_dir = cls.Repository.path('stage')
with utils.error_handler("Error downloading from " + cls._host, continuable=True), \
utils.make_temp_dir(prefix='fetchtmp', dir=stage_dir) as td_name:
qs_rv = cls.query_service(asset, tile, date)
if qs_rv is None:
return []
remote_fn = qs_rv['basename']
local_fn = fn_prefix + remote_fn
temp_fp = os.path.join(td_name, local_fn)
stage_fp = os.path.join(stage_dir, local_fn)
utils.verbose_out(
"Downloading {}, local name {}".format(remote_fn, local_fn), 2)
conn = cls.ftp_connect(asset, date)
with open(temp_fp, "wb") as temp_fo:
conn.retrbinary('RETR ' + remote_fn, temp_fo.write)
conn.quit()
os.rename(temp_fp, stage_fp)
return [stage_fp]
return []
def process(self, products=None, overwrite=False, **kwargs):
"""Produce data products and save them to files.
Only one product; it's processed in the usual way, but for this
driver, it's extracted from the gzip file and saved (not
symlink/vsi). Method signature is largely for campatibilty with
the rest of gips, eg kwargs is unused.
"""
needed_products = self.needed_products(products, overwrite)
if len(needed_products) == 0:
utils.verbose_out('No new processing required.')
return
# sanity check that requested product & asset look ok
assert (needed_products.requested == {
_product_type: [_product_type]
} and _asset_type in self.assets)
asset = self.assets[_asset_type]
err_msg = 'Error creating product {} from {}'.format(
_product_type, os.path.basename(asset.filename))
with utils.error_handler(err_msg, continuable=True):
temp_fp = self.temp_product_filename(_sensor, _product_type)
# make gdal/gippy-readable path to the inner file
vsi_inner_path = '/vsigzip/' + asset.filename
os.symlink(vsi_inner_path, temp_fp)
archive_fp = self.archive_temp_path(temp_fp)
self.AddFile(_sensor, _product_type, archive_fp)
def _read_point(cls, filename, roi, nodata):
""" Read single point from mean/var file and return if valid, or mean/var of 3x3 neighborhood """
if not os.path.exists(filename):
return (numpy.nan, numpy.nan)
with utils.error_handler(
'Unable to read point from {}'.format(filename),
continuable=True):
img = gippy.GeoImage(filename)
vals = img[0].Read(roi).squeeze()
variances = img[1].Read(roi)
vals[numpy.where(vals == nodata)] = numpy.nan
variances[numpy.where(variances == nodata)] = numpy.nan
val = numpy.nan
var = numpy.nan
if ~numpy.isnan(vals[1, 1]):
val = vals[1, 1]
elif numpy.any(~numpy.isnan(vals)):
val = numpy.mean(vals[~numpy.isnan(vals)])
if ~numpy.isnan(variances[1, 1]):
var = variances[1, 1]
elif numpy.any(~numpy.isnan(variances)):
var = numpy.mean(variances[~numpy.isnan(variances)])
img = None
return (val, var)
return (numpy.nan, numpy.nan)
def main(cls):
""" Main for algorithm classes """
dhf = argparse.ArgumentDefaultsHelpFormatter
# Top level parser
parser = argparse.ArgumentParser(formatter_class=dhf,
description=cls.info())
parser.add_argument('-v',
'--verbose',
help='Verbosity - 0: quiet, 1: normal, 2+: debug',
default=1,
type=int)
parser = cls.parser(parser)
args = parser.parse_args()
gippy.Options.SetVerbose(args.verbose)
VerboseOut(cls.info())
utils.gips_script_setup(driver_string=None, setup_orm=False)
with utils.error_handler('Error in {}'.format(cls.name)):
alg = cls(**vars(args))
alg.run_command(**vars(args))
utils.gips_exit()
def query_provider(cls, asset, tile, date):
year, month, day = date.timetuple()[:3]
if asset != "ASM":
mainurl = "%s/%04d/%02d" % (cls._assets[asset]['url'], year, month)
pattern = cls._assets[asset]['re_pattern'] % (year, month, day)
else:
# asset ASM is for constants which all go into 1980-01-01
mainurl = cls._assets[asset]['url']
pattern = cls._assets[asset]['re_pattern'] % (0, 0, 0)
cpattern = re.compile(pattern)
with utils.error_handler("Error downloading"):
# obtain the list of files
response = cls.Repository.managed_request(mainurl, verbosity=2)
if response is None:
return None, None
for item in response.readlines():
# inspect the page and extract the full name of the needed file
if cpattern.search(item):
if 'xml' in item:
continue
basename = cpattern.findall(item)[0]
url = '/'.join([mainurl, basename])
return basename, url
utils.verbose_out(
"Unable to find a remote match for"
" {} at {}".format(pattern, mainurl), 4)
return None, None
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('additional inventory options')
group.add_argument('--md',
help='Show dates using MM-DD',
action='store_true',
default=False)
group.add_argument(
'--rectify',
help=
'Instead of displaying or fetching inventory, rectify the inventory '
'database by comparing it against the present state of the data repos.',
action='store_true',
default=False)
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
with utils.error_handler():
print(title)
if args.rectify:
if not orm.use_orm():
raise ValueError("--rectify can only be used if"
" GIPS_ORM = True.")
for k, v in vars(args).items():
# Let the user know not to expect other options to effect rectify
if v and k not in ('rectify', 'verbose', 'command'):
msg = "INFO: Option '--{}' is has no effect on --rectify."
utils.verbose_out(msg.format(k), 1)
print("Rectifying inventory DB with filesystem archive:")
print("Rectifying assets:")
dbinv.rectify_assets(cls.Asset)
print("Rectifying products:")
dbinv.rectify_products(cls)
return
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
inv.pprint(md=args.md, size=args.size)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def process(self, *args, **kwargs):
""" Process all requested products for this tile """
products = super(sarannualData, self).process(*args, **kwargs)
if len(products) == 0:
return
self.basename = self.basename + '_' + self.sensor_set[0]
for key, val in products.requested.items():
fname = os.path.join(self.path, self.basename + '_' + key)
# Verify that asset exists
a_type = self._products[val[0]]['assets'][0]
a_obj = self.assets.get(a_type)
if a_obj is None:
utils.verbose_out(
"Asset {} doesn't exist for tile {}".format(
a_type, self.id), 3)
continue
datafiles = None
with utils.error_handler(
"Error extracting files from asset {}".format(
a_obj.filename),
continuable=True):
datafiles = a_obj.extract()
if datafiles is None:
continue
if val[0] == 'sign':
bands = [
datafiles[b] for b in ["sl_HH", "sl_HV"] if b in datafiles
]
if len(bands) > 0:
img = gippy.GeoImage(bands)
img.SetNoData(0)
mask = gippy.GeoImage(datafiles['mask'], False)
img.AddMask(mask[0] == 255)
imgout = gippy.GeoImage(fname, img, gippy.GDT_Float32)
imgout.SetNoData(-32768)
for b in range(0, imgout.NumBands()):
imgout.SetBandName(img[b].Description(), b + 1)
(img[b].pow(2).log10() * 10 - 83.0).Process(imgout[b])
fname = imgout.Filename()
img = None
imgout = None
[
RemoveFiles([f], ['.hdr', '.aux.xml'])
for k, f in datafiles.items() if k != 'hdr'
]
if val[0] == 'fnf':
if 'C' in datafiles:
# rename both files to product name
os.rename(datafiles['C'], fname)
os.rename(datafiles['C'] + '.hdr', fname + '.hdr')
img = gippy.GeoImage(fname)
img.SetNoData(0)
img = None
self.AddFile(self.sensor_set[0], key, fname)
def main():
title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser()
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
# create output directory if needed
# tld is "{}_tiles_{}_{}".format(DATATYPE, RESOLUTION, SUFFIX)
if args.notld:
tld = args.outdir
else:
tld = os.path.join(args.outdir, '%s_tiles' % args.command)
if args.res is not None:
tld = tld + '_%sx%s' % (args.res[0], args.res[1])
if args.suffix != '':
tld = tld + '_' + args.suffix
mkdir(tld)
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
for date in inv.dates:
for tid in inv[date].tiles:
# make sure back-end tiles are processed
inv[date].tiles[tid].process(args.products,
overwrite=False)
# warp the tiles & copy into place in the output dir
inv[date].tiles[tid].copy(tld, args.products,
inv.spatial.site, args.res,
args.interpolation, args.crop,
args.overwrite, args.tree)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def fetch(cls, asset, tile, date):
year, month, day = date.timetuple()[:3]
mainurl = '%s/%s.%02d.%02d' % (cls._assets[asset]['url'], str(year), month, day)
utils.verbose_out("searching at " + mainurl, 4)
with utils.error_handler('Unable to access {}'.format(mainurl)):
response = cls.Repository.managed_request(mainurl)
if response is None:
return []
pattern = '(%s.week\d{2}.%s.%s.doy\d{3}to\d{3}.v1.5.hdf)' % (asset, str(year), tile)
cpattern = re.compile(pattern)
fetched = []
http_kw = {'timeout': 10,
'auth': (cls.Repository.get_setting('username'),
cls.Repository.get_setting('password'))}
for item in response.readlines():
if cpattern.search(item):
if 'xml' in item:
continue
name = cpattern.findall(item)[0]
url = ''.join([mainurl, '/', name])
utils.verbose_out("found " + url)
outpath = os.path.join(cls.Repository.path('stage'), name)
if os.path.exists(outpath):
continue
# match found, perform the download
err_msg = 'Unable to retrieve {} from {}'.format(name, url)
with utils.error_handler(err_msg, continuable=True):
response = cls.Repository.managed_request(url)
if response is None:
return fetched # might as well give up now since the rest probably fail too
with open(outpath, 'wb') as fd:
fd.write(response.read())
utils.verbose_out('Retrieved {}'.format(name), 2)
fetched.append(outpath)
if not fetched:
utils.verbose_out('Unable to find remote match for {} at {}'.format(pattern, mainurl),
4, sys.stderr)
return fetched
def process(self, *args, **kwargs):
""" Process assets into requested products """
# TODO - some check on if any processing was done
start = dt.now()
VerboseOut(
'Processing [%s] on %s dates (%s files)' %
(self.products, len(self.dates), self.numfiles), 3)
if len(self.products.standard) > 0:
for date in self.dates:
with utils.error_handler(continuable=True):
self.data[date].process(*args, **kwargs)
if len(self.products.composite) > 0:
self.dataclass.process_composites(self, self.products.composite,
**kwargs)
VerboseOut('Processing completed in %s' % (dt.now() - start), 2)
def fetch(cls, asset, tile, date):
qs_rv = cls.query_service(asset, tile, date)
if qs_rv is None:
return []
basename, url = qs_rv['basename'], qs_rv['url']
with utils.error_handler("Asset fetch error ({})".format(url),
continuable=True):
response = cls.Repository.managed_request(url)
if response is None:
return []
outpath = os.path.join(cls.Repository.path('stage'), basename)
with open(outpath, 'wb') as fd:
fd.write(response.read())
utils.verbose_out('Retrieved ' + basename, 2)
return [outpath]
return []
def query_provider(cls, asset, tile, date):
"""Find out from the modis servers what assets are available.
Uses the given (asset, tile, date) tuple as a search key, and
returns a tuple: base-filename, url
"""
year, month, day = date.timetuple()[:3]
if asset == "MCD12Q1" and (month, day) != (1, 1):
utils.verbose_out(
"Cannot fetch MCD12Q1: Land cover data"
" are only available for Jan. 1",
1,
stream=sys.stderr)
return None, None
mainurl = "%s/%s.%02d.%02d" % (cls._assets[asset]['url'], str(year),
month, day)
pattern = '(%s.A%s%s.%s.\d{3}.\d{13}.hdf)' % (
asset, str(year), str(date.timetuple()[7]).zfill(3), tile)
cpattern = re.compile(pattern)
if datetime.datetime.today().date().weekday() == 2:
err_msg = ("Error downloading on a Wednesday;"
" possible planned MODIS provider downtime: " + mainurl)
else:
err_msg = "Error downloading: " + mainurl
with utils.error_handler(err_msg):
response = cls.Repository.managed_request(mainurl, verbosity=2)
if response is None:
return None, None
for item in response.readlines():
# screen-scrape the content of the page and extract the full name of the needed file
# (this step is needed because part of the filename, the creation timestamp, is
# effectively random).
if cpattern.search(item):
if 'xml' in item:
continue
basename = cpattern.findall(item)[0]
url = ''.join([mainurl, '/', basename])
return basename, url
utils.verbose_out(
'Unable to find remote match for '
'{} at {}'.format(pattern, mainurl), 4)
return None, None
def main():
title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser = GIPSParser(description=title)
group = parser.add_argument_group('archive options')
group.add_argument('--keep',
help='Keep files after adding to archive',
default=False,
action='store_true')
group.add_argument('--recursive',
help='Iterate through subdirectories',
default=False,
action='store_true')
group.add_argument(
'--update',
help=
'Update asset if newer version available, (must call gips_process to regenerate products',
default=False,
action='store_true')
group.add_argument(
'--path',
default='.',
help='Path to search for files to archive, defaults to `.`')
args = parser.parse_args()
utils.gips_script_setup(None, args.stop_on_error)
with utils.error_handler('Data archive error'):
print title
cls = import_data_class(args.command)
orm.setup() # set up DB orm in case it's needed for Asset.archive()
archived_assets = cls.archive_assets(args.path, args.recursive,
args.keep, args.update)
# if DB inventory is enabled, update it to contain the newly archived assets
if orm.use_orm():
for a in archived_assets:
dbinv.update_or_add_asset(asset=a.asset,
sensor=a.sensor,
tile=a.tile,
date=a.date,
name=a.archived_filename,
driver=cls.name.lower())
utils.gips_exit()
def main():
title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF
parser0 = GIPSParser(datasources=False, description=title)
parser0.add_projdir_parser()
group = parser0.add_argument_group('masking options')
args = parser0.parse_args()
utils.gips_script_setup(stop_on_error=args.stop_on_error)
print title
# TODO - check that at least 1 of filemask or pmask is supplied
header = ['date', 'band', 'min', 'max', 'mean', 'sd', 'skew', 'count']
with utils.error_handler():
for projdir in args.projdir:
VerboseOut('Stats for Project directory: %s' % projdir, 1)
inv = ProjectInventory(projdir, args.products)
p_dates = {} # map each product to its list of valid dates
for date in inv.dates:
for p in inv.products(date):
p_dates.setdefault(p, []).append(date)
p_dates = {p: sorted(dl) for p, dl in p_dates.items()}
for p_type, valid_dates in p_dates.items():
stats_fn = os.path.join(projdir, p_type + '_stats.txt')
with open(stats_fn, 'w') as stats_fo:
sf = getattr(utils.settings(), 'STATS_FORMAT', {})
writer = csv.writer(stats_fo, **sf)
writer.writerow(header)
# print date, band description, and stats
for date in valid_dates:
img = inv[date].open(p_type)
date_str = date.strftime('%Y-%j')
utils.verbose_out('Computing stats for {} {}'.format(
p_type, date_str), 2)
for b in img:
stats = [str(s) for s in b.Stats()]
writer.writerow(
[date_str, b.Description()] + stats)
img = None
utils.gips_exit() # produce a summary error report then quit with a proper exit status
def mosaic(self, datadir, res=None, interpolation=0, crop=False,
overwrite=False, alltouch=False):
"""For each product, combine its tiles into a single mosaic.
Warp if res provided."""
if self.spatial.site is None:
raise Exception('Site required for creating mosaics')
start = datetime.now()
bname = self.date.strftime('%Y%j')
# look in each Data() and dig out its (sensor, product_type) pairs
sp_pile = [(s, p) for d in self.tiles.values() for (s, p) in d.filenames
if p in self.products.products]
# work on each product in turn, gathering up filenames as needed
for (sensor, product) in sp_pile:
# create data directory when it is needed
mkdir(datadir)
# TODO - this is assuming a tif file. Use gippy FileExtension function when it is exposed
fn = '{}_{}_{}.tif'.format(bname, sensor, product)
final_fp = os.path.join(datadir, fn)
if not os.path.exists(final_fp) or overwrite:
err_msg = ("Error mosaicking " + final_fp + ". Did you forget"
" to specify a resolution (`--res x x`)?")
with utils.error_handler(err_msg, continuable=True), \
utils.make_temp_dir(dir=datadir,
prefix='mosaic') as tmp_dir:
tmp_fp = os.path.join(tmp_dir, fn) # for safety
filenames = [self.tiles[t].filenames[(sensor, product)]
for t in self.tiles
if (sensor, product) in self.tiles[t].filenames
]
images = gippy.GeoImages(filenames)
if self.spatial.rastermask is not None:
gridded_mosaic(images, tmp_fp,
self.spatial.rastermask, interpolation)
elif self.spatial.site is not None and res is not None:
CookieCutter(
images, self.spatial.site, tmp_fp, res[0], res[1],
crop, interpolation, {}, alltouch,
)
else:
mosaic(images, tmp_fp, self.spatial.site)
os.rename(tmp_fp, final_fp)
t = datetime.now() - start
VerboseOut('%s: created project files for %s tiles in %s' % (self.date, len(self.tiles), t), 2)
def main():
title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % (
gipsversion) + Colors.OFF
# argument parsing
parser = GIPSParser(description=title)
args = parser.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
cls.print_products()
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def setup():
"""Set settings module default and run django.setup().
Prevent this from happening more than once using a global guard."""
global setup_complete
if setup_complete:
return
if use_orm():
busted_drivers = 'sarannual',
if driver_for_dbinv_feature_toggle in busted_drivers:
raise Exception("Inventory database does not support '{}'. Set"
" GIPS_ORM = False to use the filesystem inventory"
" instead.".format(driver_for_dbinv_feature_toggle))
with utils.error_handler("Error initializing Django ORM"):
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gips.inventory.orm.settings")
django.setup()
setup_complete = True
def readoutput(self, bandnum):
with open('band' + str(bandnum) + '.chn') as f:
lines = f.readlines()
data = lines[4 + bandnum]
# Get nominal band width in microns
bandwidth = float(data[85:94]) / 1000
# Convert from W/sr-cm2 to W/sr-m2-um
Lu = (float(data[59:72]) * 10000) / bandwidth
trans = float(data[239:248])
Ld = 0.0
with utils.error_handler('Error calculating Ld, falling back to default (0.0)',
continuable=True):
with open('band' + str(bandnum) + 'Ld.chn') as f:
lines = f.readlines()
data = lines[4 + bandnum]
# Convert channel radiance to spectral radiance
Ld = (float(data[59:72]) * 10000) / bandwidth
return [trans, Lu, Ld]
def __init__(self, dates=None, days=None):
""" Create temporal extent object from string input """
if dates is None:
dates = '1950,2050'
if days is None:
days = (1, 366)
else:
days = days.split(',')
days = (int(days[0]), int(days[1]))
with utils.error_handler('Bad date specification'):
if ',' not in dates:
dates = (self._parse_date(dates), self._parse_date(dates, True))
else:
(d1, d2) = dates.replace(',', ' ').split()
dates = (self._parse_date(d1), self._parse_date(d2, True))
self.datebounds = dates
self.daybounds = days
self.datearray = []
def query_provider(cls, asset, tile, date):
"""Find out from the NASA servers what AOD assets are available."""
if asset not in cls._assets:
raise ValueError('{} has no defined asset for {}'.format(
cls.Repository.name, asset))
with utils.error_handler("Error querying " + cls._host,
continuable=True):
# example full json query URL:
# https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD08_D3/2017/145.json
url_head = cls._assets[asset]['url'] + date.strftime('%Y/%j')
query_url = url_head + '.json'
utils.verbose_out('Downloading ' + query_url, 5)
resp = requests.get(query_url)
resp.raise_for_status() # some errors don't raise otherwise
found_assets = resp.json()
if len(found_assets) != 1:
raise ValueError('Expected one {} asset, found {}'.format(
asset, len(resp)))
filename = str(found_assets[0]['name']) # str() to prevent unicode
return filename, url_head + '/' + filename
return None, None
def fetch(cls, asset, tile, date):
"""Standard Asset.fetch implementation for downloading assets."""
asset_info = cls.query_service(asset, tile, date)
if asset_info is None:
return []
basename, url = asset_info['basename'], asset_info['url']
basename = asset_info['basename']
url = asset_info['url']
outpath = os.path.join(cls.Repository.path('stage'), basename)
with utils.error_handler("Error fetching {} from {}".format(
basename, url),
continuable=True):
# obtain the data
response = cls.Repository.managed_request(url)
if response is None:
return []
stage_dir = cls.Repository.path('stage')
with utils.make_temp_dir(prefix='fetch', dir=stage_dir) as tmp_dn:
tmp_outpath = os.path.join(tmp_dn, basename)
with open(tmp_outpath, 'w') as fd:
fd.write(response.read())
# verify that it is a netcdf file
try:
ncroot = Dataset(tmp_outpath)
os.rename(tmp_outpath, outpath)
except Exception as e:
text = ''
if e.message.endswith('Unknown file format'):
token = 'Authorize NASA GESDISC DATA ARCHIVE'
html = open(tmp_outpath, 'r').read(100000)
if token in html:
text = ('\n\nYou need to {t} \nfor your NASA '
'EarthData login.\n').format(t=token)
raise Exception(e.message + text)
utils.verbose_out('Retrieved ' + basename, 2)
return [outpath]
def fetch(cls, asset, tile, date):
utils.verbose_out('%s: fetch tile %s for %s' % (asset, tile, date), 3)
qs_rv = cls.query_service(asset, tile, date)
if qs_rv is None:
return []
asset_fn = qs_rv['basename']
with utils.error_handler("Error downloading from " +
cls._assets[asset]['host'],
continuable=True):
ftp = cls.ftp_connect(
asset, date) # starts chdir'd to the right directory
stage_dir_fp = cls.Repository.path('stage')
stage_fp = os.path.join(stage_dir_fp, asset_fn)
with utils.make_temp_dir(prefix='fetchtmp',
dir=stage_dir_fp) as td_name:
temp_fp = os.path.join(td_name, asset_fn)
utils.verbose_out("Downloading " + asset_fn, 2)
with open(temp_fp, "wb") as temp_fo:
ftp.retrbinary('RETR ' + asset_fn, temp_fo.write)
ftp.quit()
os.rename(temp_fp, stage_fp)
return [stage_fp]
return []
def __init__(self, projdir='', products=[]):
""" Create inventory of a GIPS project directory """
self.projdir = os.path.abspath(projdir)
if not os.path.exists(self.projdir):
raise Exception('Directory %s does not exist!' % self.projdir)
self.data = {}
product_set = set()
sensor_set = set()
with utils.error_handler("Project directory error for " +
self.projdir):
# can't import Data at module scope due to circular dependencies
from gips.data.core import Data
for dat in Data.discover(self.projdir):
self.data[dat.date] = dat
# All products and sensors used across all dates
product_set = product_set.union(dat.product_set)
sensor_set = sensor_set.union(dat.sensor_set)
if not products:
products = list(product_set)
self.requested_products = products
self.sensors = sensor_set
def main():
import gips
title = 'GIPS Configuration Utility (v%s)' % (version)
parser = GIPSParser(description=title, datasources=False)
subparser = parser.add_subparsers(dest='command')
subparser.add_parser('print', help='Print current settings')
p = subparser.add_parser(
'env', help='Configure GIPS repositories in this environment')
p.add_argument('-r',
'--repos',
help='Top level directory for repositories',
default='/data/repos')
p.add_argument('-e',
'--email',
help='Set email address (used for anonymous FTP sources)',
default='')
p = subparser.add_parser(
'user',
help=
'Configure GIPS repositories for this user (for per user customizations)'
)
#p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)')
#h = 'Install full configuration file without inheriting from environment settings'
#p.add_argument('-f', '--full', help=h, default=False, action='store_true')
args = parser.parse_args()
print title
utils.gips_script_setup(
driver_string=None, # NOTE: no driver string for gips_config
stop_on_error=args.stop_on_error,
setup_orm=False, # NOTE: ORM cannot be setup before `gips_config env`
) # has been run
if args.command == 'print':
with utils.error_handler('Unable to access settings'):
from gips.utils import settings
s = settings()
for v in dir(s):
if not v.startswith('__') and v != 'gips':
print
print v
exec('pprint.pprint(s.%s)' % v)
elif args.command == 'env':
with utils.error_handler('Could not create environment settings'):
created_cf, cfgfile = create_environment_settings(args.repos,
email=args.email)
elif args.command == 'user':
with utils.error_handler('Could not create user settings'):
# first try importing environment settings
import gips.settings
created_cf, cfgfile = create_user_settings()
if args.command in ('user', 'env'):
msg = ('Wrote new config file: {}.' if created_cf else
'Found existing config, left unmodified: {}.')
print msg.format(cfgfile)
with utils.error_handler('Could not create repos'):
print 'Creating repository directories, if needed.'
try:
create_repos()
except:
if created_cf:
print(
'Error; removing (likely broken) config file:'
' {}.'.format(cfgfile))
os.remove(cfgfile)
raise
with utils.error_handler('Could not migrate database'):
migrate_database()
utils.gips_exit()
def get_aod(cls, lat, lon, date, fetch=True):
"""Returns an aod value for the given lat/lon.
If the pixel has a no-data value, nearby values are averaged. If
no nearby values are available, it makes an estimate using
long-term averages.
"""
pixx = int(numpy.round(float(lon) + 179.5))
pixy = int(numpy.round(89.5 - float(lat)))
roi = gippy.Recti(pixx - 1, pixy - 1, 3, 3)
# try reading actual data file first
aod = numpy.nan
with utils.error_handler('Unable to load aod values',
continuable=True):
# this is just for fetching the data
inv = cls.inventory(dates=date.strftime('%Y-%j'),
fetch=fetch,
products=['aod'])
img = inv[date].tiles[cls.Asset.Repository._the_tile].open('aod')
vals = img[0].Read(roi)
# TODO - do this automagically in swig wrapper
vals[numpy.where(vals == img[0].NoDataValue())] = numpy.nan
aod = vals[1, 1]
img = None
source = 'MODIS (MOD08_D3)'
# if invalid center but valid vals exist in 3x3
if numpy.isnan(aod) and numpy.any(~numpy.isnan(vals)):
aod = numpy.mean(vals[~numpy.isnan(vals)])
source = 'MODIS (MOD08_D3) spatial average'
# Calculate best estimate from multiple sources
if numpy.isnan(aod):
day = date.strftime('%j')
repo = cls.Asset.Repository
cpath = repo.path('composites')
nodata = -32768
source = 'Weighted estimate using MODIS LTA values'
def _calculate_estimate(filename):
val, var = cls._read_point(filename, roi, nodata)
aod = numpy.nan
norm = numpy.nan
# Negative values don't make sense
if val < 0:
val = 0
if var == 0:
# There is only one observation, so make up
# the variance.
if val == 0:
var = 0.15
else:
var = val / 2
if not numpy.isnan(val) and not numpy.isnan(var):
aod = val / var
norm = 1.0 / var
utils.verbose_out('AOD: LTA-Daily = %s, %s' % (val, var),
3)
return aod, norm
# LTA-Daily
filename = os.path.join(cpath, 'ltad',
'ltad%s.tif' % str(day).zfill(4))
daily_aod, daily_norm = _calculate_estimate(filename)
# LTA
lta_aod, lta_norm = _calculate_estimate(
os.path.join(cpath, 'lta.tif'))
if numpy.isnan(lta_aod):
raise Exception("Could not retrieve AOD")
aod = lta_aod
norm = lta_norm
if not numpy.isnan(daily_aod):
aod = aod + daily_aod
norm = norm + daily_norm
# TODO - adjacent days
# Final AOD estimate
aod = aod / norm
utils.verbose_out('AOD: Source = %s Value = %s' % (source, aod), 2)
return (source, aod)
def get_bil_vsifile(d, a):
with utils.error_handler('Error accessing asset {}'.format(d),
continuable=True):
return os.path.join('/vsizip/' + d.assets[a].filename,
d.assets[a].datafiles()[0])
def process(self, *args, **kwargs):
"""Deduce which products need producing, then produce them."""
products = super(prismData, self).process(*args, **kwargs)
if len(products) == 0:
return
# overwrite = kwargs.get('overwrite', False)
# utils.verbose_out('\n\noverwrite = {}\n'.format(overwrite), 2)
# TODO: overwrite doesn't play well with pptsum -- wonder if it would
# if it was made into a composite product (which it is)
assert len(prismAsset._sensors
) == 1 # sanity check to force this code to stay current
sensor = prismAsset._sensors.keys()[0]
def get_bil_vsifile(d, a):
with utils.error_handler('Error accessing asset {}'.format(d),
continuable=True):
return os.path.join('/vsizip/' + d.assets[a].filename,
d.assets[a].datafiles()[0])
for key, val in products.requested.items():
start = datetime.now()
# check that we have required assets
requiredassets = self.products2assets([val[0]])
# val[0] s.b. key w/o product args
description = self._products['pptsum']['description']
missingassets = []
availassets = []
vsinames = {}
for asset in requiredassets:
bil = get_bil_vsifile(self, asset)
if bil is None:
missingassets.append(asset)
else:
availassets.append(asset)
vsinames[asset] = os.path.join(
'/vsizip/' + self.assets[asset].filename, bil)
if not availassets:
utils.verbose_out(
'There are no available assets ({}) on {} for tile {}'.
format(str(missingassets), str(self.date), str(self.id)),
5,
)
continue
prod_fn = '{}_{}_{}.tif'.format(self.basename, 'prism', key)
archived_fp = os.path.join(self.path, prod_fn) # final destination
if val[0] in ['ppt', 'tmin', 'tmax']:
with self.make_temp_proc_dir() as tmp_dir:
tmp_fp = os.path.join(tmp_dir, prod_fn)
os.symlink(vsinames[self._products[key]['assets'][0]],
tmp_fp)
os.rename(tmp_fp, archived_fp)
elif val[0] == 'pptsum':
if len(val) < 2:
lag = 3 # no argument provided, use default lag of 3 days SB configurable.
prod_fn = re.sub(r'\.tif$', '-{}.tif'.format(lag), prod_fn)
archived_fp = os.path.join(
self.path, prod_fn) # have to regenerate, sigh
utils.verbose_out(
'Using default lag of {} days.'.format(lag), 2)
else:
with utils.error_handler(
"Error for pptsum lag value '{}').".format(
val[1])):
lag = int(val[1])
date_spec = '{},{}'.format(
datetime.strftime(
self.date - timedelta(days=lag),
'%Y-%m-%d',
),
datetime.strftime(self.date, '%Y-%m-%d'),
)
inv = self.inventory(
dates=date_spec,
products=['ppt'],
)
inv.process()
# because DataInventory object doesn't update
inv = self.inventory(
dates=date_spec,
products=['ppt'],
)
if len(inv.data) < lag:
utils.verbose_out(
'{}: requires {} preceding days ppt ({} found).'.
format(key, lag, len(inv.data)),
3,
)
continue # go to next product to process
imgs = []
asset_fns = [] # have to grab filenames for multiple days
for tileobj in inv.data.values():
datobj = tileobj.tiles.values()[0]
asset_fns.append(
os.path.basename(datobj.assets['_ppt'].filename))
imgs.append(GeoImage(get_bil_vsifile(datobj, '_ppt')))
with self.make_temp_proc_dir() as tmp_dir:
tmp_fp = os.path.join(tmp_dir, prod_fn)
oimg = GeoImage(tmp_fp, imgs[0])
oimg.SetNoData(-9999)
oimg.SetBandName(
description + '({} day window)'.format(lag), 1)
oimg.SetMeta(self.prep_meta(sorted(asset_fns)))
for chunk in oimg.Chunks():
oarr = oimg[0].Read(chunk) * 0.0 # wat
for img in imgs:
oarr += img[0].Read(chunk)
oimg[0].Write(oarr, chunk)
oimg.Process()
os.rename(tmp_fp, archived_fp)
oimg = None # help swig+gdal with GC
products.requested.pop(key)
self.AddFile(sensor, key, archived_fp) # add product to inventory
return products
def process(self, *args, **kwargs):
"""Produce requested products."""
products = super(modisData, self).process(*args, **kwargs)
if len(products) == 0:
return
bname = os.path.join(self.path, self.basename)
# example products.requested:
# {'temp8tn': ['temp8tn'], 'clouds': ['clouds'], . . . }
# key is only used once far below, and val is only used for val[0].
for key, val in products.requested.items():
# TODO replace val[0] below with this more meaningful name
prod_type = val[0]
if prod_type in self._productgroups['Index']:
continue # indices handled differently below
start = datetime.datetime.now()
asset, version, missingassets, availassets, allsds = \
self.asset_check(prod_type)
if not availassets:
# some products aren't available for every day but this is trying every day
VerboseOut(
'There are no available assets (%s) on %s for tile %s' % (
str(missingassets),
str(self.date),
str(self.id),
), 5)
continue
meta = {
'AVAILABLE_ASSETS': ' '.join(availassets)
} # TODO obselete?
a_fnames = [self.assets[at].filename for at in availassets]
sensor = self._products[prod_type]['sensor']
fname = self.temp_product_filename(
sensor, prod_type) # moved to archive at end of loop
if val[0] == "landcover":
os.symlink(allsds[0], fname)
imgout = gippy.GeoImage(fname)
if val[0] == "quality":
if version != 6:
raise Exception('product version not supported')
os.symlink(allsds[0], fname)
imgout = gippy.GeoImage(fname)
# LAND VEGETATION INDICES PRODUCT
# now with QC layer!
if val[0] == "indices":
depr_msg = (
"'indices' is deprecated, and may be removed in"
" the future. See the Index product group instead.")
utils.verbose_out(depr_msg, 2, stream=sys.stderr)
VERSION = "2.0"
meta['VERSION'] = VERSION
refl = gippy.GeoImage(allsds)
missing = 32767
if version == 6:
redimg = refl[7].Read()
nirimg = refl[8].Read()
bluimg = refl[9].Read()
grnimg = refl[10].Read()
mirimg = refl[11].Read()
swrimg = refl[12].Read() # swir1, formerly swir2
redqcimg = refl[0].Read()
nirqcimg = refl[1].Read()
bluqcimg = refl[2].Read()
grnqcimg = refl[3].Read()
mirqcimg = refl[4].Read()
swrqcimg = refl[5].Read()
else:
raise Exception('product version not supported')
# wherever the value is too small, set it to a minimum of 0
redimg[redimg < 0.0] = 0.0
nirimg[nirimg < 0.0] = 0.0
bluimg[bluimg < 0.0] = 0.0
grnimg[grnimg < 0.0] = 0.0
mirimg[mirimg < 0.0] = 0.0
swrimg[swrimg < 0.0] = 0.0
# wherever the value is too saturated, set it to a max of 1.0
redimg[(redimg != missing) & (redimg > 1.0)] = 1.0
nirimg[(nirimg != missing) & (nirimg > 1.0)] = 1.0
bluimg[(bluimg != missing) & (bluimg > 1.0)] = 1.0
grnimg[(grnimg != missing) & (grnimg > 1.0)] = 1.0
mirimg[(mirimg != missing) & (mirimg > 1.0)] = 1.0
swrimg[(swrimg != missing) & (swrimg > 1.0)] = 1.0
# red, nir
# first setup a blank array with everything set to missing
ndvi = missing + np.zeros_like(redimg)
# compute the ndvi only where neither input is missing, AND
# no divide-by-zero error will occur
wg = np.where((redimg != missing) & (nirimg != missing)
& (redimg + nirimg != 0.0))
ndvi[wg] = (nirimg[wg] - redimg[wg]) / (nirimg[wg] +
redimg[wg])
# nir, mir
lswi = missing + np.zeros_like(redimg)
wg = np.where((nirimg != missing) & (mirimg != missing)
& (nirimg + mirimg != 0.0))
lswi[wg] = (nirimg[wg] - mirimg[wg]) / (nirimg[wg] +
mirimg[wg])
# blu, grn, red
vari = missing + np.zeros_like(redimg)
wg = np.where((grnimg != missing) & (redimg != missing)
& (bluimg != missing)
& (grnimg + redimg - bluimg != 0.0))
vari[wg] = (grnimg[wg] - redimg[wg]) / (
grnimg[wg] + redimg[wg] - bluimg[wg])
# blu, grn, red, nir
brgt = missing + np.zeros_like(redimg)
wg = np.where((nirimg != missing) & (redimg != missing)
& (bluimg != missing) & (grnimg != missing))
brgt[wg] = (0.3 * bluimg[wg] + 0.3 * redimg[wg] +
0.1 * nirimg[wg] + 0.3 * grnimg[wg])
# red, mir, swr
satvi = missing + np.zeros_like(redimg)
wg = np.where((redimg != missing) & (mirimg != missing)
& (swrimg != missing)
& ((mirimg + redimg + 0.5) != 0.0))
satvi[wg] = (((mirimg[wg] - redimg[wg]) /
(mirimg[wg] + redimg[wg] + 0.5)) *
1.5) - (swrimg[wg] / 2.0)
# blu, red, nir
evi = missing + np.zeros_like(redimg)
wg = np.where(
(bluimg != missing) & (redimg != missing)
& (nirimg != missing)
& (nirimg + 6.0 * redimg - 7.5 * bluimg + 1.0 != 0.0))
evi[wg] = (
(2.5 * (nirimg[wg] - redimg[wg])) /
(nirimg[wg] + 6.0 * redimg[wg] - 7.5 * bluimg[wg] + 1.0))
qc = np.ones_like(
redimg
) # mark as poor if all are not missing and not all are good
w0 = np.where((redqcimg == 0) & (nirqcimg == 0)
& (bluqcimg == 0) & (grnqcimg == 0)
& (mirqcimg == 0) & (swrqcimg == 0))
w255 = np.where((redqcimg == 255) | (nirqcimg == 255)
| (bluqcimg == 255) | (grnqcimg == 255)
| (mirqcimg == 255) | (swrqcimg == 255))
qc[w0] = 0 # mark as good if they are all good
qc[w255] = missing # mark as missing if any are missing
# create output gippy image
print("writing", fname)
imgout = gippy.GeoImage(fname, refl, gippy.GDT_Int16, 7)
del refl
imgout.SetNoData(missing)
imgout.SetOffset(0.0)
imgout.SetGain(0.0001)
imgout[6].SetGain(1.0)
imgout[0].Write(ndvi)
imgout[1].Write(lswi)
imgout[2].Write(vari)
imgout[3].Write(brgt)
imgout[4].Write(satvi)
imgout[5].Write(evi)
imgout[6].Write(qc)
imgout.SetBandName('NDVI', 1)
imgout.SetBandName('LSWI', 2)
imgout.SetBandName('VARI', 3)
imgout.SetBandName('BRGT', 4)
imgout.SetBandName('SATVI', 5)
imgout.SetBandName('EVI', 6)
imgout.SetBandName('QC', 7)
if val[0] == "clouds":
# cloud mask product
meta['VERSION'] = '1.0'
img = gippy.GeoImage(allsds)
data = img[0].Read()
clouds = np.zeros_like(data)
# See table 3 in the user guide:
# https://nsidc.org/sites/nsidc.org/files/files/
# MODIS-snow-user-guide-C6.pdf
nodata = 127
for v in [200, 201, 211, 254, 255]:
clouds[data == v] = nodata
clouds[data == 237] = 0
clouds[data == 239] = 0
clouds[data == 250] = 1
# create output gippy image
imgout = gippy.GeoImage(fname, img, gippy.GDT_Byte, 1)
del img
imgout.SetNoData(nodata)
imgout.SetOffset(0.0)
imgout.SetGain(1.0)
imgout.SetBandName('Cloud Cover', 1)
imgout[0].Write(clouds)
if val[0] in ('snow', 'fsnow'):
# (fsnow was removed entirely due to being a big copypasta
# of the snow block; what follows is snow)
raise NotImplementedError("not compatible with collection 6; "
"use NDSI instead")
VERSION = "1.0"
meta['VERSION'] = VERSION
if not missingassets:
availbands = [0, 1]
snowsds = [allsds[0], allsds[3], allsds[4], allsds[7]]
elif missingassets[0] == 'MYD10A1':
availbands = [0]
snowsds = [allsds[0], allsds[3]]
elif missingassets[0] == 'MOD10A1':
availbands = [1]
snowsds = [allsds[0], allsds[3]]
else:
raise IOError("Missing both MOD10A1 and MYD10A1; can't "
"continue")
img = gippy.GeoImage(snowsds)
# there are two snow bands
for iband, band in enumerate(availbands):
# get the data values for both bands
# for both MOD10A1 and MYD10A1, bands 0 & 3 are --v
cover = img[2 * iband].Read() # Snow_Cover_Daily_Tile
frac = img[2 * iband + 1].Read() # Fractional_Snow_Cover
# check out frac
# meanings of special values, see C5 user guide, table 4:
# https://modis-snow-ice.gsfc.nasa.gov/uploads/sug_c5.pdf
wbad1 = np.where((frac == 200) | (frac == 201)
| (frac == 211) | (frac == 250)
| (frac == 254) | (frac == 255))
wsurface1 = np.where((frac == 225) | (frac == 237)
| (frac == 239))
wvalid1 = np.where((frac >= 0) & (frac <= 100))
nbad1 = len(wbad1[0])
nsurface1 = len(wsurface1[0])
nvalid1 = len(wvalid1[0])
assert nbad1 + nsurface1 + nvalid1 == frac.size, "frac contains invalid values"
# check out cover
# meanings of special values, see C5 user guide, table 3:
# https://modis-snow-ice.gsfc.nasa.gov/uploads/sug_c5.pdf
wbad2 = np.where((cover == 0) | (cover == 1)
| (cover == 11) | (cover == 50)
| (cover == 254) | (cover == 255))
wsurface2 = np.where((cover == 25) | (cover == 37)
| (cover == 39))
wvalid2 = np.where((cover == 100) | (cover == 200))
nbad2 = len(wbad2[0])
nsurface2 = len(wsurface2[0])
nvalid2 = len(wvalid2[0])
assert nbad2 + nsurface2 + nvalid2 == cover.size, "cover contains invalid values"
# assign output data here
coverout = np.zeros_like(cover, dtype=np.uint8)
fracout = np.zeros_like(frac, dtype=np.uint8)
fracout[wvalid1] = frac[wvalid1]
fracout[wsurface1] = 0
fracout[wbad1] = 127
coverout[wvalid2] = 100
coverout[wsurface2] = 0
coverout[wbad2] = 127
if len(availbands) == 2:
if iband == 0:
fracout1 = np.copy(fracout)
coverout1 = np.copy(coverout)
else:
# both the current and previous are valid
w = np.where((fracout != 127) & (fracout1 != 127))
fracout[w] = np.mean(np.array(
[fracout[w], fracout1[w]]),
axis=0).astype('uint8')
# the current is not valid but previous is valid
w = np.where((fracout == 127) & (fracout1 != 127))
fracout[w] = fracout1[w]
# both the current and previous are valid
w = np.where((coverout != 127)
& (coverout1 != 127))
coverout[w] = np.mean(np.array(
[coverout[w], coverout1[w]]),
axis=0).astype('uint8')
# the current is not valid but previous is valid
w = np.where((coverout == 127)
& (coverout1 != 127))
coverout[w] = coverout1[w]
fracmissingcoverclear = np.sum((fracout == 127)
& (coverout == 0))
fracmissingcoversnow = np.sum((fracout == 127)
& (coverout == 100))
fracclearcovermissing = np.sum((fracout == 0)
& (coverout == 127))
fracclearcoversnow = np.sum((fracout == 0) & (coverout == 100))
fracsnowcovermissing = np.sum((fracout > 0) & (fracout <= 100)
& (coverout == 127))
fracsnowcoverclear = np.sum((fracout > 0) & (fracout <= 100)
& (coverout == 0))
# fracmostlycoverclear = np.sum((fracout > 50) & (fracout <= 100) & (coverout == 0))
totsnowfrac = int(0.01 * np.sum(fracout[fracout <= 100]))
totsnowcover = int(0.01 * np.sum(coverout[coverout <= 100]))
numvalidfrac = np.sum(fracout != 127)
numvalidcover = np.sum(coverout != 127)
if totsnowcover == 0 or totsnowfrac == 0:
print("no snow or ice: skipping", str(self.date),
str(self.id), str(missingassets))
meta['FRACMISSINGCOVERCLEAR'] = fracmissingcoverclear
meta['FRACMISSINGCOVERSNOW'] = fracmissingcoversnow
meta['FRACCLEARCOVERMISSING'] = fracclearcovermissing
meta['FRACCLEARCOVERSNOW'] = fracclearcoversnow
meta['FRACSNOWCOVERMISSING'] = fracsnowcovermissing
meta['FRACSNOWCOVERCLEAR'] = fracsnowcoverclear
meta['FRACMOSTLYCOVERCLEAR'] = np.sum((fracout > 50)
& (fracout <= 100)
& (coverout == 0))
meta['TOTSNOWFRAC'] = totsnowfrac
meta['TOTSNOWCOVER'] = totsnowcover
meta['NUMVALIDFRAC'] = numvalidfrac
meta['NUMVALIDCOVER'] = numvalidcover
# create output gippy image
imgout = gippy.GeoImage(fname, img, gippy.GDT_Byte, 2)
del img
imgout.SetNoData(127)
imgout.SetOffset(0.0)
imgout.SetGain(1.0)
imgout.SetBandName('Snow Cover', 1)
imgout.SetBandName('Fractional Snow Cover', 2)
imgout[0].Write(coverout)
imgout[1].Write(fracout)
###################################################################
# TEMPERATURE PRODUCT (DAILY)
if val[0] == "temp":
VERSION = "1.1"
meta['VERSION'] = VERSION
if not missingassets:
availbands = [0, 1, 2, 3]
tempsds = [allsds[0], allsds[4], allsds[12], allsds[16]]
qcsds = [allsds[1], allsds[5], allsds[13], allsds[17]]
hoursds = [allsds[2], allsds[6], allsds[14], allsds[18]]
elif missingassets[0] == 'MYD11A1':
availbands = [0, 1]
tempsds = [allsds[0], allsds[4]]
qcsds = [allsds[1], allsds[5]]
hoursds = [allsds[2], allsds[6]]
elif missingassets[0] == 'MOD11A1':
availbands = [2, 3]
tempsds = [allsds[0], allsds[4]]
qcsds = [allsds[1], allsds[5]]
hoursds = [allsds[2], allsds[6]]
else:
raise IOError("Missing both MOD11A1 and MYD11A1; can't "
"continue")
tempbands = gippy.GeoImage(tempsds)
qcbands = gippy.GeoImage(qcsds)
hourbands = gippy.GeoImage(hoursds)
imgout = gippy.GeoImage(fname, tempbands, gippy.GDT_UInt16, 5)
imgout.SetNoData(65535)
imgout.SetGain(0.02)
# there are four temperature bands
for iband, band in enumerate(availbands):
# get meta name template info
basename = tempbands[iband].Basename()
platform = self.Asset._sensors[basename[:3]]['description']
if basename.find('daytime'):
dayornight = 'day'
elif basename.find('nighttime'):
dayornight = 'night'
else:
raise Exception(
'%s appears to be an invalid MODIS temperature project'
% basename)
qc = qcbands[iband].Read()
# first two bits are 10 or 11
newmaskbad = binmask(qc, 2)
# first two bits are 00 or 01
newmaskgood = ~binmask(qc, 2)
# first two bits are 00
newmaskbest = ~binmask(qc, 1) & ~binmask(qc, 2)
if iband == 0:
bestmask = np.zeros_like(qc, dtype='uint16')
bestmask += (math.pow(2, band) *
newmaskbest).astype('uint16')
numbad = np.sum(newmaskbad)
# fracbad = np.sum(newmaskbad) / float(newmaskbad.size)
numgood = np.sum(newmaskgood)
# fracgood = np.sum(newmaskgood) / float(newmaskgood.size)
assert numgood == qc.size - numbad
numbest = np.sum(newmaskbest)
# fracbest = np.sum(newmaskbest) / float(newmaskbest.size)
metaname = "NUMBAD_%s_%s" % (dayornight, platform)
metaname = metaname.upper()
# print "metaname", metaname
meta[metaname] = str(numbad)
metaname = "NUMGOOD_%s_%s" % (dayornight, platform)
metaname = metaname.upper()
# print "metaname", metaname
meta[metaname] = str(numgood)
metaname = "NUMBEST_%s_%s" % (dayornight, platform)
metaname = metaname.upper()
# print "metaname", metaname
meta[metaname] = str(numbest)
# overpass time
hournodatavalue = hourbands[iband].NoDataValue()
hour = hourbands[iband].Read()
hour = hour[hour != hournodatavalue]
hourmean = 0
with utils.error_handler(
"Couldn't compute hour mean for " + fname,
continuable=True):
hourmean = hour.mean()
metaname = "MEANOVERPASSTIME_%s_%s" % (dayornight,
platform)
metaname = metaname.upper()
meta[metaname] = str(hourmean)
tempbands[iband].Process(imgout[band])
imgout[4].SetGain(1.0)
imgout[4].Write(bestmask)
imgout.SetBandName('Temperature Daytime Terra', 1)
imgout.SetBandName('Temperature Nighttime Terra', 2)
imgout.SetBandName('Temperature Daytime Aqua', 3)
imgout.SetBandName('Temperature Nighttime Aqua', 4)
imgout.SetBandName('Temperature Best Quality', 5)
del tempbands
del qcbands
del hourbands
###################################################################
# OBSERVATION TIME PRODUCT (DAILY)
if val[0] == "obstime":
VERSION = "1"
meta['VERSION'] = VERSION
if not missingassets:
availbands = [0, 1, 2, 3]
hoursds = [allsds[2], allsds[6], allsds[14], allsds[18]]
elif missingassets[0] == 'MYD11A1':
availbands = [0, 1]
hoursds = [allsds[2], allsds[6]]
elif missingassets[0] == 'MOD11A1':
availbands = [2, 3]
hoursds = [allsds[2], allsds[6]]
else:
raise IOError("Missing both MOD11A1 and MYD11A1; can't "
"continue")
hourbands = gippy.GeoImage(hoursds)
imgout = gippy.GeoImage(fname, hourbands, gippy.GDT_Byte, 4)
imgout.SetNoData(0)
imgout.SetGain(0.1)
# there are four temperature bands
for iband, band in enumerate(availbands):
# get meta name template info
basename = hourbands[iband].Basename()
platform = self.Asset._sensors[basename[:3]]['description']
if basename.find('daytime'):
dayornight = 'day'
elif basename.find('nighttime'):
dayornight = 'night'
else:
raise Exception(
'%s appears to be an invalid MODIS temperature project'
% basename)
hourbands[iband].Process(imgout[band])
imgout.SetBandName('Observation Time Daytime Terra', 1)
imgout.SetBandName('Observation Time Nighttime Terra', 2)
imgout.SetBandName('Observation Time Daytime Aqua', 3)
imgout.SetBandName('Observation Time Nighttime Aqua', 4)
del hourbands
###################################################################
# NDVI (8-day) - Terra only
if val[0] == "ndvi8":
VERSION = "1.0"
meta['VERSION'] = VERSION
refl = gippy.GeoImage(allsds)
refl.SetBandName("RED", 1)
refl.SetBandName("NIR", 2)
refl.SetNoData(-28762)
fouts = dict(Indices(refl, {'ndvi': fname}, meta))
imgout = gippy.GeoImage(fouts['ndvi'])
del refl
# TEMPERATURE PRODUCT (8-day) - Terra only
if val[0] == "temp8td":
os.symlink(allsds[0], fname)
imgout = gippy.GeoImage(fname)
if val[0] == "temp8tn":
os.symlink(allsds[4], fname)
imgout = gippy.GeoImage(fname)
# set metadata
imgout.SetMeta(self.prep_meta(a_fnames, meta))
# add product to inventory
archive_fp = self.archive_temp_path(fname)
self.AddFile(sensor, key, archive_fp)
del imgout # to cover for GDAL's internal problems
utils.verbose_out(' -> {}: processed in {}'.format(
os.path.basename(fname),
datetime.datetime.now() - start),
level=1)
# process some index products (not all, see above)
requested_ipt = products.groups()['Index'].keys()
if requested_ipt:
model_pt = requested_ipt[0] # all should be similar
asset, version, _, availassets, allsds = self.asset_check(model_pt)
if asset is None:
raise IOError('Found no assets for {}'.format(requested_ipt))
if version < 6:
raise IOError('index products require MCD43A4 version 6,'
' but found {}'.format(version))
img = gippy.GeoImage(allsds[7:]) # don't need the QC bands
# GIPPY uses expected band names to find data:
"""
index (ie img[i])
| band num
| | wavelength name
0 1 620 - 670 RED
1 2 841 - 876 NIR
2 3 459 - 479 BLUE
3 4 545 - 565 GREEN
4 5 1230 - 1250 CIRRUS (not used by gippy)
5 6 1628 - 1652 SWIR1
6 7 2105 - 2155 SWIR2
"""
# SetBandName goes by band number, not index
[
img.SetBandName(name, i)
for (name, i) in [('RED', 1), ('NIR',
2), ('BLUE',
3), ('GREEN',
4), ('SWIR1',
6), ('SWIR2', 7)]
]
sensor = self._products[model_pt]['sensor']
prod_spec = {
pt: self.temp_product_filename(sensor, pt)
for pt in requested_ipt
}
a_fnames = [self.assets[at].filename for at in availassets]
pt_to_temp_fps = Indices(
img, prod_spec, self.prep_meta(a_fnames, {'VERSION': '1.0'}))
for pt, temp_fp in pt_to_temp_fps.items():
archived_fp = self.archive_temp_path(temp_fp)
self.AddFile(sensor, pt, archived_fp)
def process(self, *args, **kwargs):
"""Process requested products."""
products = super(weldData, self).process(*args, **kwargs)
if len(products) == 0:
return
sensor = 'WELD'
for key, val in products.requested.items():
start = datetime.datetime.now()
prod_type = val[0]
fname = self.temp_product_filename(sensor, prod_type) # moved to archive at end of loop
meta = self.meta_dict()
meta['AVAILABLE_ASSETS'] = ''
# Check for asset availability
needed_assets = self._products[val[0]]['assets']
allsds = []
for a in needed_assets:
if a in self.assets:
with utils.error_handler('Error reading datafiles for ' + a, continuable=True):
allsds.extend(self.assets[a].datafiles())
meta['AVAILABLE_ASSETS'] += ' ' + a
if meta['AVAILABLE_ASSETS'] == '':
utils.verbose_out('There are no available assets (%s) on %s for tile %s'
% (str(needed_assets), str(self.date), str(self.id), ), 5)
continue
# SNOW ICE INDEX PRODUCT
if val[0] == "ndsi":
VERSION = "1.0"
meta['VERSION'] = VERSION
refl = gippy.GeoImage(allsds)
# band 2
grnimg = refl[1].Read()
missing = refl[1].NoDataValue()
# band 4
nirimg = refl[3].Read()
assert refl[3].NoDataValue() == missing
# band 5
swrimg = refl[4].Read()
assert refl[4].NoDataValue() == missing
cldimg = refl[11].Read()
# accaimg = refl[13].Read()
ndsi = missing + np.zeros_like(grnimg)
wg = np.where((grnimg != missing) & (swrimg != missing) & (grnimg + swrimg != 0.0) & (cldimg == 0))
ng = len(wg[0])
print "ng", ng
if ng == 0:
continue
ndsi[wg] = (grnimg[wg] - swrimg[wg]) / (grnimg[wg] + swrimg[wg])
print ndsi.min(), ndsi.max()
print ndsi[wg].min(), ndsi[wg].max()
print "writing", fname
imgout = gippy.GeoImage(fname, refl, gippy.GDT_Float32, 1)
imgout.SetNoData(float(missing))
imgout.SetOffset(0.0)
imgout.SetGain(1.0)
imgout.SetProjection(PROJ)
imgout[0].Write(ndsi)
imgout.SetBandName('NDSI', 1)
# SNOW ICE COVER PRODUCT
if val[0] == "snow":
VERSION = "1.0"
meta['VERSION'] = VERSION
refl = gippy.GeoImage(allsds)
# band 2
grnimg = refl[1].Read()
missing = refl[1].NoDataValue()
# band 4
nirimg = refl[3].Read()
assert refl[3].NoDataValue() == missing
# band 5
swrimg = refl[4].Read()
assert refl[4].NoDataValue() == missing
cldimg = refl[11].Read()
accaimg = refl[13].Read()
snow = 127 + np.zeros_like(grnimg)
ndsi = missing + np.zeros_like(grnimg)
wg = np.where((grnimg != missing) & (swrimg != missing) & (grnimg + swrimg != 0.0) & (cldimg == 0))
ng = len(wg[0])
print "ng", ng
if ng == 0:
continue
ndsi[wg] = (grnimg[wg] - swrimg[wg]) / (grnimg[wg] + swrimg[wg])
ws = np.where((ndsi != missing) & (ndsi > 0.4) & (nirimg > 0.11) & (swrimg > 0.1))
wc = np.where((ndsi != missing) & (ndsi > 0.4) & (nirimg <= 0.11) & (swrimg <= 0.1))
ns = len(ws[0])
nc = len(wc[0])
print ng, ns, nc
if (ns > 0):
snow[ws] = 1
if (nc > 0):
snow[wc] = 0
print "writing", fname
imgout = gippy.GeoImage(fname, refl, gippy.GDT_Byte, 1)
imgout.SetNoData(127)
imgout.SetOffset(0.)
imgout.SetGain(1.)
imgout.SetProjection(PROJ)
imgout[0].Write(snow)
imgout.SetBandName('SNOW', 1)
# VEGETATION INDEX PRODUCT
if val[0] == "ndvi":
VERSION = "1.0"
meta['VERSION'] = VERSION
refl = gippy.GeoImage(allsds)
# band 3
redimg = refl[2].Read()
missing = refl[2].NoDataValue()
# band 4
nirimg = refl[3].Read()
assert refl[3].NoDataValue() == missing
cldimg = refl[11].Read()
accaimg = refl[13].Read()
ndvi = missing + np.zeros_like(redimg)
wg = np.where((redimg != missing) & (nirimg != missing) & (redimg + nirimg != 0.0) & (cldimg == 0))
ng = len(wg[0])
print "ng", ng
if ng == 0:
continue
ndvi[wg] = (nirimg[wg] - redimg[wg]) / (nirimg[wg] + redimg[wg])
print ndvi.min(), ndvi.max()
print ndvi[wg].min(), ndvi[wg].max()
print "writing", fname
imgout = gippy.GeoImage(fname, refl, gippy.GDT_Float32, 1)
imgout.SetNoData(float(missing))
imgout.SetOffset(0.0)
imgout.SetGain(1.0)
imgout.SetProjection(PROJ)
imgout[0].Write(ndvi)
imgout.SetBandName('NDVI', 1)
# BRIGHTNESS PRODUCT
if val[0] == "brgt":
VERSION = "1.0"
meta['VERSION'] = VERSION
refl = gippy.GeoImage(allsds)
# band 2
grnimg = refl[1].Read()
missing = refl[1].NoDataValue()
# band 3
redimg = refl[2].Read()
assert refl[2].NoDataValue() == missing
# band 4
nirimg = refl[3].Read()
assert refl[3].NoDataValue() == missing
cldimg = refl[11].Read()
brgt = missing + np.zeros_like(redimg)
wg = np.where((grnimg != missing) & (redimg != missing) & (nirimg != missing) & (cldimg == 0))
ng = len(wg[0])
print "ng", ng
if ng == 0:
continue
brgt[wg] = (grnimg[wg] + redimg[wg] + nirimg[wg])/3.
print brgt.min(), brgt.max()
print brgt[wg].min(), brgt[wg].max()
print "writing", fname
imgout = gippy.GeoImage(fname, refl, gippy.GDT_Float32, 1)
imgout.SetNoData(float(missing))
imgout.SetOffset(0.0)
imgout.SetGain(1.0)
imgout.SetProjection(PROJ)
imgout[0].Write(brgt)
imgout.SetBandName('BRGT', 1)
# set metadata
meta = {k: str(v) for k, v in meta.iteritems()}
imgout.SetMeta(meta)
# add product to inventory
archive_fp = self.archive_temp_path(fname)
self.AddFile(sensor, key, archive_fp)
VerboseOut(' -> %s: processed in %s' % (os.path.basename(fname), datetime.datetime.now() - start), 1)
def main():
title = Colors.BOLD + 'GIPS Data Export (v%s)' % __version__ + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser0.add_inventory_parser(site_required=True)
parser0.add_process_parser()
parser0.add_project_parser()
parser0.add_warp_parser()
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
print title
with utils.error_handler():
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
# create tld: SITENAME--KEY_DATATYPE_SUFFIX
if args.notld:
tld = args.outdir
else:
key = '' if args.key == '' else '--' + args.key
suffix = '' if args.suffix == '' else '_' + args.suffix
res = '' if args.res is None else '_%sx%s' % (args.res[0],
args.res[1])
bname = (extents[0].site.LayerName() + key + res + '_' +
args.command + suffix)
tld = os.path.join(args.outdir, bname)
for extent in extents:
t_extent = TemporalExtent(args.dates, args.days)
inv = DataInventory(cls, extent, t_extent, **vars(args))
datadir = os.path.join(tld, extent.site.Value())
if inv.numfiles > 0:
inv.mosaic(
datadir=datadir,
tree=args.tree,
overwrite=args.overwrite,
res=args.res,
interpolation=args.interpolation,
crop=args.crop,
alltouch=args.alltouch,
process=(not args.dont_process),
)
inv = ProjectInventory(datadir)
inv.pprint()
else:
VerboseOut(
'No data found for {} within temporal extent {}'.format(
str(t_extent), str(t_extent)),
2,
)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
