def fetch(cls, asset, tile, date): """Fetch to the stage. Returns a list with one item, the full path to the staged asset. """ utils.verbose_out('{}: fetch tile {} for {}'.format(asset, tile, date), 3) fn_prefix = cls._assets[asset]['fn-prefix'] stage_dir = cls.Repository.path('stage') with utils.error_handler("Error downloading from " + cls._host, continuable=True), \ utils.make_temp_dir(prefix='fetchtmp', dir=stage_dir) as td_name: qs_rv = cls.query_service(asset, tile, date) if qs_rv is None: return [] remote_fn = qs_rv['basename'] local_fn = fn_prefix + remote_fn temp_fp = os.path.join(td_name, local_fn) stage_fp = os.path.join(stage_dir, local_fn) utils.verbose_out( "Downloading {}, local name {}".format(remote_fn, local_fn), 2) conn = cls.ftp_connect(asset, date) with open(temp_fp, "wb") as temp_fo: conn.retrbinary('RETR ' + remote_fn, temp_fo.write) conn.quit() os.rename(temp_fp, stage_fp) return [stage_fp] return []
def process(self, products=None, overwrite=False, **kwargs): """Produce data products and save them to files. Only one product; it's processed in the usual way, but for this driver, it's extracted from the gzip file and saved (not symlink/vsi). Method signature is largely for campatibilty with the rest of gips, eg kwargs is unused. """ needed_products = self.needed_products(products, overwrite) if len(needed_products) == 0: utils.verbose_out('No new processing required.') return # sanity check that requested product & asset look ok assert (needed_products.requested == { _product_type: [_product_type] } and _asset_type in self.assets) asset = self.assets[_asset_type] err_msg = 'Error creating product {} from {}'.format( _product_type, os.path.basename(asset.filename)) with utils.error_handler(err_msg, continuable=True): temp_fp = self.temp_product_filename(_sensor, _product_type) # make gdal/gippy-readable path to the inner file vsi_inner_path = '/vsigzip/' + asset.filename os.symlink(vsi_inner_path, temp_fp) archive_fp = self.archive_temp_path(temp_fp) self.AddFile(_sensor, _product_type, archive_fp)
def _read_point(cls, filename, roi, nodata): """ Read single point from mean/var file and return if valid, or mean/var of 3x3 neighborhood """ if not os.path.exists(filename): return (numpy.nan, numpy.nan) with utils.error_handler( 'Unable to read point from {}'.format(filename), continuable=True): img = gippy.GeoImage(filename) vals = img[0].Read(roi).squeeze() variances = img[1].Read(roi) vals[numpy.where(vals == nodata)] = numpy.nan variances[numpy.where(variances == nodata)] = numpy.nan val = numpy.nan var = numpy.nan if ~numpy.isnan(vals[1, 1]): val = vals[1, 1] elif numpy.any(~numpy.isnan(vals)): val = numpy.mean(vals[~numpy.isnan(vals)]) if ~numpy.isnan(variances[1, 1]): var = variances[1, 1] elif numpy.any(~numpy.isnan(variances)): var = numpy.mean(variances[~numpy.isnan(variances)]) img = None return (val, var) return (numpy.nan, numpy.nan)
def main(cls): """ Main for algorithm classes """ dhf = argparse.ArgumentDefaultsHelpFormatter # Top level parser parser = argparse.ArgumentParser(formatter_class=dhf, description=cls.info()) parser.add_argument('-v', '--verbose', help='Verbosity - 0: quiet, 1: normal, 2+: debug', default=1, type=int) parser = cls.parser(parser) args = parser.parse_args() gippy.Options.SetVerbose(args.verbose) VerboseOut(cls.info()) utils.gips_script_setup(driver_string=None, setup_orm=False) with utils.error_handler('Error in {}'.format(cls.name)): alg = cls(**vars(args)) alg.run_command(**vars(args)) utils.gips_exit()
def query_provider(cls, asset, tile, date): year, month, day = date.timetuple()[:3] if asset != "ASM": mainurl = "%s/%04d/%02d" % (cls._assets[asset]['url'], year, month) pattern = cls._assets[asset]['re_pattern'] % (year, month, day) else: # asset ASM is for constants which all go into 1980-01-01 mainurl = cls._assets[asset]['url'] pattern = cls._assets[asset]['re_pattern'] % (0, 0, 0) cpattern = re.compile(pattern) with utils.error_handler("Error downloading"): # obtain the list of files response = cls.Repository.managed_request(mainurl, verbosity=2) if response is None: return None, None for item in response.readlines(): # inspect the page and extract the full name of the needed file if cpattern.search(item): if 'xml' in item: continue basename = cpattern.findall(item)[0] url = '/'.join([mainurl, basename]) return basename, url utils.verbose_out( "Unable to find a remote match for" " {} at {}".format(pattern, mainurl), 4) return None, None
def main(): title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser = parser0.add_inventory_parser() group = parser.add_argument_group('additional inventory options') group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False) group.add_argument( '--rectify', help= 'Instead of displaying or fetching inventory, rectify the inventory ' 'database by comparing it against the present state of the data repos.', action='store_true', default=False) args = parser0.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) with utils.error_handler(): print(title) if args.rectify: if not orm.use_orm(): raise ValueError("--rectify can only be used if" " GIPS_ORM = True.") for k, v in vars(args).items(): # Let the user know not to expect other options to effect rectify if v and k not in ('rectify', 'verbose', 'command'): msg = "INFO: Option '--{}' is has no effect on --rectify." utils.verbose_out(msg.format(k), 1) print("Rectifying inventory DB with filesystem archive:") print("Rectifying assets:") dbinv.rectify_assets(cls.Asset) print("Rectifying products:") dbinv.rectify_products(cls) return extents = SpatialExtent.factory(cls, site=args.site, rastermask=args.rastermask, key=args.key, where=args.where, tiles=args.tiles, pcov=args.pcov, ptile=args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) inv.pprint(md=args.md, size=args.size) utils.gips_exit( ) # produce a summary error report then quit with a proper exit status
def process(self, *args, **kwargs): """ Process all requested products for this tile """ products = super(sarannualData, self).process(*args, **kwargs) if len(products) == 0: return self.basename = self.basename + '_' + self.sensor_set[0] for key, val in products.requested.items(): fname = os.path.join(self.path, self.basename + '_' + key) # Verify that asset exists a_type = self._products[val[0]]['assets'][0] a_obj = self.assets.get(a_type) if a_obj is None: utils.verbose_out( "Asset {} doesn't exist for tile {}".format( a_type, self.id), 3) continue datafiles = None with utils.error_handler( "Error extracting files from asset {}".format( a_obj.filename), continuable=True): datafiles = a_obj.extract() if datafiles is None: continue if val[0] == 'sign': bands = [ datafiles[b] for b in ["sl_HH", "sl_HV"] if b in datafiles ] if len(bands) > 0: img = gippy.GeoImage(bands) img.SetNoData(0) mask = gippy.GeoImage(datafiles['mask'], False) img.AddMask(mask[0] == 255) imgout = gippy.GeoImage(fname, img, gippy.GDT_Float32) imgout.SetNoData(-32768) for b in range(0, imgout.NumBands()): imgout.SetBandName(img[b].Description(), b + 1) (img[b].pow(2).log10() * 10 - 83.0).Process(imgout[b]) fname = imgout.Filename() img = None imgout = None [ RemoveFiles([f], ['.hdr', '.aux.xml']) for k, f in datafiles.items() if k != 'hdr' ] if val[0] == 'fnf': if 'C' in datafiles: # rename both files to product name os.rename(datafiles['C'], fname) os.rename(datafiles['C'] + '.hdr', fname + '.hdr') img = gippy.GeoImage(fname) img.SetNoData(0) img = None self.AddFile(self.sensor_set[0], key, fname)
def main(): title = Colors.BOLD + 'GIPS Tiles (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser() parser0.add_process_parser() parser0.add_project_parser() parser0.add_warp_parser() args = parser0.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) print title with utils.error_handler(): # create output directory if needed # tld is "{}_tiles_{}_{}".format(DATATYPE, RESOLUTION, SUFFIX) if args.notld: tld = args.outdir else: tld = os.path.join(args.outdir, '%s_tiles' % args.command) if args.res is not None: tld = tld + '_%sx%s' % (args.res[0], args.res[1]) if args.suffix != '': tld = tld + '_' + args.suffix mkdir(tld) extents = SpatialExtent.factory(cls, site=args.site, rastermask=args.rastermask, key=args.key, where=args.where, tiles=args.tiles, pcov=args.pcov, ptile=args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) for date in inv.dates: for tid in inv[date].tiles: # make sure back-end tiles are processed inv[date].tiles[tid].process(args.products, overwrite=False) # warp the tiles & copy into place in the output dir inv[date].tiles[tid].copy(tld, args.products, inv.spatial.site, args.res, args.interpolation, args.crop, args.overwrite, args.tree) utils.gips_exit( ) # produce a summary error report then quit with a proper exit status
def fetch(cls, asset, tile, date): year, month, day = date.timetuple()[:3] mainurl = '%s/%s.%02d.%02d' % (cls._assets[asset]['url'], str(year), month, day) utils.verbose_out("searching at " + mainurl, 4) with utils.error_handler('Unable to access {}'.format(mainurl)): response = cls.Repository.managed_request(mainurl) if response is None: return [] pattern = '(%s.week\d{2}.%s.%s.doy\d{3}to\d{3}.v1.5.hdf)' % (asset, str(year), tile) cpattern = re.compile(pattern) fetched = [] http_kw = {'timeout': 10, 'auth': (cls.Repository.get_setting('username'), cls.Repository.get_setting('password'))} for item in response.readlines(): if cpattern.search(item): if 'xml' in item: continue name = cpattern.findall(item)[0] url = ''.join([mainurl, '/', name]) utils.verbose_out("found " + url) outpath = os.path.join(cls.Repository.path('stage'), name) if os.path.exists(outpath): continue # match found, perform the download err_msg = 'Unable to retrieve {} from {}'.format(name, url) with utils.error_handler(err_msg, continuable=True): response = cls.Repository.managed_request(url) if response is None: return fetched # might as well give up now since the rest probably fail too with open(outpath, 'wb') as fd: fd.write(response.read()) utils.verbose_out('Retrieved {}'.format(name), 2) fetched.append(outpath) if not fetched: utils.verbose_out('Unable to find remote match for {} at {}'.format(pattern, mainurl), 4, sys.stderr) return fetched
def process(self, *args, **kwargs): """ Process assets into requested products """ # TODO - some check on if any processing was done start = dt.now() VerboseOut( 'Processing [%s] on %s dates (%s files)' % (self.products, len(self.dates), self.numfiles), 3) if len(self.products.standard) > 0: for date in self.dates: with utils.error_handler(continuable=True): self.data[date].process(*args, **kwargs) if len(self.products.composite) > 0: self.dataclass.process_composites(self, self.products.composite, **kwargs) VerboseOut('Processing completed in %s' % (dt.now() - start), 2)
def fetch(cls, asset, tile, date): qs_rv = cls.query_service(asset, tile, date) if qs_rv is None: return [] basename, url = qs_rv['basename'], qs_rv['url'] with utils.error_handler("Asset fetch error ({})".format(url), continuable=True): response = cls.Repository.managed_request(url) if response is None: return [] outpath = os.path.join(cls.Repository.path('stage'), basename) with open(outpath, 'wb') as fd: fd.write(response.read()) utils.verbose_out('Retrieved ' + basename, 2) return [outpath] return []
def query_provider(cls, asset, tile, date): """Find out from the modis servers what assets are available. Uses the given (asset, tile, date) tuple as a search key, and returns a tuple: base-filename, url """ year, month, day = date.timetuple()[:3] if asset == "MCD12Q1" and (month, day) != (1, 1): utils.verbose_out( "Cannot fetch MCD12Q1: Land cover data" " are only available for Jan. 1", 1, stream=sys.stderr) return None, None mainurl = "%s/%s.%02d.%02d" % (cls._assets[asset]['url'], str(year), month, day) pattern = '(%s.A%s%s.%s.\d{3}.\d{13}.hdf)' % ( asset, str(year), str(date.timetuple()[7]).zfill(3), tile) cpattern = re.compile(pattern) if datetime.datetime.today().date().weekday() == 2: err_msg = ("Error downloading on a Wednesday;" " possible planned MODIS provider downtime: " + mainurl) else: err_msg = "Error downloading: " + mainurl with utils.error_handler(err_msg): response = cls.Repository.managed_request(mainurl, verbosity=2) if response is None: return None, None for item in response.readlines(): # screen-scrape the content of the page and extract the full name of the needed file # (this step is needed because part of the filename, the creation timestamp, is # effectively random). if cpattern.search(item): if 'xml' in item: continue basename = cpattern.findall(item)[0] url = ''.join([mainurl, '/', basename]) return basename, url utils.verbose_out( 'Unable to find remote match for ' '{} at {}'.format(pattern, mainurl), 4) return None, None
def main(): title = Colors.BOLD + 'GIPS Data Archive Utility (v%s)' % gipsversion + Colors.OFF # argument parsing parser = GIPSParser(description=title) group = parser.add_argument_group('archive options') group.add_argument('--keep', help='Keep files after adding to archive', default=False, action='store_true') group.add_argument('--recursive', help='Iterate through subdirectories', default=False, action='store_true') group.add_argument( '--update', help= 'Update asset if newer version available, (must call gips_process to regenerate products', default=False, action='store_true') group.add_argument( '--path', default='.', help='Path to search for files to archive, defaults to `.`') args = parser.parse_args() utils.gips_script_setup(None, args.stop_on_error) with utils.error_handler('Data archive error'): print title cls = import_data_class(args.command) orm.setup() # set up DB orm in case it's needed for Asset.archive() archived_assets = cls.archive_assets(args.path, args.recursive, args.keep, args.update) # if DB inventory is enabled, update it to contain the newly archived assets if orm.use_orm(): for a in archived_assets: dbinv.update_or_add_asset(asset=a.asset, sensor=a.sensor, tile=a.tile, date=a.date, name=a.archived_filename, driver=cls.name.lower()) utils.gips_exit()
def main(): title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF parser0 = GIPSParser(datasources=False, description=title) parser0.add_projdir_parser() group = parser0.add_argument_group('masking options') args = parser0.parse_args() utils.gips_script_setup(stop_on_error=args.stop_on_error) print title # TODO - check that at least 1 of filemask or pmask is supplied header = ['date', 'band', 'min', 'max', 'mean', 'sd', 'skew', 'count'] with utils.error_handler(): for projdir in args.projdir: VerboseOut('Stats for Project directory: %s' % projdir, 1) inv = ProjectInventory(projdir, args.products) p_dates = {} # map each product to its list of valid dates for date in inv.dates: for p in inv.products(date): p_dates.setdefault(p, []).append(date) p_dates = {p: sorted(dl) for p, dl in p_dates.items()} for p_type, valid_dates in p_dates.items(): stats_fn = os.path.join(projdir, p_type + '_stats.txt') with open(stats_fn, 'w') as stats_fo: sf = getattr(utils.settings(), 'STATS_FORMAT', {}) writer = csv.writer(stats_fo, **sf) writer.writerow(header) # print date, band description, and stats for date in valid_dates: img = inv[date].open(p_type) date_str = date.strftime('%Y-%j') utils.verbose_out('Computing stats for {} {}'.format( p_type, date_str), 2) for b in img: stats = [str(s) for s in b.Stats()] writer.writerow( [date_str, b.Description()] + stats) img = None utils.gips_exit() # produce a summary error report then quit with a proper exit status
def mosaic(self, datadir, res=None, interpolation=0, crop=False, overwrite=False, alltouch=False): """For each product, combine its tiles into a single mosaic. Warp if res provided.""" if self.spatial.site is None: raise Exception('Site required for creating mosaics') start = datetime.now() bname = self.date.strftime('%Y%j') # look in each Data() and dig out its (sensor, product_type) pairs sp_pile = [(s, p) for d in self.tiles.values() for (s, p) in d.filenames if p in self.products.products] # work on each product in turn, gathering up filenames as needed for (sensor, product) in sp_pile: # create data directory when it is needed mkdir(datadir) # TODO - this is assuming a tif file. Use gippy FileExtension function when it is exposed fn = '{}_{}_{}.tif'.format(bname, sensor, product) final_fp = os.path.join(datadir, fn) if not os.path.exists(final_fp) or overwrite: err_msg = ("Error mosaicking " + final_fp + ". Did you forget" " to specify a resolution (`--res x x`)?") with utils.error_handler(err_msg, continuable=True), \ utils.make_temp_dir(dir=datadir, prefix='mosaic') as tmp_dir: tmp_fp = os.path.join(tmp_dir, fn) # for safety filenames = [self.tiles[t].filenames[(sensor, product)] for t in self.tiles if (sensor, product) in self.tiles[t].filenames ] images = gippy.GeoImages(filenames) if self.spatial.rastermask is not None: gridded_mosaic(images, tmp_fp, self.spatial.rastermask, interpolation) elif self.spatial.site is not None and res is not None: CookieCutter( images, self.spatial.site, tmp_fp, res[0], res[1], crop, interpolation, {}, alltouch, ) else: mosaic(images, tmp_fp, self.spatial.site) os.rename(tmp_fp, final_fp) t = datetime.now() - start VerboseOut('%s: created project files for %s tiles in %s' % (self.date, len(self.tiles), t), 2)
def main(): title = Colors.BOLD + 'GIPS Data Repositories (v%s)' % ( gipsversion) + Colors.OFF # argument parsing parser = GIPSParser(description=title) args = parser.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) print title with utils.error_handler(): cls.print_products() utils.gips_exit( ) # produce a summary error report then quit with a proper exit status
def setup(): """Set settings module default and run django.setup(). Prevent this from happening more than once using a global guard.""" global setup_complete if setup_complete: return if use_orm(): busted_drivers = 'sarannual', if driver_for_dbinv_feature_toggle in busted_drivers: raise Exception("Inventory database does not support '{}'. Set" " GIPS_ORM = False to use the filesystem inventory" " instead.".format(driver_for_dbinv_feature_toggle)) with utils.error_handler("Error initializing Django ORM"): os.environ.setdefault("DJANGO_SETTINGS_MODULE", "gips.inventory.orm.settings") django.setup() setup_complete = True
def readoutput(self, bandnum): with open('band' + str(bandnum) + '.chn') as f: lines = f.readlines() data = lines[4 + bandnum] # Get nominal band width in microns bandwidth = float(data[85:94]) / 1000 # Convert from W/sr-cm2 to W/sr-m2-um Lu = (float(data[59:72]) * 10000) / bandwidth trans = float(data[239:248]) Ld = 0.0 with utils.error_handler('Error calculating Ld, falling back to default (0.0)', continuable=True): with open('band' + str(bandnum) + 'Ld.chn') as f: lines = f.readlines() data = lines[4 + bandnum] # Convert channel radiance to spectral radiance Ld = (float(data[59:72]) * 10000) / bandwidth return [trans, Lu, Ld]
def __init__(self, dates=None, days=None): """ Create temporal extent object from string input """ if dates is None: dates = '1950,2050' if days is None: days = (1, 366) else: days = days.split(',') days = (int(days[0]), int(days[1])) with utils.error_handler('Bad date specification'): if ',' not in dates: dates = (self._parse_date(dates), self._parse_date(dates, True)) else: (d1, d2) = dates.replace(',', ' ').split() dates = (self._parse_date(d1), self._parse_date(d2, True)) self.datebounds = dates self.daybounds = days self.datearray = []
def query_provider(cls, asset, tile, date): """Find out from the NASA servers what AOD assets are available.""" if asset not in cls._assets: raise ValueError('{} has no defined asset for {}'.format( cls.Repository.name, asset)) with utils.error_handler("Error querying " + cls._host, continuable=True): # example full json query URL: # https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD08_D3/2017/145.json url_head = cls._assets[asset]['url'] + date.strftime('%Y/%j') query_url = url_head + '.json' utils.verbose_out('Downloading ' + query_url, 5) resp = requests.get(query_url) resp.raise_for_status() # some errors don't raise otherwise found_assets = resp.json() if len(found_assets) != 1: raise ValueError('Expected one {} asset, found {}'.format( asset, len(resp))) filename = str(found_assets[0]['name']) # str() to prevent unicode return filename, url_head + '/' + filename return None, None
def fetch(cls, asset, tile, date): """Standard Asset.fetch implementation for downloading assets.""" asset_info = cls.query_service(asset, tile, date) if asset_info is None: return [] basename, url = asset_info['basename'], asset_info['url'] basename = asset_info['basename'] url = asset_info['url'] outpath = os.path.join(cls.Repository.path('stage'), basename) with utils.error_handler("Error fetching {} from {}".format( basename, url), continuable=True): # obtain the data response = cls.Repository.managed_request(url) if response is None: return [] stage_dir = cls.Repository.path('stage') with utils.make_temp_dir(prefix='fetch', dir=stage_dir) as tmp_dn: tmp_outpath = os.path.join(tmp_dn, basename) with open(tmp_outpath, 'w') as fd: fd.write(response.read()) # verify that it is a netcdf file try: ncroot = Dataset(tmp_outpath) os.rename(tmp_outpath, outpath) except Exception as e: text = '' if e.message.endswith('Unknown file format'): token = 'Authorize NASA GESDISC DATA ARCHIVE' html = open(tmp_outpath, 'r').read(100000) if token in html: text = ('\n\nYou need to {t} \nfor your NASA ' 'EarthData login.\n').format(t=token) raise Exception(e.message + text) utils.verbose_out('Retrieved ' + basename, 2) return [outpath]
def fetch(cls, asset, tile, date): utils.verbose_out('%s: fetch tile %s for %s' % (asset, tile, date), 3) qs_rv = cls.query_service(asset, tile, date) if qs_rv is None: return [] asset_fn = qs_rv['basename'] with utils.error_handler("Error downloading from " + cls._assets[asset]['host'], continuable=True): ftp = cls.ftp_connect( asset, date) # starts chdir'd to the right directory stage_dir_fp = cls.Repository.path('stage') stage_fp = os.path.join(stage_dir_fp, asset_fn) with utils.make_temp_dir(prefix='fetchtmp', dir=stage_dir_fp) as td_name: temp_fp = os.path.join(td_name, asset_fn) utils.verbose_out("Downloading " + asset_fn, 2) with open(temp_fp, "wb") as temp_fo: ftp.retrbinary('RETR ' + asset_fn, temp_fo.write) ftp.quit() os.rename(temp_fp, stage_fp) return [stage_fp] return []
def __init__(self, projdir='', products=[]): """ Create inventory of a GIPS project directory """ self.projdir = os.path.abspath(projdir) if not os.path.exists(self.projdir): raise Exception('Directory %s does not exist!' % self.projdir) self.data = {} product_set = set() sensor_set = set() with utils.error_handler("Project directory error for " + self.projdir): # can't import Data at module scope due to circular dependencies from gips.data.core import Data for dat in Data.discover(self.projdir): self.data[dat.date] = dat # All products and sensors used across all dates product_set = product_set.union(dat.product_set) sensor_set = sensor_set.union(dat.sensor_set) if not products: products = list(product_set) self.requested_products = products self.sensors = sensor_set
def main(): import gips title = 'GIPS Configuration Utility (v%s)' % (version) parser = GIPSParser(description=title, datasources=False) subparser = parser.add_subparsers(dest='command') subparser.add_parser('print', help='Print current settings') p = subparser.add_parser( 'env', help='Configure GIPS repositories in this environment') p.add_argument('-r', '--repos', help='Top level directory for repositories', default='/data/repos') p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)', default='') p = subparser.add_parser( 'user', help= 'Configure GIPS repositories for this user (for per user customizations)' ) #p.add_argument('-e', '--email', help='Set email address (used for anonymous FTP sources)') #h = 'Install full configuration file without inheriting from environment settings' #p.add_argument('-f', '--full', help=h, default=False, action='store_true') args = parser.parse_args() print title utils.gips_script_setup( driver_string=None, # NOTE: no driver string for gips_config stop_on_error=args.stop_on_error, setup_orm=False, # NOTE: ORM cannot be setup before `gips_config env` ) # has been run if args.command == 'print': with utils.error_handler('Unable to access settings'): from gips.utils import settings s = settings() for v in dir(s): if not v.startswith('__') and v != 'gips': print print v exec('pprint.pprint(s.%s)' % v) elif args.command == 'env': with utils.error_handler('Could not create environment settings'): created_cf, cfgfile = create_environment_settings(args.repos, email=args.email) elif args.command == 'user': with utils.error_handler('Could not create user settings'): # first try importing environment settings import gips.settings created_cf, cfgfile = create_user_settings() if args.command in ('user', 'env'): msg = ('Wrote new config file: {}.' if created_cf else 'Found existing config, left unmodified: {}.') print msg.format(cfgfile) with utils.error_handler('Could not create repos'): print 'Creating repository directories, if needed.' try: create_repos() except: if created_cf: print( 'Error; removing (likely broken) config file:' ' {}.'.format(cfgfile)) os.remove(cfgfile) raise with utils.error_handler('Could not migrate database'): migrate_database() utils.gips_exit()
def get_aod(cls, lat, lon, date, fetch=True): """Returns an aod value for the given lat/lon. If the pixel has a no-data value, nearby values are averaged. If no nearby values are available, it makes an estimate using long-term averages. """ pixx = int(numpy.round(float(lon) + 179.5)) pixy = int(numpy.round(89.5 - float(lat))) roi = gippy.Recti(pixx - 1, pixy - 1, 3, 3) # try reading actual data file first aod = numpy.nan with utils.error_handler('Unable to load aod values', continuable=True): # this is just for fetching the data inv = cls.inventory(dates=date.strftime('%Y-%j'), fetch=fetch, products=['aod']) img = inv[date].tiles[cls.Asset.Repository._the_tile].open('aod') vals = img[0].Read(roi) # TODO - do this automagically in swig wrapper vals[numpy.where(vals == img[0].NoDataValue())] = numpy.nan aod = vals[1, 1] img = None source = 'MODIS (MOD08_D3)' # if invalid center but valid vals exist in 3x3 if numpy.isnan(aod) and numpy.any(~numpy.isnan(vals)): aod = numpy.mean(vals[~numpy.isnan(vals)]) source = 'MODIS (MOD08_D3) spatial average' # Calculate best estimate from multiple sources if numpy.isnan(aod): day = date.strftime('%j') repo = cls.Asset.Repository cpath = repo.path('composites') nodata = -32768 source = 'Weighted estimate using MODIS LTA values' def _calculate_estimate(filename): val, var = cls._read_point(filename, roi, nodata) aod = numpy.nan norm = numpy.nan # Negative values don't make sense if val < 0: val = 0 if var == 0: # There is only one observation, so make up # the variance. if val == 0: var = 0.15 else: var = val / 2 if not numpy.isnan(val) and not numpy.isnan(var): aod = val / var norm = 1.0 / var utils.verbose_out('AOD: LTA-Daily = %s, %s' % (val, var), 3) return aod, norm # LTA-Daily filename = os.path.join(cpath, 'ltad', 'ltad%s.tif' % str(day).zfill(4)) daily_aod, daily_norm = _calculate_estimate(filename) # LTA lta_aod, lta_norm = _calculate_estimate( os.path.join(cpath, 'lta.tif')) if numpy.isnan(lta_aod): raise Exception("Could not retrieve AOD") aod = lta_aod norm = lta_norm if not numpy.isnan(daily_aod): aod = aod + daily_aod norm = norm + daily_norm # TODO - adjacent days # Final AOD estimate aod = aod / norm utils.verbose_out('AOD: Source = %s Value = %s' % (source, aod), 2) return (source, aod)
def get_bil_vsifile(d, a): with utils.error_handler('Error accessing asset {}'.format(d), continuable=True): return os.path.join('/vsizip/' + d.assets[a].filename, d.assets[a].datafiles()[0])
def process(self, *args, **kwargs): """Deduce which products need producing, then produce them.""" products = super(prismData, self).process(*args, **kwargs) if len(products) == 0: return # overwrite = kwargs.get('overwrite', False) # utils.verbose_out('\n\noverwrite = {}\n'.format(overwrite), 2) # TODO: overwrite doesn't play well with pptsum -- wonder if it would # if it was made into a composite product (which it is) assert len(prismAsset._sensors ) == 1 # sanity check to force this code to stay current sensor = prismAsset._sensors.keys()[0] def get_bil_vsifile(d, a): with utils.error_handler('Error accessing asset {}'.format(d), continuable=True): return os.path.join('/vsizip/' + d.assets[a].filename, d.assets[a].datafiles()[0]) for key, val in products.requested.items(): start = datetime.now() # check that we have required assets requiredassets = self.products2assets([val[0]]) # val[0] s.b. key w/o product args description = self._products['pptsum']['description'] missingassets = [] availassets = [] vsinames = {} for asset in requiredassets: bil = get_bil_vsifile(self, asset) if bil is None: missingassets.append(asset) else: availassets.append(asset) vsinames[asset] = os.path.join( '/vsizip/' + self.assets[asset].filename, bil) if not availassets: utils.verbose_out( 'There are no available assets ({}) on {} for tile {}'. format(str(missingassets), str(self.date), str(self.id)), 5, ) continue prod_fn = '{}_{}_{}.tif'.format(self.basename, 'prism', key) archived_fp = os.path.join(self.path, prod_fn) # final destination if val[0] in ['ppt', 'tmin', 'tmax']: with self.make_temp_proc_dir() as tmp_dir: tmp_fp = os.path.join(tmp_dir, prod_fn) os.symlink(vsinames[self._products[key]['assets'][0]], tmp_fp) os.rename(tmp_fp, archived_fp) elif val[0] == 'pptsum': if len(val) < 2: lag = 3 # no argument provided, use default lag of 3 days SB configurable. prod_fn = re.sub(r'\.tif$', '-{}.tif'.format(lag), prod_fn) archived_fp = os.path.join( self.path, prod_fn) # have to regenerate, sigh utils.verbose_out( 'Using default lag of {} days.'.format(lag), 2) else: with utils.error_handler( "Error for pptsum lag value '{}').".format( val[1])): lag = int(val[1]) date_spec = '{},{}'.format( datetime.strftime( self.date - timedelta(days=lag), '%Y-%m-%d', ), datetime.strftime(self.date, '%Y-%m-%d'), ) inv = self.inventory( dates=date_spec, products=['ppt'], ) inv.process() # because DataInventory object doesn't update inv = self.inventory( dates=date_spec, products=['ppt'], ) if len(inv.data) < lag: utils.verbose_out( '{}: requires {} preceding days ppt ({} found).'. format(key, lag, len(inv.data)), 3, ) continue # go to next product to process imgs = [] asset_fns = [] # have to grab filenames for multiple days for tileobj in inv.data.values(): datobj = tileobj.tiles.values()[0] asset_fns.append( os.path.basename(datobj.assets['_ppt'].filename)) imgs.append(GeoImage(get_bil_vsifile(datobj, '_ppt'))) with self.make_temp_proc_dir() as tmp_dir: tmp_fp = os.path.join(tmp_dir, prod_fn) oimg = GeoImage(tmp_fp, imgs[0]) oimg.SetNoData(-9999) oimg.SetBandName( description + '({} day window)'.format(lag), 1) oimg.SetMeta(self.prep_meta(sorted(asset_fns))) for chunk in oimg.Chunks(): oarr = oimg[0].Read(chunk) * 0.0 # wat for img in imgs: oarr += img[0].Read(chunk) oimg[0].Write(oarr, chunk) oimg.Process() os.rename(tmp_fp, archived_fp) oimg = None # help swig+gdal with GC products.requested.pop(key) self.AddFile(sensor, key, archived_fp) # add product to inventory return products
def process(self, *args, **kwargs): """Produce requested products.""" products = super(modisData, self).process(*args, **kwargs) if len(products) == 0: return bname = os.path.join(self.path, self.basename) # example products.requested: # {'temp8tn': ['temp8tn'], 'clouds': ['clouds'], . . . } # key is only used once far below, and val is only used for val[0]. for key, val in products.requested.items(): # TODO replace val[0] below with this more meaningful name prod_type = val[0] if prod_type in self._productgroups['Index']: continue # indices handled differently below start = datetime.datetime.now() asset, version, missingassets, availassets, allsds = \ self.asset_check(prod_type) if not availassets: # some products aren't available for every day but this is trying every day VerboseOut( 'There are no available assets (%s) on %s for tile %s' % ( str(missingassets), str(self.date), str(self.id), ), 5) continue meta = { 'AVAILABLE_ASSETS': ' '.join(availassets) } # TODO obselete? a_fnames = [self.assets[at].filename for at in availassets] sensor = self._products[prod_type]['sensor'] fname = self.temp_product_filename( sensor, prod_type) # moved to archive at end of loop if val[0] == "landcover": os.symlink(allsds[0], fname) imgout = gippy.GeoImage(fname) if val[0] == "quality": if version != 6: raise Exception('product version not supported') os.symlink(allsds[0], fname) imgout = gippy.GeoImage(fname) # LAND VEGETATION INDICES PRODUCT # now with QC layer! if val[0] == "indices": depr_msg = ( "'indices' is deprecated, and may be removed in" " the future. See the Index product group instead.") utils.verbose_out(depr_msg, 2, stream=sys.stderr) VERSION = "2.0" meta['VERSION'] = VERSION refl = gippy.GeoImage(allsds) missing = 32767 if version == 6: redimg = refl[7].Read() nirimg = refl[8].Read() bluimg = refl[9].Read() grnimg = refl[10].Read() mirimg = refl[11].Read() swrimg = refl[12].Read() # swir1, formerly swir2 redqcimg = refl[0].Read() nirqcimg = refl[1].Read() bluqcimg = refl[2].Read() grnqcimg = refl[3].Read() mirqcimg = refl[4].Read() swrqcimg = refl[5].Read() else: raise Exception('product version not supported') # wherever the value is too small, set it to a minimum of 0 redimg[redimg < 0.0] = 0.0 nirimg[nirimg < 0.0] = 0.0 bluimg[bluimg < 0.0] = 0.0 grnimg[grnimg < 0.0] = 0.0 mirimg[mirimg < 0.0] = 0.0 swrimg[swrimg < 0.0] = 0.0 # wherever the value is too saturated, set it to a max of 1.0 redimg[(redimg != missing) & (redimg > 1.0)] = 1.0 nirimg[(nirimg != missing) & (nirimg > 1.0)] = 1.0 bluimg[(bluimg != missing) & (bluimg > 1.0)] = 1.0 grnimg[(grnimg != missing) & (grnimg > 1.0)] = 1.0 mirimg[(mirimg != missing) & (mirimg > 1.0)] = 1.0 swrimg[(swrimg != missing) & (swrimg > 1.0)] = 1.0 # red, nir # first setup a blank array with everything set to missing ndvi = missing + np.zeros_like(redimg) # compute the ndvi only where neither input is missing, AND # no divide-by-zero error will occur wg = np.where((redimg != missing) & (nirimg != missing) & (redimg + nirimg != 0.0)) ndvi[wg] = (nirimg[wg] - redimg[wg]) / (nirimg[wg] + redimg[wg]) # nir, mir lswi = missing + np.zeros_like(redimg) wg = np.where((nirimg != missing) & (mirimg != missing) & (nirimg + mirimg != 0.0)) lswi[wg] = (nirimg[wg] - mirimg[wg]) / (nirimg[wg] + mirimg[wg]) # blu, grn, red vari = missing + np.zeros_like(redimg) wg = np.where((grnimg != missing) & (redimg != missing) & (bluimg != missing) & (grnimg + redimg - bluimg != 0.0)) vari[wg] = (grnimg[wg] - redimg[wg]) / ( grnimg[wg] + redimg[wg] - bluimg[wg]) # blu, grn, red, nir brgt = missing + np.zeros_like(redimg) wg = np.where((nirimg != missing) & (redimg != missing) & (bluimg != missing) & (grnimg != missing)) brgt[wg] = (0.3 * bluimg[wg] + 0.3 * redimg[wg] + 0.1 * nirimg[wg] + 0.3 * grnimg[wg]) # red, mir, swr satvi = missing + np.zeros_like(redimg) wg = np.where((redimg != missing) & (mirimg != missing) & (swrimg != missing) & ((mirimg + redimg + 0.5) != 0.0)) satvi[wg] = (((mirimg[wg] - redimg[wg]) / (mirimg[wg] + redimg[wg] + 0.5)) * 1.5) - (swrimg[wg] / 2.0) # blu, red, nir evi = missing + np.zeros_like(redimg) wg = np.where( (bluimg != missing) & (redimg != missing) & (nirimg != missing) & (nirimg + 6.0 * redimg - 7.5 * bluimg + 1.0 != 0.0)) evi[wg] = ( (2.5 * (nirimg[wg] - redimg[wg])) / (nirimg[wg] + 6.0 * redimg[wg] - 7.5 * bluimg[wg] + 1.0)) qc = np.ones_like( redimg ) # mark as poor if all are not missing and not all are good w0 = np.where((redqcimg == 0) & (nirqcimg == 0) & (bluqcimg == 0) & (grnqcimg == 0) & (mirqcimg == 0) & (swrqcimg == 0)) w255 = np.where((redqcimg == 255) | (nirqcimg == 255) | (bluqcimg == 255) | (grnqcimg == 255) | (mirqcimg == 255) | (swrqcimg == 255)) qc[w0] = 0 # mark as good if they are all good qc[w255] = missing # mark as missing if any are missing # create output gippy image print("writing", fname) imgout = gippy.GeoImage(fname, refl, gippy.GDT_Int16, 7) del refl imgout.SetNoData(missing) imgout.SetOffset(0.0) imgout.SetGain(0.0001) imgout[6].SetGain(1.0) imgout[0].Write(ndvi) imgout[1].Write(lswi) imgout[2].Write(vari) imgout[3].Write(brgt) imgout[4].Write(satvi) imgout[5].Write(evi) imgout[6].Write(qc) imgout.SetBandName('NDVI', 1) imgout.SetBandName('LSWI', 2) imgout.SetBandName('VARI', 3) imgout.SetBandName('BRGT', 4) imgout.SetBandName('SATVI', 5) imgout.SetBandName('EVI', 6) imgout.SetBandName('QC', 7) if val[0] == "clouds": # cloud mask product meta['VERSION'] = '1.0' img = gippy.GeoImage(allsds) data = img[0].Read() clouds = np.zeros_like(data) # See table 3 in the user guide: # https://nsidc.org/sites/nsidc.org/files/files/ # MODIS-snow-user-guide-C6.pdf nodata = 127 for v in [200, 201, 211, 254, 255]: clouds[data == v] = nodata clouds[data == 237] = 0 clouds[data == 239] = 0 clouds[data == 250] = 1 # create output gippy image imgout = gippy.GeoImage(fname, img, gippy.GDT_Byte, 1) del img imgout.SetNoData(nodata) imgout.SetOffset(0.0) imgout.SetGain(1.0) imgout.SetBandName('Cloud Cover', 1) imgout[0].Write(clouds) if val[0] in ('snow', 'fsnow'): # (fsnow was removed entirely due to being a big copypasta # of the snow block; what follows is snow) raise NotImplementedError("not compatible with collection 6; " "use NDSI instead") VERSION = "1.0" meta['VERSION'] = VERSION if not missingassets: availbands = [0, 1] snowsds = [allsds[0], allsds[3], allsds[4], allsds[7]] elif missingassets[0] == 'MYD10A1': availbands = [0] snowsds = [allsds[0], allsds[3]] elif missingassets[0] == 'MOD10A1': availbands = [1] snowsds = [allsds[0], allsds[3]] else: raise IOError("Missing both MOD10A1 and MYD10A1; can't " "continue") img = gippy.GeoImage(snowsds) # there are two snow bands for iband, band in enumerate(availbands): # get the data values for both bands # for both MOD10A1 and MYD10A1, bands 0 & 3 are --v cover = img[2 * iband].Read() # Snow_Cover_Daily_Tile frac = img[2 * iband + 1].Read() # Fractional_Snow_Cover # check out frac # meanings of special values, see C5 user guide, table 4: # https://modis-snow-ice.gsfc.nasa.gov/uploads/sug_c5.pdf wbad1 = np.where((frac == 200) | (frac == 201) | (frac == 211) | (frac == 250) | (frac == 254) | (frac == 255)) wsurface1 = np.where((frac == 225) | (frac == 237) | (frac == 239)) wvalid1 = np.where((frac >= 0) & (frac <= 100)) nbad1 = len(wbad1[0]) nsurface1 = len(wsurface1[0]) nvalid1 = len(wvalid1[0]) assert nbad1 + nsurface1 + nvalid1 == frac.size, "frac contains invalid values" # check out cover # meanings of special values, see C5 user guide, table 3: # https://modis-snow-ice.gsfc.nasa.gov/uploads/sug_c5.pdf wbad2 = np.where((cover == 0) | (cover == 1) | (cover == 11) | (cover == 50) | (cover == 254) | (cover == 255)) wsurface2 = np.where((cover == 25) | (cover == 37) | (cover == 39)) wvalid2 = np.where((cover == 100) | (cover == 200)) nbad2 = len(wbad2[0]) nsurface2 = len(wsurface2[0]) nvalid2 = len(wvalid2[0]) assert nbad2 + nsurface2 + nvalid2 == cover.size, "cover contains invalid values" # assign output data here coverout = np.zeros_like(cover, dtype=np.uint8) fracout = np.zeros_like(frac, dtype=np.uint8) fracout[wvalid1] = frac[wvalid1] fracout[wsurface1] = 0 fracout[wbad1] = 127 coverout[wvalid2] = 100 coverout[wsurface2] = 0 coverout[wbad2] = 127 if len(availbands) == 2: if iband == 0: fracout1 = np.copy(fracout) coverout1 = np.copy(coverout) else: # both the current and previous are valid w = np.where((fracout != 127) & (fracout1 != 127)) fracout[w] = np.mean(np.array( [fracout[w], fracout1[w]]), axis=0).astype('uint8') # the current is not valid but previous is valid w = np.where((fracout == 127) & (fracout1 != 127)) fracout[w] = fracout1[w] # both the current and previous are valid w = np.where((coverout != 127) & (coverout1 != 127)) coverout[w] = np.mean(np.array( [coverout[w], coverout1[w]]), axis=0).astype('uint8') # the current is not valid but previous is valid w = np.where((coverout == 127) & (coverout1 != 127)) coverout[w] = coverout1[w] fracmissingcoverclear = np.sum((fracout == 127) & (coverout == 0)) fracmissingcoversnow = np.sum((fracout == 127) & (coverout == 100)) fracclearcovermissing = np.sum((fracout == 0) & (coverout == 127)) fracclearcoversnow = np.sum((fracout == 0) & (coverout == 100)) fracsnowcovermissing = np.sum((fracout > 0) & (fracout <= 100) & (coverout == 127)) fracsnowcoverclear = np.sum((fracout > 0) & (fracout <= 100) & (coverout == 0)) # fracmostlycoverclear = np.sum((fracout > 50) & (fracout <= 100) & (coverout == 0)) totsnowfrac = int(0.01 * np.sum(fracout[fracout <= 100])) totsnowcover = int(0.01 * np.sum(coverout[coverout <= 100])) numvalidfrac = np.sum(fracout != 127) numvalidcover = np.sum(coverout != 127) if totsnowcover == 0 or totsnowfrac == 0: print("no snow or ice: skipping", str(self.date), str(self.id), str(missingassets)) meta['FRACMISSINGCOVERCLEAR'] = fracmissingcoverclear meta['FRACMISSINGCOVERSNOW'] = fracmissingcoversnow meta['FRACCLEARCOVERMISSING'] = fracclearcovermissing meta['FRACCLEARCOVERSNOW'] = fracclearcoversnow meta['FRACSNOWCOVERMISSING'] = fracsnowcovermissing meta['FRACSNOWCOVERCLEAR'] = fracsnowcoverclear meta['FRACMOSTLYCOVERCLEAR'] = np.sum((fracout > 50) & (fracout <= 100) & (coverout == 0)) meta['TOTSNOWFRAC'] = totsnowfrac meta['TOTSNOWCOVER'] = totsnowcover meta['NUMVALIDFRAC'] = numvalidfrac meta['NUMVALIDCOVER'] = numvalidcover # create output gippy image imgout = gippy.GeoImage(fname, img, gippy.GDT_Byte, 2) del img imgout.SetNoData(127) imgout.SetOffset(0.0) imgout.SetGain(1.0) imgout.SetBandName('Snow Cover', 1) imgout.SetBandName('Fractional Snow Cover', 2) imgout[0].Write(coverout) imgout[1].Write(fracout) ################################################################### # TEMPERATURE PRODUCT (DAILY) if val[0] == "temp": VERSION = "1.1" meta['VERSION'] = VERSION if not missingassets: availbands = [0, 1, 2, 3] tempsds = [allsds[0], allsds[4], allsds[12], allsds[16]] qcsds = [allsds[1], allsds[5], allsds[13], allsds[17]] hoursds = [allsds[2], allsds[6], allsds[14], allsds[18]] elif missingassets[0] == 'MYD11A1': availbands = [0, 1] tempsds = [allsds[0], allsds[4]] qcsds = [allsds[1], allsds[5]] hoursds = [allsds[2], allsds[6]] elif missingassets[0] == 'MOD11A1': availbands = [2, 3] tempsds = [allsds[0], allsds[4]] qcsds = [allsds[1], allsds[5]] hoursds = [allsds[2], allsds[6]] else: raise IOError("Missing both MOD11A1 and MYD11A1; can't " "continue") tempbands = gippy.GeoImage(tempsds) qcbands = gippy.GeoImage(qcsds) hourbands = gippy.GeoImage(hoursds) imgout = gippy.GeoImage(fname, tempbands, gippy.GDT_UInt16, 5) imgout.SetNoData(65535) imgout.SetGain(0.02) # there are four temperature bands for iband, band in enumerate(availbands): # get meta name template info basename = tempbands[iband].Basename() platform = self.Asset._sensors[basename[:3]]['description'] if basename.find('daytime'): dayornight = 'day' elif basename.find('nighttime'): dayornight = 'night' else: raise Exception( '%s appears to be an invalid MODIS temperature project' % basename) qc = qcbands[iband].Read() # first two bits are 10 or 11 newmaskbad = binmask(qc, 2) # first two bits are 00 or 01 newmaskgood = ~binmask(qc, 2) # first two bits are 00 newmaskbest = ~binmask(qc, 1) & ~binmask(qc, 2) if iband == 0: bestmask = np.zeros_like(qc, dtype='uint16') bestmask += (math.pow(2, band) * newmaskbest).astype('uint16') numbad = np.sum(newmaskbad) # fracbad = np.sum(newmaskbad) / float(newmaskbad.size) numgood = np.sum(newmaskgood) # fracgood = np.sum(newmaskgood) / float(newmaskgood.size) assert numgood == qc.size - numbad numbest = np.sum(newmaskbest) # fracbest = np.sum(newmaskbest) / float(newmaskbest.size) metaname = "NUMBAD_%s_%s" % (dayornight, platform) metaname = metaname.upper() # print "metaname", metaname meta[metaname] = str(numbad) metaname = "NUMGOOD_%s_%s" % (dayornight, platform) metaname = metaname.upper() # print "metaname", metaname meta[metaname] = str(numgood) metaname = "NUMBEST_%s_%s" % (dayornight, platform) metaname = metaname.upper() # print "metaname", metaname meta[metaname] = str(numbest) # overpass time hournodatavalue = hourbands[iband].NoDataValue() hour = hourbands[iband].Read() hour = hour[hour != hournodatavalue] hourmean = 0 with utils.error_handler( "Couldn't compute hour mean for " + fname, continuable=True): hourmean = hour.mean() metaname = "MEANOVERPASSTIME_%s_%s" % (dayornight, platform) metaname = metaname.upper() meta[metaname] = str(hourmean) tempbands[iband].Process(imgout[band]) imgout[4].SetGain(1.0) imgout[4].Write(bestmask) imgout.SetBandName('Temperature Daytime Terra', 1) imgout.SetBandName('Temperature Nighttime Terra', 2) imgout.SetBandName('Temperature Daytime Aqua', 3) imgout.SetBandName('Temperature Nighttime Aqua', 4) imgout.SetBandName('Temperature Best Quality', 5) del tempbands del qcbands del hourbands ################################################################### # OBSERVATION TIME PRODUCT (DAILY) if val[0] == "obstime": VERSION = "1" meta['VERSION'] = VERSION if not missingassets: availbands = [0, 1, 2, 3] hoursds = [allsds[2], allsds[6], allsds[14], allsds[18]] elif missingassets[0] == 'MYD11A1': availbands = [0, 1] hoursds = [allsds[2], allsds[6]] elif missingassets[0] == 'MOD11A1': availbands = [2, 3] hoursds = [allsds[2], allsds[6]] else: raise IOError("Missing both MOD11A1 and MYD11A1; can't " "continue") hourbands = gippy.GeoImage(hoursds) imgout = gippy.GeoImage(fname, hourbands, gippy.GDT_Byte, 4) imgout.SetNoData(0) imgout.SetGain(0.1) # there are four temperature bands for iband, band in enumerate(availbands): # get meta name template info basename = hourbands[iband].Basename() platform = self.Asset._sensors[basename[:3]]['description'] if basename.find('daytime'): dayornight = 'day' elif basename.find('nighttime'): dayornight = 'night' else: raise Exception( '%s appears to be an invalid MODIS temperature project' % basename) hourbands[iband].Process(imgout[band]) imgout.SetBandName('Observation Time Daytime Terra', 1) imgout.SetBandName('Observation Time Nighttime Terra', 2) imgout.SetBandName('Observation Time Daytime Aqua', 3) imgout.SetBandName('Observation Time Nighttime Aqua', 4) del hourbands ################################################################### # NDVI (8-day) - Terra only if val[0] == "ndvi8": VERSION = "1.0" meta['VERSION'] = VERSION refl = gippy.GeoImage(allsds) refl.SetBandName("RED", 1) refl.SetBandName("NIR", 2) refl.SetNoData(-28762) fouts = dict(Indices(refl, {'ndvi': fname}, meta)) imgout = gippy.GeoImage(fouts['ndvi']) del refl # TEMPERATURE PRODUCT (8-day) - Terra only if val[0] == "temp8td": os.symlink(allsds[0], fname) imgout = gippy.GeoImage(fname) if val[0] == "temp8tn": os.symlink(allsds[4], fname) imgout = gippy.GeoImage(fname) # set metadata imgout.SetMeta(self.prep_meta(a_fnames, meta)) # add product to inventory archive_fp = self.archive_temp_path(fname) self.AddFile(sensor, key, archive_fp) del imgout # to cover for GDAL's internal problems utils.verbose_out(' -> {}: processed in {}'.format( os.path.basename(fname), datetime.datetime.now() - start), level=1) # process some index products (not all, see above) requested_ipt = products.groups()['Index'].keys() if requested_ipt: model_pt = requested_ipt[0] # all should be similar asset, version, _, availassets, allsds = self.asset_check(model_pt) if asset is None: raise IOError('Found no assets for {}'.format(requested_ipt)) if version < 6: raise IOError('index products require MCD43A4 version 6,' ' but found {}'.format(version)) img = gippy.GeoImage(allsds[7:]) # don't need the QC bands # GIPPY uses expected band names to find data: """ index (ie img[i]) | band num | | wavelength name 0 1 620 - 670 RED 1 2 841 - 876 NIR 2 3 459 - 479 BLUE 3 4 545 - 565 GREEN 4 5 1230 - 1250 CIRRUS (not used by gippy) 5 6 1628 - 1652 SWIR1 6 7 2105 - 2155 SWIR2 """ # SetBandName goes by band number, not index [ img.SetBandName(name, i) for (name, i) in [('RED', 1), ('NIR', 2), ('BLUE', 3), ('GREEN', 4), ('SWIR1', 6), ('SWIR2', 7)] ] sensor = self._products[model_pt]['sensor'] prod_spec = { pt: self.temp_product_filename(sensor, pt) for pt in requested_ipt } a_fnames = [self.assets[at].filename for at in availassets] pt_to_temp_fps = Indices( img, prod_spec, self.prep_meta(a_fnames, {'VERSION': '1.0'})) for pt, temp_fp in pt_to_temp_fps.items(): archived_fp = self.archive_temp_path(temp_fp) self.AddFile(sensor, pt, archived_fp)
def process(self, *args, **kwargs): """Process requested products.""" products = super(weldData, self).process(*args, **kwargs) if len(products) == 0: return sensor = 'WELD' for key, val in products.requested.items(): start = datetime.datetime.now() prod_type = val[0] fname = self.temp_product_filename(sensor, prod_type) # moved to archive at end of loop meta = self.meta_dict() meta['AVAILABLE_ASSETS'] = '' # Check for asset availability needed_assets = self._products[val[0]]['assets'] allsds = [] for a in needed_assets: if a in self.assets: with utils.error_handler('Error reading datafiles for ' + a, continuable=True): allsds.extend(self.assets[a].datafiles()) meta['AVAILABLE_ASSETS'] += ' ' + a if meta['AVAILABLE_ASSETS'] == '': utils.verbose_out('There are no available assets (%s) on %s for tile %s' % (str(needed_assets), str(self.date), str(self.id), ), 5) continue # SNOW ICE INDEX PRODUCT if val[0] == "ndsi": VERSION = "1.0" meta['VERSION'] = VERSION refl = gippy.GeoImage(allsds) # band 2 grnimg = refl[1].Read() missing = refl[1].NoDataValue() # band 4 nirimg = refl[3].Read() assert refl[3].NoDataValue() == missing # band 5 swrimg = refl[4].Read() assert refl[4].NoDataValue() == missing cldimg = refl[11].Read() # accaimg = refl[13].Read() ndsi = missing + np.zeros_like(grnimg) wg = np.where((grnimg != missing) & (swrimg != missing) & (grnimg + swrimg != 0.0) & (cldimg == 0)) ng = len(wg[0]) print "ng", ng if ng == 0: continue ndsi[wg] = (grnimg[wg] - swrimg[wg]) / (grnimg[wg] + swrimg[wg]) print ndsi.min(), ndsi.max() print ndsi[wg].min(), ndsi[wg].max() print "writing", fname imgout = gippy.GeoImage(fname, refl, gippy.GDT_Float32, 1) imgout.SetNoData(float(missing)) imgout.SetOffset(0.0) imgout.SetGain(1.0) imgout.SetProjection(PROJ) imgout[0].Write(ndsi) imgout.SetBandName('NDSI', 1) # SNOW ICE COVER PRODUCT if val[0] == "snow": VERSION = "1.0" meta['VERSION'] = VERSION refl = gippy.GeoImage(allsds) # band 2 grnimg = refl[1].Read() missing = refl[1].NoDataValue() # band 4 nirimg = refl[3].Read() assert refl[3].NoDataValue() == missing # band 5 swrimg = refl[4].Read() assert refl[4].NoDataValue() == missing cldimg = refl[11].Read() accaimg = refl[13].Read() snow = 127 + np.zeros_like(grnimg) ndsi = missing + np.zeros_like(grnimg) wg = np.where((grnimg != missing) & (swrimg != missing) & (grnimg + swrimg != 0.0) & (cldimg == 0)) ng = len(wg[0]) print "ng", ng if ng == 0: continue ndsi[wg] = (grnimg[wg] - swrimg[wg]) / (grnimg[wg] + swrimg[wg]) ws = np.where((ndsi != missing) & (ndsi > 0.4) & (nirimg > 0.11) & (swrimg > 0.1)) wc = np.where((ndsi != missing) & (ndsi > 0.4) & (nirimg <= 0.11) & (swrimg <= 0.1)) ns = len(ws[0]) nc = len(wc[0]) print ng, ns, nc if (ns > 0): snow[ws] = 1 if (nc > 0): snow[wc] = 0 print "writing", fname imgout = gippy.GeoImage(fname, refl, gippy.GDT_Byte, 1) imgout.SetNoData(127) imgout.SetOffset(0.) imgout.SetGain(1.) imgout.SetProjection(PROJ) imgout[0].Write(snow) imgout.SetBandName('SNOW', 1) # VEGETATION INDEX PRODUCT if val[0] == "ndvi": VERSION = "1.0" meta['VERSION'] = VERSION refl = gippy.GeoImage(allsds) # band 3 redimg = refl[2].Read() missing = refl[2].NoDataValue() # band 4 nirimg = refl[3].Read() assert refl[3].NoDataValue() == missing cldimg = refl[11].Read() accaimg = refl[13].Read() ndvi = missing + np.zeros_like(redimg) wg = np.where((redimg != missing) & (nirimg != missing) & (redimg + nirimg != 0.0) & (cldimg == 0)) ng = len(wg[0]) print "ng", ng if ng == 0: continue ndvi[wg] = (nirimg[wg] - redimg[wg]) / (nirimg[wg] + redimg[wg]) print ndvi.min(), ndvi.max() print ndvi[wg].min(), ndvi[wg].max() print "writing", fname imgout = gippy.GeoImage(fname, refl, gippy.GDT_Float32, 1) imgout.SetNoData(float(missing)) imgout.SetOffset(0.0) imgout.SetGain(1.0) imgout.SetProjection(PROJ) imgout[0].Write(ndvi) imgout.SetBandName('NDVI', 1) # BRIGHTNESS PRODUCT if val[0] == "brgt": VERSION = "1.0" meta['VERSION'] = VERSION refl = gippy.GeoImage(allsds) # band 2 grnimg = refl[1].Read() missing = refl[1].NoDataValue() # band 3 redimg = refl[2].Read() assert refl[2].NoDataValue() == missing # band 4 nirimg = refl[3].Read() assert refl[3].NoDataValue() == missing cldimg = refl[11].Read() brgt = missing + np.zeros_like(redimg) wg = np.where((grnimg != missing) & (redimg != missing) & (nirimg != missing) & (cldimg == 0)) ng = len(wg[0]) print "ng", ng if ng == 0: continue brgt[wg] = (grnimg[wg] + redimg[wg] + nirimg[wg])/3. print brgt.min(), brgt.max() print brgt[wg].min(), brgt[wg].max() print "writing", fname imgout = gippy.GeoImage(fname, refl, gippy.GDT_Float32, 1) imgout.SetNoData(float(missing)) imgout.SetOffset(0.0) imgout.SetGain(1.0) imgout.SetProjection(PROJ) imgout[0].Write(brgt) imgout.SetBandName('BRGT', 1) # set metadata meta = {k: str(v) for k, v in meta.iteritems()} imgout.SetMeta(meta) # add product to inventory archive_fp = self.archive_temp_path(fname) self.AddFile(sensor, key, archive_fp) VerboseOut(' -> %s: processed in %s' % (os.path.basename(fname), datetime.datetime.now() - start), 1)
def main(): title = Colors.BOLD + 'GIPS Data Export (v%s)' % __version__ + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser0.add_inventory_parser(site_required=True) parser0.add_process_parser() parser0.add_project_parser() parser0.add_warp_parser() args = parser0.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) print title with utils.error_handler(): extents = SpatialExtent.factory(cls, site=args.site, rastermask=args.rastermask, key=args.key, where=args.where, tiles=args.tiles, pcov=args.pcov, ptile=args.ptile) # create tld: SITENAME--KEY_DATATYPE_SUFFIX if args.notld: tld = args.outdir else: key = '' if args.key == '' else '--' + args.key suffix = '' if args.suffix == '' else '_' + args.suffix res = '' if args.res is None else '_%sx%s' % (args.res[0], args.res[1]) bname = (extents[0].site.LayerName() + key + res + '_' + args.command + suffix) tld = os.path.join(args.outdir, bname) for extent in extents: t_extent = TemporalExtent(args.dates, args.days) inv = DataInventory(cls, extent, t_extent, **vars(args)) datadir = os.path.join(tld, extent.site.Value()) if inv.numfiles > 0: inv.mosaic( datadir=datadir, tree=args.tree, overwrite=args.overwrite, res=args.res, interpolation=args.interpolation, crop=args.crop, alltouch=args.alltouch, process=(not args.dont_process), ) inv = ProjectInventory(datadir) inv.pprint() else: VerboseOut( 'No data found for {} within temporal extent {}'.format( str(t_extent), str(t_extent)), 2, ) utils.gips_exit( ) # produce a summary error report then quit with a proper exit status