def fetch(cls, asset, tile, date): # The nassgeodata site is known to have an invalid certificate. # We don't want to clutter up the output with SSL warnings. if asset == _cdlmkii: verbose_out("Fetching not supported for cdlmkii", 2) return [] verbose_out("Fetching tile for {} on {}".format(tile, date.year), 2) query_rv = cls.query_service(asset, tile, date) if query_rv is None: verbose_out("No CDL data for {} on {}".format(tile, date.year), 2) return [] file_response = requests.get( query_rv['url'], verify=False, stream=True) with utils.make_temp_dir( prefix='fetch', dir=cls.Repository.path('stage')) as tmp_dir: fname = "{}_{}_cdl_cdl.tif".format(tile, date.year) tmp_fname = tmp_dir + '/' + fname with open(tmp_fname, 'w') as asset: asset.write(file_response.content) imgout = GeoImage(tmp_fname, True) imgout.SetNoData(0) imgout.SetMeta('GIPS_Version', gips.__version__) imgout = None shutil.copy(tmp_fname, cls.Repository.path('stage'))
def process(self, *args, **kwargs): """Deduce which products need producing, then produce them.""" products = super(prismData, self).process(*args, **kwargs) if len(products) == 0: return # overwrite = kwargs.get('overwrite', False) # utils.verbose_out('\n\noverwrite = {}\n'.format(overwrite), 2) # TODO: overwrite doesn't play well with pptsum -- wonder if it would # if it was made into a composite product (which it is) assert len(prismAsset._sensors ) == 1 # sanity check to force this code to stay current sensor = prismAsset._sensors.keys()[0] def get_bil_vsifile(d, a): with utils.error_handler('Error accessing asset {}'.format(d), continuable=True): return os.path.join('/vsizip/' + d.assets[a].filename, d.assets[a].datafiles()[0]) for key, val in products.requested.items(): start = datetime.now() # check that we have required assets requiredassets = self.products2assets([val[0]]) # val[0] s.b. key w/o product args description = self._products['pptsum']['description'] missingassets = [] availassets = [] vsinames = {} for asset in requiredassets: bil = get_bil_vsifile(self, asset) if bil is None: missingassets.append(asset) else: availassets.append(asset) vsinames[asset] = os.path.join( '/vsizip/' + self.assets[asset].filename, bil) if not availassets: utils.verbose_out( 'There are no available assets ({}) on {} for tile {}'. format(str(missingassets), str(self.date), str(self.id)), 5, ) continue prod_fn = '{}_{}_{}.tif'.format(self.basename, 'prism', key) archived_fp = os.path.join(self.path, prod_fn) # final destination if val[0] in ['ppt', 'tmin', 'tmax']: with self.make_temp_proc_dir() as tmp_dir: tmp_fp = os.path.join(tmp_dir, prod_fn) os.symlink(vsinames[self._products[key]['assets'][0]], tmp_fp) os.rename(tmp_fp, archived_fp) elif val[0] == 'pptsum': if len(val) < 2: lag = 3 # no argument provided, use default lag of 3 days SB configurable. prod_fn = re.sub(r'\.tif$', '-{}.tif'.format(lag), prod_fn) archived_fp = os.path.join( self.path, prod_fn) # have to regenerate, sigh utils.verbose_out( 'Using default lag of {} days.'.format(lag), 2) else: with utils.error_handler( "Error for pptsum lag value '{}').".format( val[1])): lag = int(val[1]) date_spec = '{},{}'.format( datetime.strftime( self.date - timedelta(days=lag), '%Y-%m-%d', ), datetime.strftime(self.date, '%Y-%m-%d'), ) inv = self.inventory( dates=date_spec, products=['ppt'], ) inv.process() # because DataInventory object doesn't update inv = self.inventory( dates=date_spec, products=['ppt'], ) if len(inv.data) < lag: utils.verbose_out( '{}: requires {} preceding days ppt ({} found).'. format(key, lag, len(inv.data)), 3, ) continue # go to next product to process imgs = [] asset_fns = [] # have to grab filenames for multiple days for tileobj in inv.data.values(): datobj = tileobj.tiles.values()[0] asset_fns.append( os.path.basename(datobj.assets['_ppt'].filename)) imgs.append(GeoImage(get_bil_vsifile(datobj, '_ppt'))) with self.make_temp_proc_dir() as tmp_dir: tmp_fp = os.path.join(tmp_dir, prod_fn) oimg = GeoImage(tmp_fp, imgs[0]) oimg.SetNoData(-9999) oimg.SetBandName( description + '({} day window)'.format(lag), 1) oimg.SetMeta(self.prep_meta(sorted(asset_fns))) for chunk in oimg.Chunks(): oarr = oimg[0].Read(chunk) * 0.0 # wat for img in imgs: oarr += img[0].Read(chunk) oimg[0].Write(oarr, chunk) oimg.Process() os.rename(tmp_fp, archived_fp) oimg = None # help swig+gdal with GC products.requested.pop(key) self.AddFile(sensor, key, archived_fp) # add product to inventory return products