def fetch(cls, asset, tile, date):
# The nassgeodata site is known to have an invalid certificate.
# We don't want to clutter up the output with SSL warnings.
if asset == _cdlmkii:
verbose_out("Fetching not supported for cdlmkii", 2)
return []
verbose_out("Fetching tile for {} on {}".format(tile, date.year), 2)
query_rv = cls.query_service(asset, tile, date)
if query_rv is None:
verbose_out("No CDL data for {} on {}".format(tile, date.year), 2)
return []
file_response = requests.get(
query_rv['url'], verify=False, stream=True)
with utils.make_temp_dir(
prefix='fetch', dir=cls.Repository.path('stage')) as tmp_dir:
fname = "{}_{}_cdl_cdl.tif".format(tile, date.year)
tmp_fname = tmp_dir + '/' + fname
with open(tmp_fname, 'w') as asset:
asset.write(file_response.content)
imgout = GeoImage(tmp_fname, True)
imgout.SetNoData(0)
imgout.SetMeta('GIPS_Version', gips.__version__)
imgout = None
shutil.copy(tmp_fname, cls.Repository.path('stage'))
def process(self, *args, **kwargs):
"""Deduce which products need producing, then produce them."""
products = super(prismData, self).process(*args, **kwargs)
if len(products) == 0:
return
# overwrite = kwargs.get('overwrite', False)
# utils.verbose_out('\n\noverwrite = {}\n'.format(overwrite), 2)
# TODO: overwrite doesn't play well with pptsum -- wonder if it would
# if it was made into a composite product (which it is)
assert len(prismAsset._sensors
) == 1 # sanity check to force this code to stay current
sensor = prismAsset._sensors.keys()[0]
def get_bil_vsifile(d, a):
with utils.error_handler('Error accessing asset {}'.format(d),
continuable=True):
return os.path.join('/vsizip/' + d.assets[a].filename,
d.assets[a].datafiles()[0])
for key, val in products.requested.items():
start = datetime.now()
# check that we have required assets
requiredassets = self.products2assets([val[0]])
# val[0] s.b. key w/o product args
description = self._products['pptsum']['description']
missingassets = []
availassets = []
vsinames = {}
for asset in requiredassets:
bil = get_bil_vsifile(self, asset)
if bil is None:
missingassets.append(asset)
else:
availassets.append(asset)
vsinames[asset] = os.path.join(
'/vsizip/' + self.assets[asset].filename, bil)
if not availassets:
utils.verbose_out(
'There are no available assets ({}) on {} for tile {}'.
format(str(missingassets), str(self.date), str(self.id)),
5,
)
continue
prod_fn = '{}_{}_{}.tif'.format(self.basename, 'prism', key)
archived_fp = os.path.join(self.path, prod_fn) # final destination
if val[0] in ['ppt', 'tmin', 'tmax']:
with self.make_temp_proc_dir() as tmp_dir:
tmp_fp = os.path.join(tmp_dir, prod_fn)
os.symlink(vsinames[self._products[key]['assets'][0]],
tmp_fp)
os.rename(tmp_fp, archived_fp)
elif val[0] == 'pptsum':
if len(val) < 2:
lag = 3 # no argument provided, use default lag of 3 days SB configurable.
prod_fn = re.sub(r'\.tif$', '-{}.tif'.format(lag), prod_fn)
archived_fp = os.path.join(
self.path, prod_fn) # have to regenerate, sigh
utils.verbose_out(
'Using default lag of {} days.'.format(lag), 2)
else:
with utils.error_handler(
"Error for pptsum lag value '{}').".format(
val[1])):
lag = int(val[1])
date_spec = '{},{}'.format(
datetime.strftime(
self.date - timedelta(days=lag),
'%Y-%m-%d',
),
datetime.strftime(self.date, '%Y-%m-%d'),
)
inv = self.inventory(
dates=date_spec,
products=['ppt'],
)
inv.process()
# because DataInventory object doesn't update
inv = self.inventory(
dates=date_spec,
products=['ppt'],
)
if len(inv.data) < lag:
utils.verbose_out(
'{}: requires {} preceding days ppt ({} found).'.
format(key, lag, len(inv.data)),
3,
)
continue # go to next product to process
imgs = []
asset_fns = [] # have to grab filenames for multiple days
for tileobj in inv.data.values():
datobj = tileobj.tiles.values()[0]
asset_fns.append(
os.path.basename(datobj.assets['_ppt'].filename))
imgs.append(GeoImage(get_bil_vsifile(datobj, '_ppt')))
with self.make_temp_proc_dir() as tmp_dir:
tmp_fp = os.path.join(tmp_dir, prod_fn)
oimg = GeoImage(tmp_fp, imgs[0])
oimg.SetNoData(-9999)
oimg.SetBandName(
description + '({} day window)'.format(lag), 1)
oimg.SetMeta(self.prep_meta(sorted(asset_fns)))
for chunk in oimg.Chunks():
oarr = oimg[0].Read(chunk) * 0.0 # wat
for img in imgs:
oarr += img[0].Read(chunk)
oimg[0].Write(oarr, chunk)
oimg.Process()
os.rename(tmp_fp, archived_fp)
oimg = None # help swig+gdal with GC
products.requested.pop(key)
self.AddFile(sensor, key, archived_fp) # add product to inventory
return products