def process(self, *args, **kwargs):
"""Deduce which products need producing, then produce them."""
products = super(prismData, self).process(*args, **kwargs)
if len(products) == 0:
return
# overwrite = kwargs.get('overwrite', False)
# utils.verbose_out('\n\noverwrite = {}\n'.format(overwrite), 2)
# TODO: overwrite doesn't play well with pptsum -- wonder if it would
# if it was made into a composite product (which it is)
assert len(prismAsset._sensors
) == 1 # sanity check to force this code to stay current
sensor = prismAsset._sensors.keys()[0]
def get_bil_vsifile(d, a):
with utils.error_handler('Error accessing asset {}'.format(d),
continuable=True):
return os.path.join('/vsizip/' + d.assets[a].filename,
d.assets[a].datafiles()[0])
for key, val in products.requested.items():
start = datetime.now()
# check that we have required assets
requiredassets = self.products2assets([val[0]])
# val[0] s.b. key w/o product args
description = self._products['pptsum']['description']
missingassets = []
availassets = []
vsinames = {}
for asset in requiredassets:
bil = get_bil_vsifile(self, asset)
if bil is None:
missingassets.append(asset)
else:
availassets.append(asset)
vsinames[asset] = os.path.join(
'/vsizip/' + self.assets[asset].filename, bil)
if not availassets:
utils.verbose_out(
'There are no available assets ({}) on {} for tile {}'.
format(str(missingassets), str(self.date), str(self.id)),
5,
)
continue
prod_fn = '{}_{}_{}.tif'.format(self.basename, 'prism', key)
archived_fp = os.path.join(self.path, prod_fn) # final destination
if val[0] in ['ppt', 'tmin', 'tmax']:
with self.make_temp_proc_dir() as tmp_dir:
tmp_fp = os.path.join(tmp_dir, prod_fn)
os.symlink(vsinames[self._products[key]['assets'][0]],
tmp_fp)
os.rename(tmp_fp, archived_fp)
elif val[0] == 'pptsum':
if len(val) < 2:
lag = 3 # no argument provided, use default lag of 3 days SB configurable.
prod_fn = re.sub(r'\.tif$', '-{}.tif'.format(lag), prod_fn)
archived_fp = os.path.join(
self.path, prod_fn) # have to regenerate, sigh
utils.verbose_out(
'Using default lag of {} days.'.format(lag), 2)
else:
with utils.error_handler(
"Error for pptsum lag value '{}').".format(
val[1])):
lag = int(val[1])
date_spec = '{},{}'.format(
datetime.strftime(
self.date - timedelta(days=lag),
'%Y-%m-%d',
),
datetime.strftime(self.date, '%Y-%m-%d'),
)
inv = self.inventory(
dates=date_spec,
products=['ppt'],
)
inv.process()
# because DataInventory object doesn't update
inv = self.inventory(
dates=date_spec,
products=['ppt'],
)
if len(inv.data) < lag:
utils.verbose_out(
'{}: requires {} preceding days ppt ({} found).'.
format(key, lag, len(inv.data)),
3,
)
continue # go to next product to process
imgs = []
asset_fns = [] # have to grab filenames for multiple days
for tileobj in inv.data.values():
datobj = tileobj.tiles.values()[0]
asset_fns.append(
os.path.basename(datobj.assets['_ppt'].filename))
imgs.append(GeoImage(get_bil_vsifile(datobj, '_ppt')))
with self.make_temp_proc_dir() as tmp_dir:
tmp_fp = os.path.join(tmp_dir, prod_fn)
oimg = GeoImage(tmp_fp, imgs[0])
oimg.SetNoData(-9999)
oimg.SetBandName(
description + '({} day window)'.format(lag), 1)
oimg.SetMeta(self.prep_meta(sorted(asset_fns)))
for chunk in oimg.Chunks():
oarr = oimg[0].Read(chunk) * 0.0 # wat
for img in imgs:
oarr += img[0].Read(chunk)
oimg[0].Write(oarr, chunk)
oimg.Process()
os.rename(tmp_fp, archived_fp)
oimg = None # help swig+gdal with GC
products.requested.pop(key)
self.AddFile(sensor, key, archived_fp) # add product to inventory
return products
def process(self, *args, **kwargs):
"""
Cribbed from super:
Make sure all products exist and return those that need processing.
"""
products = super(prismData, self).process(*args, **kwargs)
if len(products) == 0:
return
bname = os.path.join(self.path, self.basename)
for key, val in products.requested.items():
start = datetime.now()
# check that we have required assets
requiredassets = self.products2assets([val[0]])
# val[0] s.b. key w/o product args
missingassets = []
availassets = []
vsinames = {}
def get_bil_file(d, a):
return filter(
lambda f: f.endswith('.bil'),
d.assets[a].datafiles(),
)[0]
for asset in requiredassets:
try:
bil = get_bil_file(self, asset)
except KeyError:
missingassets.append(asset)
else:
availassets.append(asset)
vsinames[asset] = bil
if not availassets:
VerboseOut(
'There are no available assets ({}) on {} for tile {}'.
format(
str(missingassets),
str(self.date),
str(self.id),
),
5,
)
continue
fname = '{}_{}_{}.tif'.format(bname, 'prism', key)
if val[0] in ['ppt', 'tmin', 'tmax']:
if os.path.lexists(fname):
os.remove(fname)
os.symlink(vsinames[self._products[key]['assets'][0]], fname)
elif val[0] == 'pptsum':
try:
lag = int(val[1])
except ValueError, TypeError:
raise Exception(
'pptsum argument format error (given: {}).')
except IndexError:
# no argument provided, use
# default lag of 3 days SB configurable.
lag = 3
fname = re.sub(r'\.tif$', '-{}.tif'.format(lag), fname)
VerboseOut('Using default lag of {} days.'.format(lag), 2)
if os.path.exists(fname):
os.remove(fname)
inv = self.inventory(
dates='{},{}'.format(
datetime.strftime(
self.date - timedelta(days=lag - 1),
'%Y-%m-%d',
),
datetime.strftime(self.date, '%Y-%m-%d'),
),
products=['ppt'],
)
inv.process()
inv = self.inventory(
dates='{},{}'.format(
datetime.strftime(
self.date - timedelta(days=lag - 1),
'%Y-%m-%d',
),
datetime.strftime(self.date, '%Y-%m-%d'),
),
products=['ppt'],
)
if len(inv.data) < lag:
raise Exception(
key +
': requires {} preceding days ppt assets.'.format(lag)
+ '\nOnly {} days found.'.format(len(inv.data)))
imgs = []
for tileobj in inv.data.values():
datobj = tileobj.tiles.values()[0]
imgs.append(datobj.open_assets('ppt'))
oimg = GeoImage(fname, imgs[0])
for chunk in oimg.Chunks():
oarr = oimg[0].Read(chunk) * 0.0
for img in imgs:
print('mean value of image: {}'.format(
mean(img[0].Read())))
oarr += img[0].Read(chunk)
oimg[0].Write(oarr, chunk)
oimg.Process()
#oimg = None
print('nih')
products.requested.pop(key)