def get_data(self, dates=None, products=None, chunk=None):
""" Read all files as time series, stacking all products """
# TODO - change to absolute dates
days = numpy.array([int(d.strftime('%j')) for d in dates])
imgarr = []
if dates is None:
dates = self.dates
if products is None:
products = self.requested_products
for p in products:
gimg = self.get_timeseries(p, dates=dates)
# TODO - move numpy.squeeze into swig interface file?
ch = gippy.Recti(chunk[0], chunk[1], chunk[2], chunk[3])
arr = numpy.squeeze(gimg.TimeSeries(days.astype('float64'), ch))
arr[arr == gimg[0].NoDataValue()] = numpy.nan
if len(days) == 1:
dims = arr.shape
arr = arr.reshape(1, dims[0], dims[1])
imgarr.append(arr)
data = numpy.vstack(tuple(imgarr))
return data
def get_aod(cls, lat, lon, date, fetch=True):
"""Returns an aod value for the given lat/lon.
If the pixel has a no-data value, nearby values are averaged. If
no nearby values are available, it makes an estimate using
long-term averages.
"""
pixx = int(numpy.round(float(lon) + 179.5))
pixy = int(numpy.round(89.5 - float(lat)))
roi = gippy.Recti(pixx - 1, pixy - 1, 3, 3)
# try reading actual data file first
aod = numpy.nan
with utils.error_handler('Unable to load aod values',
continuable=True):
# this is just for fetching the data
inv = cls.inventory(dates=date.strftime('%Y-%j'),
fetch=fetch,
products=['aod'])
img = inv[date].tiles[cls.Asset.Repository._the_tile].open('aod')
vals = img[0].Read(roi)
# TODO - do this automagically in swig wrapper
vals[numpy.where(vals == img[0].NoDataValue())] = numpy.nan
aod = vals[1, 1]
img = None
source = 'MODIS (MOD08_D3)'
# if invalid center but valid vals exist in 3x3
if numpy.isnan(aod) and numpy.any(~numpy.isnan(vals)):
aod = numpy.mean(vals[~numpy.isnan(vals)])
source = 'MODIS (MOD08_D3) spatial average'
# Calculate best estimate from multiple sources
if numpy.isnan(aod):
day = date.strftime('%j')
repo = cls.Asset.Repository
cpath = repo.path('composites')
nodata = -32768
source = 'Weighted estimate using MODIS LTA values'
def _calculate_estimate(filename):
val, var = cls._read_point(filename, roi, nodata)
aod = numpy.nan
norm = numpy.nan
# Negative values don't make sense
if val < 0:
val = 0
if var == 0:
# There is only one observation, so make up
# the variance.
if val == 0:
var = 0.15
else:
var = val / 2
if not numpy.isnan(val) and not numpy.isnan(var):
aod = val / var
norm = 1.0 / var
utils.verbose_out('AOD: LTA-Daily = %s, %s' % (val, var),
3)
return aod, norm
# LTA-Daily
filename = os.path.join(cpath, 'ltad',
'ltad%s.tif' % str(day).zfill(4))
daily_aod, daily_norm = _calculate_estimate(filename)
# LTA
lta_aod, lta_norm = _calculate_estimate(
os.path.join(cpath, 'lta.tif'))
if numpy.isnan(lta_aod):
raise Exception("Could not retrieve AOD")
aod = lta_aod
norm = lta_norm
if not numpy.isnan(daily_aod):
aod = aod + daily_aod
norm = norm + daily_norm
# TODO - adjacent days
# Final AOD estimate
aod = aod / norm
utils.verbose_out('AOD: Source = %s Value = %s' % (source, aod), 2)
return (source, aod)
def get_aod(cls, lat, lon, date, fetch=True):
pixx = int(numpy.round(float(lon) + 179.5))
pixy = int(numpy.round(89.5 - float(lat)))
roi = gippy.Recti(pixx - 1, pixy - 1, 3, 3)
# try reading actual data file first
try:
# this is just for fetching the data
inv = cls.inventory(dates=date.strftime('%Y-%j'),
fetch=fetch,
products=['aod'])
img = inv[date].tiles[''].open('aod')
vals = img[0].Read(roi)
# TODO - do this automagically in swig wrapper
vals[numpy.where(vals == img[0].NoDataValue())] = numpy.nan
aod = vals[1, 1]
img = None
source = 'MODIS (MOD08_D3)'
# if invalid center but valid vals exist in 3x3
if numpy.isnan(aod) and numpy.any(~numpy.isnan(vals)):
aod = numpy.mean(vals[~numpy.isnan(vals)])
source = 'MODIS (MOD08_D3) spatial average'
except Exception:
VerboseOut(traceback.format_exc(), 4)
aod = numpy.nan
var = 0
totalvar = 0
day = date.strftime('%j')
# Calculate best estimate from multiple sources
repo = cls.Asset.Repository
cpath = repo.path('composites')
if numpy.isnan(aod):
aod = 0.0
norm = 0.0
cnt = 0
nodata = -32768
source = 'Weighted estimate using MODIS LTA values'
# LTA-Daily
filename = os.path.join(cpath, 'ltad',
'ltad%s.tif' % str(day).zfill(3))
val, var = cls._read_point(filename, roi, nodata)
var = var if var != 0.0 else val
if not numpy.isnan(val) and not numpy.isnan(var):
aod = val / var
totalvar = var
norm = 1.0 / var
cnt = cnt + 1
VerboseOut('AOD: LTA-Daily = %s, %s' % (val, var), 3)
# LTA
val, var = cls._read_point(os.path.join(cpath, 'lta.tif'), roi,
nodata)
var = var if var != 0.0 else val
if not numpy.isnan(val) and not numpy.isnan(var):
aod = aod + val / var
totalvar = totalvar + var
norm = norm + 1.0 / var
cnt = cnt + 1
VerboseOut('AOD: LTA = %s, %s' % (val, var), 3)
# TODO - adjacent days
# Final AOD estimate
aod = aod / norm
totalvar = totalvar / cnt
if numpy.isnan(aod):
raise Exception("Could not retrieve AOD")
VerboseOut('AOD: Source = %s Value = %s' % (source, aod), 2)
return (source, aod)