def process(self, products, overwrite=False, **kwargs):
for asset_type, asset in self.assets.iteritems():
if asset_type != _cdlmkii: # with older cdl products, the asset is the product
continue
fname = self.temp_product_filename(_cdl, _cdlmkii)
fname_without_ext, _ = os.path.splitext(fname)
with ZipFile(asset.filename, 'r') as zipfile:
for member in zipfile.infolist():
member_ext = member.filename.split('.', 1)[1]
extracted = zipfile.extract(member, fname_without_ext)
os.rename(extracted, fname_without_ext + '.' + member_ext)
image = GeoImage(fname, True)
image[0].SetNoData(0)
image = None
image = gdal.Open(fname, gdal.GA_Update)
dbf = DBF(fname + '.vat.dbf')
for i, record in enumerate(dbf):
image.SetMetadataItem(str("CLASS_NAME_%s" % record['CLASS_NAME']), str(i))
image = None
archive_fp = self.archive_temp_path(fname)
self.AddFile(_cdl, _cdl, archive_fp)
def fetch(cls, asset, tile, date):
# The nassgeodata site is known to have an invalid certificate.
# We don't want to clutter up the output with SSL warnings.
if asset == _cdlmkii:
verbose_out("Fetching not supported for cdlmkii", 2)
return []
verbose_out("Fetching tile for {} on {}".format(tile, date.year), 2)
query_rv = cls.query_service(asset, tile, date)
if query_rv is None:
verbose_out("No CDL data for {} on {}".format(tile, date.year), 2)
return []
file_response = requests.get(
query_rv['url'], verify=False, stream=True)
with utils.make_temp_dir(
prefix='fetch', dir=cls.Repository.path('stage')) as tmp_dir:
fname = "{}_{}_cdl_cdl.tif".format(tile, date.year)
tmp_fname = tmp_dir + '/' + fname
with open(tmp_fname, 'w') as asset:
asset.write(file_response.content)
imgout = GeoImage(tmp_fname, True)
imgout.SetNoData(0)
imgout.SetMeta('GIPS_Version', gips.__version__)
imgout = None
shutil.copy(tmp_fname, cls.Repository.path('stage'))
def calulate_indices(filepath, asset_dict, indices):
''' Create image files for indices
:param filepath (str): Full path to directory containing satellite scenes in default structure created
by sat-search load --download
:param asset_dict (dict): Keys = asset (band) names in scene files (e.g. 'B01', 'B02'); Values = value names
corresponding to keys (e.g. 'red', 'nir')
:param indices (list): Which indices to generate? Options include any index included in gippy.alg.indices
:return: None (writes files to disk)
'''
subdirs = [x[0] for x in walk(filepath)]
subdirs = subdirs[1:len(subdirs)]
for folder in subdirs:
# Filepath points to folder of geotiffs of Sentinel 2 time-series of bands 4 (red) and 8 (nir)
files = [folder + '/' + f for f in listdir(folder) if not f.startswith('.')]
# Asset (band) names
pattern = '[^_.]+(?=\.[^_.]*$)'
bands = [re.search(pattern, f).group(0) for f in files]
# Match band names
bands = [asset_dict.get(band, band) for band in bands]
img = GeoImage.open(filenames=files, bandnames=bands, nodata=0)
for ind in indices:
alg.indices(img, products=[ind], filename=folder + '/index_' + ind + '.tif')
img = None
def create_image(fout='', xsz=10, ysz=10, nodata=99, empty=False):
""" Create test image, empty means all nodata, otherwise 0 """
geoimg = GeoImage.create(fout, xsz=xsz, ysz=ysz, dtype='byte')
geoimg.set_nodata(nodata)
if not empty:
arr = numpy.zeros((geoimg.xsize(), geoimg.ysize()), dtype='uint8')
geoimg[0].write(arr)
return geoimg
def open_image(item, keys=None, nodata=0, download=False):
""" Open these asset keys from scene as a gippy GeoImage """
if keys is None:
keys = item.assets.keys()
logger.debug('Opening item %s (%s)' % (item.id, ','.join(keys)))
if download:
# download items first
fnames = item.download(keys)
geoimg = GeoImage.open(fnames)
else:
# use GDAL vsi curl driver
filenames = [
item.asset(k)['href'].replace('https:/', '/vsicurl/https:/')
for k in keys
]
geoimg = GeoImage.open(filenames, update=False)
geoimg.set_bandnames(keys)
geoimg.set_nodata(nodata)
return geoimg
def download_image(url):
""" Download a test image """
fout = os.path.join(os.path.dirname(__file__), os.path.basename(url))
if not os.path.exists(fout):
print('Downloading image %s' % fout)
stream = requests.get(url, stream=True)
try:
with open(fout, 'wb') as f:
for chunk in stream.iter_content(1024):
f.write(chunk)
except:
raise Exception("Problem downloading %s" % url)
return GeoImage(fout)
def create_mask_from_bitmask(geoimg, filename=''):
""" Mask geoimg with a series of provided bitmasks """
# medium and high confidence clouds
nodata = int('0000000000000001', 2)
clouds = int('1000000000000000', 2)
cirrus = int('0011000000000000', 2)
# calculate mask
arr = geoimg.read().astype('int16')
# it is a good data mask
mask = (np.bitwise_and(arr, nodata) != nodata) & \
(np.bitwise_and(arr, clouds) < clouds) & \
(np.bitwise_and(arr, cirrus) < cirrus)
# create mask file
logger.info('Saving to file %s' % filename,
action='Save file',
actee=filename,
actor=__name__)
maskimg = GeoImage.create_from(geoimg, filename=filename, dtype='uint8')
maskimg.set_nodata(0)
maskimg[0].write(mask.astype('uint8'))
return maskimg
def process(self, *args, **kwargs):
"""Deduce which products need producing, then produce them."""
products = super(prismData, self).process(*args, **kwargs)
if len(products) == 0:
return
# overwrite = kwargs.get('overwrite', False)
# utils.verbose_out('\n\noverwrite = {}\n'.format(overwrite), 2)
# TODO: overwrite doesn't play well with pptsum -- wonder if it would
# if it was made into a composite product (which it is)
assert len(prismAsset._sensors
) == 1 # sanity check to force this code to stay current
sensor = prismAsset._sensors.keys()[0]
def get_bil_vsifile(d, a):
with utils.error_handler('Error accessing asset {}'.format(d),
continuable=True):
return os.path.join('/vsizip/' + d.assets[a].filename,
d.assets[a].datafiles()[0])
for key, val in products.requested.items():
start = datetime.now()
# check that we have required assets
requiredassets = self.products2assets([val[0]])
# val[0] s.b. key w/o product args
description = self._products['pptsum']['description']
missingassets = []
availassets = []
vsinames = {}
for asset in requiredassets:
bil = get_bil_vsifile(self, asset)
if bil is None:
missingassets.append(asset)
else:
availassets.append(asset)
vsinames[asset] = os.path.join(
'/vsizip/' + self.assets[asset].filename, bil)
if not availassets:
utils.verbose_out(
'There are no available assets ({}) on {} for tile {}'.
format(str(missingassets), str(self.date), str(self.id)),
5,
)
continue
prod_fn = '{}_{}_{}.tif'.format(self.basename, 'prism', key)
archived_fp = os.path.join(self.path, prod_fn) # final destination
if val[0] in ['ppt', 'tmin', 'tmax']:
with self.make_temp_proc_dir() as tmp_dir:
tmp_fp = os.path.join(tmp_dir, prod_fn)
os.symlink(vsinames[self._products[key]['assets'][0]],
tmp_fp)
os.rename(tmp_fp, archived_fp)
elif val[0] == 'pptsum':
if len(val) < 2:
lag = 3 # no argument provided, use default lag of 3 days SB configurable.
prod_fn = re.sub(r'\.tif$', '-{}.tif'.format(lag), prod_fn)
archived_fp = os.path.join(
self.path, prod_fn) # have to regenerate, sigh
utils.verbose_out(
'Using default lag of {} days.'.format(lag), 2)
else:
with utils.error_handler(
"Error for pptsum lag value '{}').".format(
val[1])):
lag = int(val[1])
date_spec = '{},{}'.format(
datetime.strftime(
self.date - timedelta(days=lag),
'%Y-%m-%d',
),
datetime.strftime(self.date, '%Y-%m-%d'),
)
inv = self.inventory(
dates=date_spec,
products=['ppt'],
)
inv.process()
# because DataInventory object doesn't update
inv = self.inventory(
dates=date_spec,
products=['ppt'],
)
if len(inv.data) < lag:
utils.verbose_out(
'{}: requires {} preceding days ppt ({} found).'.
format(key, lag, len(inv.data)),
3,
)
continue # go to next product to process
imgs = []
asset_fns = [] # have to grab filenames for multiple days
for tileobj in inv.data.values():
datobj = tileobj.tiles.values()[0]
asset_fns.append(
os.path.basename(datobj.assets['_ppt'].filename))
imgs.append(GeoImage(get_bil_vsifile(datobj, '_ppt')))
with self.make_temp_proc_dir() as tmp_dir:
tmp_fp = os.path.join(tmp_dir, prod_fn)
oimg = GeoImage(tmp_fp, imgs[0])
oimg.SetNoData(-9999)
oimg.SetBandName(
description + '({} day window)'.format(lag), 1)
oimg.SetMeta(self.prep_meta(sorted(asset_fns)))
for chunk in oimg.Chunks():
oarr = oimg[0].Read(chunk) * 0.0 # wat
for img in imgs:
oarr += img[0].Read(chunk)
oimg[0].Write(oarr, chunk)
oimg.Process()
os.rename(tmp_fp, archived_fp)
oimg = None # help swig+gdal with GC
products.requested.pop(key)
self.AddFile(sensor, key, archived_fp) # add product to inventory
return products
def process(self, *args, **kwargs):
"""
Cribbed from super:
Make sure all products exist and return those that need processing.
"""
products = super(prismData, self).process(*args, **kwargs)
if len(products) == 0:
return
bname = os.path.join(self.path, self.basename)
for key, val in products.requested.items():
start = datetime.now()
# check that we have required assets
requiredassets = self.products2assets([val[0]])
# val[0] s.b. key w/o product args
missingassets = []
availassets = []
vsinames = {}
def get_bil_file(d, a):
return filter(
lambda f: f.endswith('.bil'),
d.assets[a].datafiles(),
)[0]
for asset in requiredassets:
try:
bil = get_bil_file(self, asset)
except KeyError:
missingassets.append(asset)
else:
availassets.append(asset)
vsinames[asset] = bil
if not availassets:
VerboseOut(
'There are no available assets ({}) on {} for tile {}'.
format(
str(missingassets),
str(self.date),
str(self.id),
),
5,
)
continue
fname = '{}_{}_{}.tif'.format(bname, 'prism', key)
if val[0] in ['ppt', 'tmin', 'tmax']:
if os.path.lexists(fname):
os.remove(fname)
os.symlink(vsinames[self._products[key]['assets'][0]], fname)
elif val[0] == 'pptsum':
try:
lag = int(val[1])
except ValueError, TypeError:
raise Exception(
'pptsum argument format error (given: {}).')
except IndexError:
# no argument provided, use
# default lag of 3 days SB configurable.
lag = 3
fname = re.sub(r'\.tif$', '-{}.tif'.format(lag), fname)
VerboseOut('Using default lag of {} days.'.format(lag), 2)
if os.path.exists(fname):
os.remove(fname)
inv = self.inventory(
dates='{},{}'.format(
datetime.strftime(
self.date - timedelta(days=lag - 1),
'%Y-%m-%d',
),
datetime.strftime(self.date, '%Y-%m-%d'),
),
products=['ppt'],
)
inv.process()
inv = self.inventory(
dates='{},{}'.format(
datetime.strftime(
self.date - timedelta(days=lag - 1),
'%Y-%m-%d',
),
datetime.strftime(self.date, '%Y-%m-%d'),
),
products=['ppt'],
)
if len(inv.data) < lag:
raise Exception(
key +
': requires {} preceding days ppt assets.'.format(lag)
+ '\nOnly {} days found.'.format(len(inv.data)))
imgs = []
for tileobj in inv.data.values():
datobj = tileobj.tiles.values()[0]
imgs.append(datobj.open_assets('ppt'))
oimg = GeoImage(fname, imgs[0])
for chunk in oimg.Chunks():
oarr = oimg[0].Read(chunk) * 0.0
for img in imgs:
print('mean value of image: {}'.format(
mean(img[0].Read())))
oarr += img[0].Read(chunk)
oimg[0].Write(oarr, chunk)
oimg.Process()
#oimg = None
print('nih')
products.requested.pop(key)