def process(self, products, overwrite=False, **kwargs): for asset_type, asset in self.assets.iteritems(): if asset_type != _cdlmkii: # with older cdl products, the asset is the product continue fname = self.temp_product_filename(_cdl, _cdlmkii) fname_without_ext, _ = os.path.splitext(fname) with ZipFile(asset.filename, 'r') as zipfile: for member in zipfile.infolist(): member_ext = member.filename.split('.', 1)[1] extracted = zipfile.extract(member, fname_without_ext) os.rename(extracted, fname_without_ext + '.' + member_ext) image = GeoImage(fname, True) image[0].SetNoData(0) image = None image = gdal.Open(fname, gdal.GA_Update) dbf = DBF(fname + '.vat.dbf') for i, record in enumerate(dbf): image.SetMetadataItem(str("CLASS_NAME_%s" % record['CLASS_NAME']), str(i)) image = None archive_fp = self.archive_temp_path(fname) self.AddFile(_cdl, _cdl, archive_fp)
def fetch(cls, asset, tile, date): # The nassgeodata site is known to have an invalid certificate. # We don't want to clutter up the output with SSL warnings. if asset == _cdlmkii: verbose_out("Fetching not supported for cdlmkii", 2) return [] verbose_out("Fetching tile for {} on {}".format(tile, date.year), 2) query_rv = cls.query_service(asset, tile, date) if query_rv is None: verbose_out("No CDL data for {} on {}".format(tile, date.year), 2) return [] file_response = requests.get( query_rv['url'], verify=False, stream=True) with utils.make_temp_dir( prefix='fetch', dir=cls.Repository.path('stage')) as tmp_dir: fname = "{}_{}_cdl_cdl.tif".format(tile, date.year) tmp_fname = tmp_dir + '/' + fname with open(tmp_fname, 'w') as asset: asset.write(file_response.content) imgout = GeoImage(tmp_fname, True) imgout.SetNoData(0) imgout.SetMeta('GIPS_Version', gips.__version__) imgout = None shutil.copy(tmp_fname, cls.Repository.path('stage'))
def calulate_indices(filepath, asset_dict, indices): ''' Create image files for indices :param filepath (str): Full path to directory containing satellite scenes in default structure created by sat-search load --download :param asset_dict (dict): Keys = asset (band) names in scene files (e.g. 'B01', 'B02'); Values = value names corresponding to keys (e.g. 'red', 'nir') :param indices (list): Which indices to generate? Options include any index included in gippy.alg.indices :return: None (writes files to disk) ''' subdirs = [x[0] for x in walk(filepath)] subdirs = subdirs[1:len(subdirs)] for folder in subdirs: # Filepath points to folder of geotiffs of Sentinel 2 time-series of bands 4 (red) and 8 (nir) files = [folder + '/' + f for f in listdir(folder) if not f.startswith('.')] # Asset (band) names pattern = '[^_.]+(?=\.[^_.]*$)' bands = [re.search(pattern, f).group(0) for f in files] # Match band names bands = [asset_dict.get(band, band) for band in bands] img = GeoImage.open(filenames=files, bandnames=bands, nodata=0) for ind in indices: alg.indices(img, products=[ind], filename=folder + '/index_' + ind + '.tif') img = None
def create_image(fout='', xsz=10, ysz=10, nodata=99, empty=False): """ Create test image, empty means all nodata, otherwise 0 """ geoimg = GeoImage.create(fout, xsz=xsz, ysz=ysz, dtype='byte') geoimg.set_nodata(nodata) if not empty: arr = numpy.zeros((geoimg.xsize(), geoimg.ysize()), dtype='uint8') geoimg[0].write(arr) return geoimg
def open_image(item, keys=None, nodata=0, download=False): """ Open these asset keys from scene as a gippy GeoImage """ if keys is None: keys = item.assets.keys() logger.debug('Opening item %s (%s)' % (item.id, ','.join(keys))) if download: # download items first fnames = item.download(keys) geoimg = GeoImage.open(fnames) else: # use GDAL vsi curl driver filenames = [ item.asset(k)['href'].replace('https:/', '/vsicurl/https:/') for k in keys ] geoimg = GeoImage.open(filenames, update=False) geoimg.set_bandnames(keys) geoimg.set_nodata(nodata) return geoimg
def download_image(url): """ Download a test image """ fout = os.path.join(os.path.dirname(__file__), os.path.basename(url)) if not os.path.exists(fout): print('Downloading image %s' % fout) stream = requests.get(url, stream=True) try: with open(fout, 'wb') as f: for chunk in stream.iter_content(1024): f.write(chunk) except: raise Exception("Problem downloading %s" % url) return GeoImage(fout)
def create_mask_from_bitmask(geoimg, filename=''): """ Mask geoimg with a series of provided bitmasks """ # medium and high confidence clouds nodata = int('0000000000000001', 2) clouds = int('1000000000000000', 2) cirrus = int('0011000000000000', 2) # calculate mask arr = geoimg.read().astype('int16') # it is a good data mask mask = (np.bitwise_and(arr, nodata) != nodata) & \ (np.bitwise_and(arr, clouds) < clouds) & \ (np.bitwise_and(arr, cirrus) < cirrus) # create mask file logger.info('Saving to file %s' % filename, action='Save file', actee=filename, actor=__name__) maskimg = GeoImage.create_from(geoimg, filename=filename, dtype='uint8') maskimg.set_nodata(0) maskimg[0].write(mask.astype('uint8')) return maskimg
def process(self, *args, **kwargs): """Deduce which products need producing, then produce them.""" products = super(prismData, self).process(*args, **kwargs) if len(products) == 0: return # overwrite = kwargs.get('overwrite', False) # utils.verbose_out('\n\noverwrite = {}\n'.format(overwrite), 2) # TODO: overwrite doesn't play well with pptsum -- wonder if it would # if it was made into a composite product (which it is) assert len(prismAsset._sensors ) == 1 # sanity check to force this code to stay current sensor = prismAsset._sensors.keys()[0] def get_bil_vsifile(d, a): with utils.error_handler('Error accessing asset {}'.format(d), continuable=True): return os.path.join('/vsizip/' + d.assets[a].filename, d.assets[a].datafiles()[0]) for key, val in products.requested.items(): start = datetime.now() # check that we have required assets requiredassets = self.products2assets([val[0]]) # val[0] s.b. key w/o product args description = self._products['pptsum']['description'] missingassets = [] availassets = [] vsinames = {} for asset in requiredassets: bil = get_bil_vsifile(self, asset) if bil is None: missingassets.append(asset) else: availassets.append(asset) vsinames[asset] = os.path.join( '/vsizip/' + self.assets[asset].filename, bil) if not availassets: utils.verbose_out( 'There are no available assets ({}) on {} for tile {}'. format(str(missingassets), str(self.date), str(self.id)), 5, ) continue prod_fn = '{}_{}_{}.tif'.format(self.basename, 'prism', key) archived_fp = os.path.join(self.path, prod_fn) # final destination if val[0] in ['ppt', 'tmin', 'tmax']: with self.make_temp_proc_dir() as tmp_dir: tmp_fp = os.path.join(tmp_dir, prod_fn) os.symlink(vsinames[self._products[key]['assets'][0]], tmp_fp) os.rename(tmp_fp, archived_fp) elif val[0] == 'pptsum': if len(val) < 2: lag = 3 # no argument provided, use default lag of 3 days SB configurable. prod_fn = re.sub(r'\.tif$', '-{}.tif'.format(lag), prod_fn) archived_fp = os.path.join( self.path, prod_fn) # have to regenerate, sigh utils.verbose_out( 'Using default lag of {} days.'.format(lag), 2) else: with utils.error_handler( "Error for pptsum lag value '{}').".format( val[1])): lag = int(val[1]) date_spec = '{},{}'.format( datetime.strftime( self.date - timedelta(days=lag), '%Y-%m-%d', ), datetime.strftime(self.date, '%Y-%m-%d'), ) inv = self.inventory( dates=date_spec, products=['ppt'], ) inv.process() # because DataInventory object doesn't update inv = self.inventory( dates=date_spec, products=['ppt'], ) if len(inv.data) < lag: utils.verbose_out( '{}: requires {} preceding days ppt ({} found).'. format(key, lag, len(inv.data)), 3, ) continue # go to next product to process imgs = [] asset_fns = [] # have to grab filenames for multiple days for tileobj in inv.data.values(): datobj = tileobj.tiles.values()[0] asset_fns.append( os.path.basename(datobj.assets['_ppt'].filename)) imgs.append(GeoImage(get_bil_vsifile(datobj, '_ppt'))) with self.make_temp_proc_dir() as tmp_dir: tmp_fp = os.path.join(tmp_dir, prod_fn) oimg = GeoImage(tmp_fp, imgs[0]) oimg.SetNoData(-9999) oimg.SetBandName( description + '({} day window)'.format(lag), 1) oimg.SetMeta(self.prep_meta(sorted(asset_fns))) for chunk in oimg.Chunks(): oarr = oimg[0].Read(chunk) * 0.0 # wat for img in imgs: oarr += img[0].Read(chunk) oimg[0].Write(oarr, chunk) oimg.Process() os.rename(tmp_fp, archived_fp) oimg = None # help swig+gdal with GC products.requested.pop(key) self.AddFile(sensor, key, archived_fp) # add product to inventory return products
def process(self, *args, **kwargs): """ Cribbed from super: Make sure all products exist and return those that need processing. """ products = super(prismData, self).process(*args, **kwargs) if len(products) == 0: return bname = os.path.join(self.path, self.basename) for key, val in products.requested.items(): start = datetime.now() # check that we have required assets requiredassets = self.products2assets([val[0]]) # val[0] s.b. key w/o product args missingassets = [] availassets = [] vsinames = {} def get_bil_file(d, a): return filter( lambda f: f.endswith('.bil'), d.assets[a].datafiles(), )[0] for asset in requiredassets: try: bil = get_bil_file(self, asset) except KeyError: missingassets.append(asset) else: availassets.append(asset) vsinames[asset] = bil if not availassets: VerboseOut( 'There are no available assets ({}) on {} for tile {}'. format( str(missingassets), str(self.date), str(self.id), ), 5, ) continue fname = '{}_{}_{}.tif'.format(bname, 'prism', key) if val[0] in ['ppt', 'tmin', 'tmax']: if os.path.lexists(fname): os.remove(fname) os.symlink(vsinames[self._products[key]['assets'][0]], fname) elif val[0] == 'pptsum': try: lag = int(val[1]) except ValueError, TypeError: raise Exception( 'pptsum argument format error (given: {}).') except IndexError: # no argument provided, use # default lag of 3 days SB configurable. lag = 3 fname = re.sub(r'\.tif$', '-{}.tif'.format(lag), fname) VerboseOut('Using default lag of {} days.'.format(lag), 2) if os.path.exists(fname): os.remove(fname) inv = self.inventory( dates='{},{}'.format( datetime.strftime( self.date - timedelta(days=lag - 1), '%Y-%m-%d', ), datetime.strftime(self.date, '%Y-%m-%d'), ), products=['ppt'], ) inv.process() inv = self.inventory( dates='{},{}'.format( datetime.strftime( self.date - timedelta(days=lag - 1), '%Y-%m-%d', ), datetime.strftime(self.date, '%Y-%m-%d'), ), products=['ppt'], ) if len(inv.data) < lag: raise Exception( key + ': requires {} preceding days ppt assets.'.format(lag) + '\nOnly {} days found.'.format(len(inv.data))) imgs = [] for tileobj in inv.data.values(): datobj = tileobj.tiles.values()[0] imgs.append(datobj.open_assets('ppt')) oimg = GeoImage(fname, imgs[0]) for chunk in oimg.Chunks(): oarr = oimg[0].Read(chunk) * 0.0 for img in imgs: print('mean value of image: {}'.format( mean(img[0].Read()))) oarr += img[0].Read(chunk) oimg[0].Write(oarr, chunk) oimg.Process() #oimg = None print('nih') products.requested.pop(key)