def make_dem_mosaic_index_ts(index_tif_fn):
index_txt_fn = index_tif_fn + '-index-map.txt'
out_fn = os.path.splitext(index_tif_fn)[0] + '_ts.tif'
if not os.path.exists(index_tif_fn):
print("Unable to find input file: %s" % index_tif_fn)
return False
if not os.path.exists(index_txt_fn):
print("Unable to find input file: %s" % index_txt_fn)
return False
if os.path.exists(out_fn):
print("Existing ts output found: %s" % out_fn)
return True
#Load dem_mosaic index tif
index_tif_ds = iolib.fn_getds(index_tif_fn)
index_tif = iolib.ds_getma(index_tif_ds)
#Create output array
#index_ts_tif = np.zeros(index_tif.shape, dtype=np.float32)
#index_ts_tif = np.ma.masked_all_like(index_tif)
index_ts_tif = np.ma.masked_all(index_tif.shape, dtype=np.float32)
#Read in dem_mosaic index txt file - should be "fn, index" for each record
index_txt = np.atleast_2d(np.genfromtxt(index_txt_fn, dtype=str))
#Extract Python datetime object from filename (should pull out YYYYMMDD_HHMM)
index_ts = [timelib.fn_getdatetime(fn) for fn in index_txt[:, 0]]
#Convert to desired output timestamp format
#Python ordinal
#index_ts = timelib.dt2o(index_ts)
#YYYYMMDD integer
#index_ts = [ts.strftime('%Y%m%d') for ts in index_ts]
#Decimal year
index_ts = [timelib.dt2decyear(ts) for ts in index_ts]
for n, dt in enumerate(index_ts):
index_ts_tif[index_tif == n] = dt
iolib.writeGTiff(index_ts_tif, out_fn, index_tif_ds, ndv=0)
return True
if __name__ == '__main__':
files = RasterSource(
'/Volumes/warehouse/projects/UofU/geohackweek/galcier_data/khumbu_DEM_32m/'
).file_list()
raster_list = warplib.memwarp_multi_fn(files,
extent='union',
res='min',
t_srs=files[0])
band_data = [iolib.ds_getma(i).filled(np.NaN) for i in raster_list]
x, y = geolib.get_xy_grids(raster_list[0])
time_list = np.array([timelib.fn_getdatetime(fn) for fn in files])
x_band_data = xr.DataArray(np.stack(band_data),
coords={
'lat': y[::, 0],
'lon': x[0, ::],
'time': time_list
},
dims=('time', 'lat', 'lon'),
name='elevation')
x_band_data = x_band_data.sortby('time')
x_band_data['mean'] = x_band_data.mean(dim='time')
x_band_data['count'] = x_band_data.count(dim='time')
# shade = geolib.gdaldem_mem_ma(
#dem_list = [iolib.ds_getma(i) for i in ds_list]
import matplotlib
matplotlib.pyplot.imshow(dem_2007)
matplotlib.pyplot.imshow(dem_2009)
titles = ['2007', '2009']
clim = malib.calcperc(dem_list[0], (2,98))
plot_panels(2, dem_list, clim, titles, 'inferno', 'Elevation (m WGS84)', fn='dem.png')
# Its possible sometimes to get date times from TIFFS but apparently these DEMS don't have any datetimes assigned.
# We can get the date times from the original *(unmerged) files though.
[timelib.fn_getdatetime(fn) for fn in
[
'/Users/elischwat/Downloads/datasetsA/lewis_2009/dtm/lewis_2009_dtm_44.tif',
'/Users/elischwat/Downloads/datasetsA/rainier_2007/rainier_2007_dtm_12.tif',
'/Users/elischwat/Downloads/datasetsA/rainier_2007/rainier_2007_dtm_13.tif',
'/Users/elischwat/Downloads/datasetsA/rainier_2007/rainier_2007_dtm_14.tif',
'/Users/elischwat/Downloads/datasetsA/rainier_2007/rainier_2007_dtm_7.tif',
'/Users/elischwat/Downloads/datasetsA/rainier_2007/rainier_2007_dtm_8.tif',
'/Users/elischwat/Downloads/datasetsA/rainier_2007/rainier_2007_dtm_9.tif'
]]
#Extract timestamps from filenames
t_list = np.array([
datetime.datetime(2007, 4, 22, 0, 0),
datetime.datetime(2009, 4, 22, 0, 0)])
#Compute time differences, convert decimal years
#dem_clim = (1766, 3247)
#SBB
#dem_clim = (2934, 3983)
hs_clim = (1, 255)
for i, dem_fn in enumerate(dem_fn_list):
ax = grid[i]
print(dem_fn)
dem_ds = iolib.fn_getds(dem_fn)
dem = iolib.ds_getma_sub(dem_ds)
dem_hs_fn = os.path.splitext(dem_fn)[0] + '_hs_az315.tif'
if os.path.exists(dem_hs_fn):
dem_hs = iolib.fn_getma_sub(dem_hs_fn)
else:
dem_hs = geolib.gdaldem_mem_ds(dem_ds, 'hillshade', returnma=True)
dt = timelib.fn_getdatetime(dem_fn)
if dt is not None:
title = dt.strftime('%Y-%m-%d')
t = ax.set_title(title, fontdict={'fontsize': 6})
t.set_position([0.5, 0.95])
hs_im = ax.imshow(dem_hs, vmin=hs_clim[0], vmax=hs_clim[1], cmap='gray')
dem_im = ax.imshow(dem,
vmin=dem_clim[0],
vmax=dem_clim[1],
cmap='cpt_rainbow',
alpha=0.5)
ax.set_facecolor('k')
pltlib.hide_ticks(ax)
for ax in grid[i + 1:]:
ax.axis('off')
if os.path.exists(t1_fn) and os.path.exists(t2_fn):
constant_dt = False
print "Preparing timestamp arrays"
t1_ds, t2_ds = warplib.memwarp_multi_fn([t1_fn, t2_fn],
extent=dem1_ds,
res=dem1_ds)
print "Loading timestamps into masked arrays"
t1 = iolib.ds_getma(t1_ds)
t2 = iolib.ds_getma(t2_ds)
#Compute dt in days
t_factor = t2 - t1
t_factor /= 365.25
else:
from datetime import datetime, timedelta
from pygeotools.lib import timelib
t1 = timelib.fn_getdatetime(dem1_fn)
t2 = timelib.fn_getdatetime(dem2_fn)
if t1 is not None and t2 is not None and t1 != t2:
constant_dt = True
dt = t2 - t1
#Might be better to do this with dateutil - not sure about leap years
#from dateutil.relativedelta import relativedelta
#dt = relativedelta(dt1, dt2))
#dt.years
year = timedelta(days=365.25)
t_factor = abs(dt.total_seconds() / year.total_seconds())
print "Time differences is %s, dh/%0.3f" % (dt, t_factor)
else:
print "Unable to extract timestamps for input images"
rates = False
if iolib.fn_check(fn):
temp.append(fn)
f.write(fn + '\n')
else:
print("Unable to find %s" % fn)
f = None
if make_stacks:
dem_fn_list = temp
#print "Generating stack"
#s = malib.DEMStack(fn_list=dem_fn_list, outdir=stackdir, res=res, extent=site_extent, srs=dst_srs, trend=True)
dem_fn_list = np.array(dem_fn_list)
dem_dt_list = np.array(
[timelib.fn_getdatetime(i) for i in dem_fn_list])
#Make annual and seasonal products
if True:
#These are OrderedDict
summer_dict = timelib.dt_filter_rel_annual_idx(dem_dt_list,
min_rel_dt=(8,
1),
max_rel_dt=(10,
31))
spring_dict = timelib.dt_filter_rel_annual_idx(dem_dt_list,
min_rel_dt=(4,
1),
max_rel_dt=(6,
15))
def get_mask(dem_ds,
mask_list,
dem_fn=None,
writeout=False,
outdir=None,
args=None):
mask_list = check_mask_list(mask_list)
if not mask_list or 'none' in mask_list:
newmask = False
else:
#Basename for output files
if outdir is not None:
if not os.path.exists(outdir):
os.makedirs(outdir)
else:
outdir = os.path.split(os.path.realpath(dem_fn))[0]
if dem_fn is not None:
#Extract DEM timestamp
dem_dt = timelib.fn_getdatetime(dem_fn)
out_fn_base = os.path.join(
outdir,
os.path.splitext(os.path.split(dem_fn)[-1])[0])
if args is None:
#Get default values
parser = getparser()
args = parser.parse_args([
'',
])
newmask = True
if 'glaciers' in mask_list:
icemask = get_icemask(dem_ds)
if writeout:
out_fn = out_fn_base + '_ice_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(icemask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(icemask, newmask)
#Need to process NLCD separately, with nearest neighbor inteprolatin
if 'nlcd' in mask_list and args.nlcd_filter != 'none':
rs = 'near'
nlcd_ds = gdal.Open(get_nlcd_fn())
nlcd_ds_warp = warplib.memwarp_multi([
nlcd_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r=rs)[0]
out_fn = None
if writeout:
out_fn = out_fn_base + '_nlcd.tif'
nlcdmask = get_nlcd_mask(nlcd_ds_warp,
filter=args.nlcd_filter,
out_fn=out_fn)
if writeout:
out_fn = os.path.splitext(out_fn)[0] + '_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(nlcdmask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(nlcdmask, newmask)
if 'bareground' in mask_list and args.bareground_thresh > 0:
bareground_ds = gdal.Open(get_bareground_fn())
bareground_ds_warp = warplib.memwarp_multi([
bareground_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r='cubicspline')[0]
out_fn = None
if writeout:
out_fn = out_fn_base + '_bareground.tif'
baregroundmask = get_bareground_mask(
bareground_ds_warp,
bareground_thresh=args.bareground_thresh,
out_fn=out_fn)
if writeout:
out_fn = os.path.splitext(out_fn)[0] + '_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(baregroundmask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(baregroundmask, newmask)
if 'snodas' in mask_list and args.snodas_thresh > 0:
#Get SNODAS snow depth products for DEM timestamp
snodas_min_dt = datetime(2003, 9, 30)
if dem_dt >= snodas_min_dt:
snodas_ds = get_snodas_ds(dem_dt)
if snodas_ds is not None:
snodas_ds_warp = warplib.memwarp_multi([
snodas_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r='cubicspline')[0]
#snow depth values are mm, convert to meters
snodas_depth = iolib.ds_getma(snodas_ds_warp) / 1000.
if snodas_depth.count() > 0:
print(
"Applying SNODAS snow depth filter (masking values >= %0.2f m)"
% args.snodas_thresh)
out_fn = None
if writeout:
out_fn = out_fn_base + '_snodas_depth.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(snodas_depth,
out_fn,
src_ds=dem_ds)
snodas_mask = np.ma.masked_greater(
snodas_depth, args.snodas_thresh)
snodas_mask = ~(np.ma.getmaskarray(snodas_mask))
if writeout:
out_fn = os.path.splitext(out_fn)[0] + '_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(snodas_mask,
out_fn,
src_ds=dem_ds)
newmask = np.logical_and(snodas_mask, newmask)
else:
print(
"SNODAS grid for input location and timestamp is empty"
)
#These tiles cover CONUS
#tile_list=('h08v04', 'h09v04', 'h10v04', 'h08v05', 'h09v05')
if 'modscag' in mask_list and args.modscag_thresh > 0:
modscag_min_dt = datetime(2000, 2, 24)
if dem_dt < modscag_min_dt:
print("Warning: DEM timestamp (%s) is before earliest MODSCAG timestamp (%s)" \
% (dem_dt, modscag_min_dt))
else:
tile_list = get_modis_tile_list(dem_ds)
print(tile_list)
pad_days = 7
modscag_fn_list = get_modscag_fn_list(dem_dt,
tile_list=tile_list,
pad_days=pad_days)
if modscag_fn_list:
modscag_ds = proc_modscag(modscag_fn_list,
extent=dem_ds,
t_srs=dem_ds)
modscag_ds_warp = warplib.memwarp_multi([
modscag_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r='cubicspline')[0]
print(
"Applying MODSCAG fractional snow cover percent filter (masking values >= %0.1f%%)"
% args.modscag_thresh)
modscag_fsca = iolib.ds_getma(modscag_ds_warp)
out_fn = None
if writeout:
out_fn = out_fn_base + '_modscag_fsca.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(modscag_fsca, out_fn, src_ds=dem_ds)
modscag_mask = (modscag_fsca.filled(0) >=
args.modscag_thresh)
modscag_mask = ~(modscag_mask)
if writeout:
out_fn = os.path.splitext(out_fn)[0] + '_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(modscag_mask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(modscag_mask, newmask)
#Use reflectance values to estimate snowcover
if 'toa' in mask_list:
#Use top of atmosphere scaled reflectance values (0-1)
toa_ds = gdal.Open(get_toa_fn(dem_fn))
toa_ds_warp = warplib.memwarp_multi([
toa_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds)[0]
toa_mask = get_toa_mask(toa_ds_warp, args.toa_thresh)
if writeout:
out_fn = out_fn_base + '_toa_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(toa_mask, newmask)
if False:
#Filter based on expected snowline
#Simplest approach uses altitude cutoff
max_elev = 1500
newdem = np.ma.masked_greater(dem, max_elev)
newmask = np.ma.getmaskarray(newdem)
print(
"Generating final mask to use for reference surfaces, and applying to input DEM"
)
#Now invert to use to create final masked array
#True (1) represents "invalid" pixel to match numpy ma convetion
newmask = ~newmask
#Dilate the mask
if args.dilate is not None:
niter = args.dilate
print("Dilating mask with %i iterations" % niter)
from scipy import ndimage
newmask = ~(ndimage.morphology.binary_dilation(~newmask,
iterations=niter))
return newmask
def main():
parser = getparser()
args = parser.parse_args()
#This is output ndv, avoid using 0 for differences
diffndv = -9999
dem1_fn = args.fn1
dem2_fn = args.fn2
if dem1_fn == dem2_fn:
sys.exit('Input filenames are identical')
fn_list = [dem1_fn, dem2_fn]
print("Warping DEMs to same res/extent/proj")
#This will check input param for validity, could do beforehand
dem1_ds, dem2_ds = warplib.memwarp_multi_fn(fn_list, extent=args.te, res=args.tr, t_srs=args.t_srs)
outdir = args.outdir
if outdir is None:
outdir = os.path.dirname(os.path.abspath(dem1_fn))
if not os.path.exists(outdir):
os.makedirs(outdir)
outprefix = os.path.splitext(os.path.split(dem1_fn)[1])[0]+'_'+os.path.splitext(os.path.split(dem2_fn)[1])[0]
print("Loading input DEMs into masked arrays")
dem1 = iolib.ds_getma(dem1_ds, 1)
dem2 = iolib.ds_getma(dem2_ds, 1)
#Compute dz/dt rates if possible, in m/yr
rates = True
if rates:
#Extract basename
#This was a hack to work with timestamp array filenames that have geoid offset applied
adj = ''
if '-adj' in dem1_fn:
adj = '-adj'
dem1_fn_base = re.sub(adj, '', os.path.splitext(dem1_fn)[0])
dem2_fn_base = re.sub(adj, '', os.path.splitext(dem2_fn)[0])
#Attempt to load timestamp arrays (for mosaics) if present
t1_fn = dem1_fn_base+'_ts.tif'
t2_fn = dem2_fn_base+'_ts.tif'
if os.path.exists(t1_fn) and os.path.exists(t2_fn):
print("Preparing timestamp arrays")
t1_ds, t2_ds = warplib.memwarp_multi_fn([t1_fn, t2_fn], extent=dem1_ds, res=dem1_ds)
print("Loading timestamps into masked arrays")
t1 = iolib.ds_getma(t1_ds)
t2 = iolib.ds_getma(t2_ds)
#Compute dt in days
t_factor = t2 - t1
t_factor /= 365.25
else:
#Attempt to extract timestamps from input filenames
t1 = timelib.fn_getdatetime(dem1_fn)
t2 = timelib.fn_getdatetime(dem2_fn)
if t1 is not None and t2 is not None and t1 != t2:
dt = t2 - t1
year = timedelta(days=365.25)
t_factor = abs(dt.total_seconds()/year.total_seconds())
print("Time differences is %s, dh/%0.3f" % (dt, t_factor))
else:
print("Unable to extract timestamps for input images")
rates = False
#Compute relative elevation difference with Eulerian approach
print("Computing eulerian elevation difference")
diff_euler = dem2 - dem1
#Check to make sure inputs actually intersect
#if not np.any(~dem1.mask*~dem2.mask):
if diff_euler.count() == 0:
sys.exit("No valid overlap between input DEMs")
if True:
print("Eulerian elevation difference stats:")
diff_euler_stats = malib.print_stats(diff_euler)
diff_euler_med = diff_euler_stats[5]
if True:
print("Writing Eulerian elevation difference map")
dst_fn = os.path.join(outdir, outprefix+'_dz_eul.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler, dst_fn, dem1_ds, ndv=diffndv)
if rates:
print("Writing Eulerian rate map")
dst_fn = os.path.join(outdir, outprefix+'_dz_eul_rate.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler/t_factor, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing Eulerian relative elevation difference map")
diff_euler_rel = diff_euler - diff_euler_med
dst_fn = os.path.join(outdir, outprefix+'_dz_eul_rel.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler_rel, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing out DEM2 with median elevation difference removed")
dst_fn = os.path.splitext(dem2_fn)[0]+'_med'+diff_euler_med+'.tif'
print(dst_fn)
iolib.writeGTiff(dem2 - diff_euler_med, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing Eulerian elevation difference percentage map")
diff_euler_perc = 100.0*diff_euler/dem1
dst_fn = os.path.join(outdir, outprefix+'_dz_eul_perc.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler_perc, dst_fn, dem1_ds, ndv=diffndv)
def main():
parser = getparser()
args = parser.parse_args()
#Write out all mask products for the input DEM
writeall = True
mask_glaciers = True
if args.no_icemask:
mask_glaciers = False
#Define top-level directory containing DEM
topdir = os.getcwd()
#This directory should contain nlcd grid, glacier outlines
datadir = iolib.get_datadir()
dem_fn = args.dem_fn
dem_ds = gdal.Open(dem_fn)
print(dem_fn)
#Extract DEM timestamp
dem_dt = timelib.fn_getdatetime(dem_fn)
#This will hold datasets for memwarp and output processing
ds_dict = OrderedDict()
ds_dict['dem'] = dem_ds
ds_dict['lulc'] = None
#lulc_source = get_lulc_source(dem_ds)
#lulc_ds_full = get_lulc_ds_full(dem_ds)
#ds_dict['lulc'] = lulc_ds_full
ds_dict['snodas'] = None
if args.snodas:
#Get SNODAS snow depth products for DEM timestamp
snodas_min_dt = datetime(2003, 9, 30)
if dem_dt >= snodas_min_dt:
snodas_outdir = os.path.join(datadir, 'snodas')
if not os.path.exists(snodas_outdir):
os.makedirs(snodas_outdir)
snodas_ds = get_snodas(dem_dt, snodas_outdir)
if snodas_ds is not None:
ds_dict['snodas'] = snodas_ds
ds_dict['modscag'] = None
#Get MODSCAG products for DEM timestamp
#These tiles cover CONUS
#tile_list=('h08v04', 'h09v04', 'h10v04', 'h08v05', 'h09v05')
if args.modscag:
modscag_min_dt = datetime(2000, 2, 24)
if dem_dt < modscag_min_dt:
print("\nWarning: DEM timestamp (%s) is before earliest MODSCAG timestamp (%s)\nSkipping..." \
% (dem_dt, modscag_min_dt))
else:
tile_list = get_modis_tile_list(dem_ds)
print(tile_list)
pad_days = 7
modscag_outdir = os.path.join(datadir, 'modscag')
if not os.path.exists(modscag_outdir):
os.makedirs(modscag_outdir)
modscag_fn_list = get_modscag(dem_dt, modscag_outdir, tile_list,
pad_days)
if modscag_fn_list:
modscag_ds = proc_modscag(modscag_fn_list,
extent=dem_ds,
t_srs=dem_ds)
ds_dict['modscag'] = modscag_ds
#TODO: need to clean this up
#Better error handling
#Disabled for now
#Use reflectance values to estimate snowcover
ds_dict['toa'] = None
if args.toa:
#Use top of atmosphere scaled reflectance values (0-1)
toa_ds = get_toa_ds(dem_fn)
ds_dict['toa'] = toa_ds
#Cull all of the None ds from the ds_dict
for k, v in ds_dict.items():
if v is None:
del ds_dict[k]
#Warp all masks to DEM extent/res
#Note: use cubicspline here to avoid artifacts with negative values
if len(ds_dict) > 0:
ds_list = warplib.memwarp_multi(ds_dict.values(),
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r='cubicspline')
#Update
for n, key in enumerate(ds_dict.keys()):
ds_dict[key] = ds_list[n]
#lulc_ds_warp = get_lulc_ds_warp(dem_ds)
#ds_dict['lulc'] = lulc_ds_warp
print(' ')
#Need better handling of ds order based on input ds here
dem = iolib.ds_getma(ds_dict['dem'])
#Initialize the mask
#True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked
newmask = ~(np.ma.getmaskarray(dem))
#Basename for output files
out_fn_base = os.path.splitext(dem_fn)[0]
#Generate a rockmask
if args.filter == 'none' or args.bareground_thresh == 0:
print("Skipping LULC filter")
else:
#Note: these now have RGI glacier polygons removed
#if 'lulc' in ds_dict.keys():
#We are almost always going to want LULC mask
rockmask = get_lulc_mask(dem_ds, mask_glaciers=mask_glaciers, \
filter=args.filter, bareground_thresh=args.bareground_thresh, out_fn=out_fn_base)
if writeall:
out_fn = out_fn_base + '_rockmask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(rockmask, out_fn, src_ds=ds_dict['dem'])
newmask = np.logical_and(rockmask, newmask)
if 'snodas' in ds_dict.keys():
#SNODAS snow depth filter
snodas_thresh = args.snodas_thresh
#snow depth values are mm, convert to meters
snodas_depth = iolib.ds_getma(ds_dict['snodas']) / 1000.
if snodas_depth.count() > 0:
print(
"Applying SNODAS snow depth filter (masking values >= %0.2f m)"
% snodas_thresh)
if writeall:
out_fn = out_fn_base + '_snodas_depth.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(snodas_depth, out_fn, src_ds=ds_dict['dem'])
snodas_mask = np.ma.masked_greater(snodas_depth, snodas_thresh)
#This should be 1 for valid surfaces with no snow, 0 for snowcovered surfaces
snodas_mask = ~(np.ma.getmaskarray(snodas_mask))
if writeall:
out_fn = out_fn_base + '_snodas_mask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(snodas_mask, out_fn, src_ds=ds_dict['dem'])
newmask = np.logical_and(snodas_mask, newmask)
else:
print(
"SNODAS grid for input location and timestamp is empty!\nSkipping...\n"
)
if 'modscag' in ds_dict.keys():
#MODSCAG percent snowcover
modscag_thresh = args.modscag_thresh
print(
"Applying MODSCAG fractional snow cover percent filter (masking values >= %0.1f%%)"
% modscag_thresh)
modscag_perc = iolib.ds_getma(ds_dict['modscag'])
if writeall:
out_fn = out_fn_base + '_modscag_perc.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(modscag_perc, out_fn, src_ds=ds_dict['dem'])
modscag_mask = (modscag_perc.filled(0) >= modscag_thresh)
#This should be 1 for valid surfaces with no snow, 0 for snowcovered surfaces
modscag_mask = ~(modscag_mask)
if writeall:
out_fn = out_fn_base + '_modscag_mask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(modscag_mask, out_fn, src_ds=ds_dict['dem'])
newmask = np.logical_and(modscag_mask, newmask)
if 'toa' in ds_dict.keys():
#TOA reflectance filter
#This should be 1 for valid surfaces, 0 for snowcovered surfaces
toa_mask = get_toa_mask(ds_dict['toa'], args.toa_thresh)
if writeall:
out_fn = out_fn_base + '_toamask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_mask, out_fn, src_ds=ds_dict['dem'])
newmask = np.logical_and(toa_mask, newmask)
if False:
#Filter based on expected snowline
#Simplest approach uses altitude cutoff
max_elev = 1500
newdem = np.ma.masked_greater(dem, max_elev)
newmask = np.ma.getmaskarray(newdem)
print(
"Generating final mask to use for reference surfaces, and applying to input DEM"
)
#Now invert to use to create final masked array
newmask = ~newmask
#Dilate the mask
if args.dilate is not None:
niter = args.dilate
print("Dilating mask with %i iterations" % niter)
from scipy import ndimage
newmask = ~(ndimage.morphology.binary_dilation(~newmask,
iterations=niter))
#Check that we have enough pixels, good distribution
#Apply mask to original DEM - use these surfaces for co-registration
newdem = np.ma.array(dem, mask=newmask)
min_validpx_count = 100
min_validpx_std = 10
validpx_count = newdem.count()
validpx_std = newdem.std()
print("\n%i valid pixels in output ref.tif" % validpx_count)
print("%0.2f m std output ref.tif\n" % validpx_std)
#if (validpx_count > min_validpx_count) and (validpx_std > min_validpx_std):
if (validpx_count > min_validpx_count):
#Write out final mask
out_fn = out_fn_base + '_ref.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(newdem, out_fn, src_ds=ds_dict['dem'])
else:
print("Not enough valid pixels!")
parser.add_argument('-dem1_fn', type=str, help='DEM(t1) filename')
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-dem2_fn', type=str, default=None, help='DEM(t2) filename')
group.add_argument('-dz_fn', type=str, default=None, help='Snow depth map filename')
density_choices = ['sturm', 'snotel', 'constant']
parser.add_argument('-density', nargs=1, type=float, default=None, help='Specify density (g/cc)')
parser.add_argument('-filter', action='store_true', help='Filter SWE map to remove blunders, smooth, and fill gaps')
parser.add_argument('-prism', action='store_true', help='Include PRISM precip in analysis/plots')
return parser
#def main():
parser = getparser()
args = parser.parse_args()
dem1_fn = args.dem1_fn
dem1_ts = timelib.fn_getdatetime(dem1_fn)
res = 'min'
save = True
#For testing
#res = 64
if args.dem2_fn is not None:
dem2_fn = args.dem2_fn
print("Warping DEMs to same res/extent/proj")
#This will check input param for validity, could do beforehand
dem1_ds, dem2_ds = warplib.memwarp_multi_fn([dem1_fn, dem2_fn], extent='intersection', res=res, t_srs='first')
print("Loading input DEMs into masked arrays")
dem1 = iolib.ds_getma(dem1_ds, 1)
dem2 = iolib.ds_getma(dem2_ds, 1)
dem2_ts = timelib.fn_getdatetime(dem2_fn)
dz = dem2 - dem1
titles = ['1970', '2008', '2015']
clim = malib.calcperc(dem_list[0], (2, 98))
plot3panel(dem_list,
clim,
titles,
'inferno',
'Elevation (m WGS84)',
fn='dem.png')
#ddem_1970_2015 = dem_1970 - dem_2015
#ddem_2008_2015 = dem_2008 - dem_2015
#ddem_1970_2008 = dem_1970 - dem_2008
#Extract timestamps from filenames
t_list = np.array([timelib.fn_getdatetime(fn) for fn in dem_fn_list])
#Compute time differences, convert decimal years
dt_list = [timelib.timedelta2decyear(d) for d in np.diff(t_list)]
dt_list.append(dt_list[0] + dt_list[1])
#Calculate elevation difference for each time period
dh_list = [dem_2008 - dem_1970, dem_2015 - dem_2008, dem_2015 - dem_1970]
titles = [
'1970 to 2008 (%0.1f yr)' % dt_list[0],
'2008 to 2015 (%0.1f yr)' % dt_list[1],
'1970 to 2015 (%0.1f yr)' % dt_list[2]
]
plot3panel(dh_list, (-30, 30),
titles,
'RdBu',
'Elevation Change (m)',
def parse_pc_align_log(fn):
import re
error_dict = None
#Determine log filename
import glob
log_fn = glob.glob(fn.rsplit('-DEM', 1)[0]+'*.log')
if not log_fn:
log_fn = glob.glob(fn.rsplit('-DEM', 1)[0]+'*align/*.log')
if not log_fn:
print "Failed to locate align log for %s" % fn
else:
log_fn = log_fn[0]
print(log_fn)
f = open(log_fn)
error_dict = {}
error_dict['File'] = fn
error_dict['Date'] = timelib.fn_getdatetime(fn)
#This handles cases where no sampling was performed
error_dict['Input Sampled 16th Percentile Error'] = np.nan
error_dict['Input Sampled Median Error'] = np.nan
error_dict['Input Sampled 84th Percentile Error'] = np.nan
error_dict['Input Sampled Error Spread'] = np.nan
error_dict['Output Sampled 16th Percentile Error'] = np.nan
error_dict['Output Sampled Median Error'] = np.nan
error_dict['Output Sampled 84th Percentile Error'] = np.nan
error_dict['Output Sampled Error Spread'] = np.nan
#error_dict['Translation vector (North-East-Down, meters)'] = [np.nan, np.nan, np.nan]
#Set default reference type to point
error_dict['Ref type'] = 'point'
temp = []
for line in f:
key = 'Loaded points'
if key in line:
temp.append(int(re.split(':', line.rstrip())[1]))
key = 'Number of errors'
if key in line:
error_dict[key] = int(re.split(':', line.rstrip())[1])
key = 'Input: error percentile'
if key in line:
line_a = re.split(': |, ', line.rstrip())
error_dict['Input 16th Percentile Error'] = float(line_a[3])
error_dict['Input Median Error'] = float(line_a[5])
error_dict['Input 84th Percentile Error'] = float(line_a[7])
"""
key = 'Input: error mean'
if key in line:
line_a = re.split(': |, ', line.rstrip())
error_dict['Input Mean Error'] = float(line_a[2])
error_dict['Input Std Error'] = float(line_a[4])
"""
#This pulls the line
#Input: mean of smallest errors: 25%: 7.82061, 50%: 9.71931, 75%: 10.9917, 100%: 12.2715
#Want the final value
key = 'Input: mean'
if key in line:
line_a = re.split(': |, ', line.rstrip())
error_dict['Input Mean Error'] = float(line_a[-1])
key = 'Output: error percentile'
if key in line:
line_a = re.split(': |, ', line.rstrip())
error_dict['Output 16th Percentile Error'] = float(line_a[3])
error_dict['Output Median Error'] = float(line_a[5])
error_dict['Output 84th Percentile Error'] = float(line_a[7])
"""
key = 'Output: error mean'
if key in line:
line_a = re.split(': |, ', line.rstrip())
error_dict['Output Mean Error'] = float(line_a[2])
error_dict['Output Std Error'] = float(line_a[4])
"""
key = 'Output: mean'
if key in line:
line_a = re.split(': |, ', line.rstrip())
error_dict['Output Mean Error'] = float(line_a[-1])
key = 'Translation vector (Cartesian, meters)'
#Previous versions of pc_align output this
#key = 'Translation vector (meters)'
if key in line:
error_dict['Translation vector (Cartesian, meters)'] = list(float(i) for i in re.split('Vector3\(', line.rstrip())[1][:-1].split(','))
#error_dict['Translation vector (meters)'] = list(float(i) for i in re.split('Vector3\(', line.rstrip())[1][:-1].split(','))
key = 'Translation vector (North-East-Down, meters)'
if key in line:
error_dict['Translation vector (North-East-Down, meters)'] = list(float(i) for i in re.split('Vector3\(', line.rstrip())[1][:-1].split(','))
key = 'Translation vector magnitude (meters)'
if key in line:
error_dict[key] = float(re.split(':', line.rstrip())[1])
key = 'Translation vector (lat,lon,z)'
if key in line:
error_dict[key] = list(float(i) for i in re.split('Vector3\(', line.rstrip())[1][:-1].split(','))
shift_proj = shift_ll2proj(fn, error_dict[key])
key = 'Translation vector (Proj meters)'
error_dict[key] = shift_proj
#This is the output from the point sampling post-alignment
key = 'Error percentiles'
if key in line:
#This is a hack to capture both sampling of input and output
if 'Output Sampled 16th Percentile Error' in error_dict:
error_dict['Input Sampled 16th Percentile Error'] = error_dict['Output Sampled 16th Percentile Error']
error_dict['Input Sampled Median Error'] = error_dict['Output Sampled Median Error']
error_dict['Input Sampled 84th Percentile Error'] = error_dict['Output Sampled 84th Percentile Error']
error_dict['Input Sampled Error Spread'] = error_dict['Output Sampled Error Spread']
line_a = re.split(': |, ', line.rstrip())
error_dict['Output Sampled 16th Percentile Error'] = float(line_a[2])
error_dict['Output Sampled Median Error'] = float(line_a[4])
error_dict['Output Sampled 84th Percentile Error'] = float(line_a[6])
error_dict['Output Sampled Error Spread'] = float(line_a[6]) - float(line_a[2])
#key = 'compute_dh'
#Note: these are not computed for absolute values by compute_dh
key = 'count:'
if key in line:
error_dict['Ref type'] = 'grid'
#This is a hack to capture both sampling of input and output
if 'Output Sampled 16th Percentile Error' in error_dict:
error_dict['Input Sampled 16th Percentile Error'] = error_dict['Output Sampled 16th Percentile Error']
error_dict['Input Sampled Median Error'] = error_dict['Output Sampled Median Error']
error_dict['Input Sampled 84th Percentile Error'] = error_dict['Output Sampled 84th Percentile Error']
error_dict['Input Sampled Error Spread'] = error_dict['Output Sampled Error Spread']
#Assume the following format for stats:
#count: 349835 min: -51.39 max: 22.00 mean: 0.29 std: 0.49 med: 0.28 mad: 0.37 \
#q1: 0.04 q2: 0.54 iqr: 0.50 mode: 0.29 p16: -0.07 p84: 0.66 spread: 0.37
line_a = re.split(': | ', line.rstrip())
error_dict['Output Sampled 16th Percentile Error'] = float(line_a[23])
error_dict['Output Sampled Median Error'] = float(line_a[11])
error_dict['Output Sampled 84th Percentile Error'] = float(line_a[25])
error_dict['Output Sampled Error Spread'] = float(line_a[25]) - float(line_a[23])
key = 'Mean error'
if key in line:
if 'Output Sampled Mean Error' in error_dict:
error_dict['Input Sampled Mean Error'] = error_dict['Output Sampled Mean Error']
error_dict['Output Sampled Mean Error'] = float(re.split(':', line.rstrip())[1])
key = 'RMSE'
if key in line:
if 'Output Sampled RMSE' in error_dict:
error_dict['Input Sampled RMSE'] = error_dict['Output Sampled RMSE']
error_dict['Output Sampled RMSE'] = float(re.split(':', line.rstrip())[1])
key = 'Absolute Median Error'
if key in line:
if 'Output Absolute Median Error' in error_dict:
error_dict['Input Absolute Median Error'] = error_dict['Output Absolute Median Error']
error_dict['Output Absolute Median Error'] = float(re.split(':', line.rstrip())[1])
error_dict['Source points'] = temp[0]
error_dict['Reference points'] = temp[1]
return error_dict
def main():
parser = getparser()
args = parser.parse_args()
#Define top-level directory containing raster
topdir = os.getcwd()
#This directory will store SNODAS products
#Use centralized directory, default is $HOME/data/
#datadir = iolib.get_datadir()
datadir = args.datadir
if not os.path.exists(datadir):
os.makedirs(datadir)
fn = args.fn
ds = gdal.Open(fn)
print(fn)
#Extract timestamp from input filename
dt = timelib.fn_getdatetime(fn)
#If date is specified, extract timestamp
if args.date is not None:
dt = timelib.fn_getdatetime(args.date)
out_fn_base = os.path.splitext(fn)[0]
snodas_min_dt = datetime(2003, 9, 30)
if dt < snodas_min_dt:
sys.exit("Timestamp is earlier than valid SNODAS model range")
#snow depth values are mm, convert to meters
snodas_outdir = os.path.join(datadir, 'snodas')
if not os.path.exists(snodas_outdir):
os.makedirs(snodas_outdir)
snodas_ds_full = get_snodas(dt, snodas_outdir)
snodas_ds = warplib.memwarp_multi([
snodas_ds_full,
],
res='source',
extent=ds,
t_srs=ds,
r='cubicspline')[0]
snodas_depth = iolib.ds_getma(snodas_ds) / 1000.
if snodas_depth.count() > 0:
#Write out at original resolution
out_fn = out_fn_base + '_snodas_depth.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(snodas_depth, out_fn, src_ds=snodas_ds)
#Warp to match input raster
#Note: use cubicspline here to avoid artifacts with negative values
ds_out = warplib.memwarp_multi([
snodas_ds,
],
res=ds,
extent=ds,
t_srs=ds,
r='cubicspline')[0]
#Write out warped version
snodas_depth = iolib.ds_getma(ds_out) / 1000.
out_fn = out_fn_base + '_snodas_depth_warp.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(snodas_depth, out_fn, src_ds=ds_out)
else:
print("SNODAS grid for input location and timestamp is empty!")
def main():
parser = getparser()
args = parser.parse_args()
#This is output ndv, avoid using 0 for differences
diffndv = -9999
r1_fn = args.fn1
r2_fn = args.fn2
if r1_fn == r2_fn:
sys.exit('Input filenames are identical')
fn_list = [r1_fn, r2_fn]
outdir = args.outdir
if outdir is None:
outdir = os.path.dirname(os.path.abspath(r1_fn))
if not os.path.exists(outdir):
os.makedirs(outdir)
outprefix = os.path.splitext(os.path.split(r1_fn)[1])[0]+'_'+os.path.splitext(os.path.split(r2_fn)[1])[0]
#Compute dz/dt rate if possible, in m/yr
if args.rate:
#Extract basename
#This was a hack to work with timestamp array filenames that have geoid offset applied
adj = ''
if '-adj' in r1_fn:
adj = '-adj'
r1_fn_base = re.sub(adj, '', os.path.splitext(r1_fn)[0])
r2_fn_base = re.sub(adj, '', os.path.splitext(r2_fn)[0])
#Attempt to load ordinal timestamp arrays (for mosaics) if present
"""
import glob
t1_fn = glob.glob(r1_fn_base+'*_ts*.tif')
t2_fn = glob.glob(r1_fn_base+'*_ts*.tif')
t_unit = 'year'
"""
t1_fn = r1_fn_base+'_ts.tif'
t2_fn = r2_fn_base+'_ts.tif'
t_unit = 'day'
if not os.path.exists(t1_fn) and not os.path.exists(t2_fn):
#Try to find processed output from r_mosaic index
#These are decimal years
t1_fn = r1_fn_base+'index_ts.tif'
t2_fn = r2_fn_base+'index_ts.tif'
t_unit = 'year'
print(t1_fn, t2_fn)
if os.path.exists(t1_fn) and os.path.exists(t2_fn):
fn_list.extend([t1_fn, t2_fn])
else:
#Attempt to extract timestamps from input filenames
t1 = timelib.fn_getdatetime(r1_fn)
t2 = timelib.fn_getdatetime(r2_fn)
if t1 is not None and t2 is not None and t1 != t2:
dt = t2 - t1
year = timedelta(days=365.25)
t_factor = abs(dt.total_seconds()/year.total_seconds())
print("Time differences is %s, dh/%0.3f" % (dt, t_factor))
else:
print("Unable to extract timestamps for input images")
args.rate = False
print("Warping rasters to same res/extent/proj")
#This will check input param for validity, could do beforehand
ds_list = warplib.memwarp_multi_fn(fn_list, extent=args.te, res=args.tr, t_srs=args.t_srs, r='cubic')
r1_ds = ds_list[0]
r2_ds = ds_list[1]
print("Loading input rasters into masked arrays")
r1 = iolib.ds_getma(r1_ds, 1)
r2 = iolib.ds_getma(r2_ds, 1)
#Compute relative difference
print("Computing raster difference")
diff = r2 - r1
#Check to make sure inputs actually intersect
if diff.count() == 0:
sys.exit("No valid overlap between input rasters")
if len(fn_list) == 4:
t1_ds = ds_list[2]
t2_ds = ds_list[3]
print("Loading timestamps into masked arrays")
t1 = iolib.ds_getma(t1_ds)
t2 = iolib.ds_getma(t2_ds)
#Compute dt in years
t_factor = t2 - t1
if t_unit == 'day':
t_factor /= 365.25
if True:
print("Raster difference stats:")
diff_stats = malib.print_stats(diff)
diff_med = diff_stats[5]
if True:
print("Writing raster difference map")
dst_fn = os.path.join(outdir, outprefix+'_diff.tif')
print(dst_fn)
iolib.writeGTiff(diff, dst_fn, r1_ds, ndv=diffndv)
if args.rate:
print("Writing rate map")
dst_fn = os.path.join(outdir, outprefix+'_diff_rate.tif')
print(dst_fn)
iolib.writeGTiff(diff/t_factor, dst_fn, r1_ds, ndv=diffndv)
if len(fn_list) == 4:
print("Writing time difference map")
dst_fn = os.path.join(outdir, outprefix+'_diff_dt.tif')
print(dst_fn)
iolib.writeGTiff(t_factor, dst_fn, r1_ds, ndv=diffndv)
if False:
print("Writing relative raster difference map")
diff_rel = diff - diff_med
dst_fn = os.path.join(outdir, outprefix+'_diff_rel.tif')
print(dst_fn)
iolib.writeGTiff(diff_rel, dst_fn, r1_ds, ndv=diffndv)
if False:
print("Writing out raster2 with median difference removed")
dst_fn = os.path.splitext(r2_fn)[0]+'_med'+diff_med+'.tif'
print(dst_fn)
iolib.writeGTiff(r2 - diff_med, dst_fn, r1_ds, ndv=diffndv)
if False:
print("Writing raster difference percentage map (relative to raster1)")
diff_perc = 100.0*diff/r1
dst_fn = os.path.join(outdir, outprefix+'_diff_perc.tif')
print(dst_fn)
iolib.writeGTiff(diff_perc, dst_fn, r1_ds, ndv=diffndv)
