def corr_filter_aster(fn_dem, fn_corr, threshold=80):
dem = GeoImg(fn_dem)
corr = GeoImg(fn_corr)
out = dem.copy()
corr.img[corr.img < threshold] = 0
rem_open = binary_opening(corr.img, structure=disk(5))
out.img[~rem_open] = np.nan
return out
def generate_panchrome(imgname, outname=None, out_dir='.', interactive=False):
if outname is None:
outname = imgname + '_B8.TIF'
B5 = GeoImg(imgname + '_B5.TIF')
B4 = GeoImg(imgname + '_B4.TIF')
B3 = GeoImg(imgname + '_B3.TIF')
B2 = GeoImg(imgname + '_B2.TIF')
B8sim = 0.45 * B4.img + 0.2 * B3.img + 0.25 * B2.img + 0.1 * B5.img
B8 = B4.copy(new_raster=B8sim)
B8.write(outname, out_folder=out_dir)
if interactive:
return B8
def main():
parser = _argparser()
args = parser.parse_args()
if args.outputscene is None:
args.outputscene = args.inputscene
outfilename = args.outputscene + "_B8.TIF"
# first, read in the bands (4, 3, 2)
B4 = GeoImg( args.inputscene + "_B4.TIF" )
B3 = GeoImg( args.inputscene + "_B3.TIF" )
B2 = GeoImg( args.inputscene + "_B2.TIF" )
# now, make a new band
B8sim = 0.5 * B4.img + 0.2 * B3.img + 0.3 * B2.img
B8 = B4.copy(new_raster=B8sim)
B8.write(outfilename)
def main():
np.seterr(all='ignore')
parser = _argparser()
args = parser.parse_args()
if args.outfilename is None:
args.outfilename = args.dem.rsplit('.tif', 1)[0] + '_XAJ.tif'
# read in DEM to apply corrections to
dem = GeoImg(args.dem)
# apply optional correlation mask
if args.corr_mask is not None:
corr = GeoImg(args.corr_mask)
dem.img[corr.img < args.threshold] = np.nan
ang_mapN = GeoImg('TrackAngleMap_3N.tif')
ang_mapB = GeoImg('TrackAngleMap_3B.tif')
ang_mapNB = ang_mapN.copy(
new_raster=np.array(np.divide(ang_mapN.img + ang_mapB.img, 2)))
# first apply the cross-track correction
myang = np.deg2rad(ang_mapNB.img)
xxr, _ = mt.get_xy_rot(dem, myang)
pcoef = read_params_file(args.cross_track_params)
cross_correction = mt.fitfun_polynomial(xxr, pcoef)
dem.img = dem.img + cross_correction
# next, apply *either* the full soluton, or just the low-frequency solution
xxn_mat, xxb_mat = mt.get_atrack_coord(dem, ang_mapN, ang_mapB)
if not args.low_freq:
scoef = read_params_file(args.along_track_full)
else:
scoef = read_params_file(args.along_track_low)
sinmod = mt.fitfun_sumofsin_2angle(xxn_mat, xxb_mat, scoef)
along_correction = np.reshape(sinmod, dem.img.shape)
dem.img = dem.img + along_correction
dem.write(args.outfilename, out_folder=args.input_dir)
print("Loading {} data files.".format(yr))
ifsar = GeoImg('{}/seak.ifsar.{}.dem.30m_adj.tif'.format(yr, yr))
ifsar_srtm = GeoImg('{}/ifsar_srtm_{}_dh.tif'.format(yr, yr))
srtm = GeoImg('{}/SRTM_SE_Alaska_30m_{}IfSAR_adj.tif'.format(yr, yr))
valid_area = np.isfinite(ifsar.img)
glac_shp = '../outlines/01_rgi60_Alaska_GlacierBay_02km_UTM_{}.shp'.format(yr)
glacier_mask = it.create_mask_from_shapefile(ifsar, glac_shp)
mask_geo = mask_full.reproject(ifsar_srtm)
corrs = [35, 50, 70, 80, 90, 95]
for i, corr in enumerate(corrs):
corr_mask = mask_geo.img < corr
masked_ifsar = ifsar.copy()
masked_ifsar.img[corr_mask] = np.nan
masked_ifsar_srtm = ifsar_srtm.copy()
masked_ifsar_srtm.img[corr_mask] = np.nan
print("Linear interpolation of dH")
ifsar_srtm_lin_interp = dt.fill_holes(masked_ifsar_srtm, dt.linear, valid_area=valid_area)
ifsar_srtm_lin_interp.write('ifsar_srtm_{}_dHinterp.tif'.format(yr),
out_folder='filled_ddems/{}/void{}'.format(yr, corr))
print("Linear interpolation of Z")
ifsar_lin_interp = dt.fill_holes(masked_ifsar, dt.linear, valid_area=valid_area)
ifsar_lin_interp.write('ifsar_zinterp.tif',
out_folder='filled_ddems/{}/void{}'.format(yr, corr))
def create_mmaster_stack(filelist,
extent=None,
res=None,
epsg=None,
outfile='mmaster_stack.nc',
clobber=False,
uncert=False,
coreg=False,
ref_tiles=None,
exc_mask=None,
inc_mask=None,
outdir='tmp',
filt_dem=None,
add_ref=False,
add_corr=False,
latlontile_nodata=None,
filt_mm_corr=False,
l1a_zipped=False,
y0=1900,
tmptag=None):
"""
Given a list of DEM files, create a stacked NetCDF file.
:param filelist: List of DEM filenames to stack.
:param extent: Spatial extent of DEMs to limit stack to [xmin, xmax, ymin, ymax].
:param res: Output spatial resolution of DEMs.
:param epsg: EPSG code of output CRS.
:param outfile: Filename for output NetCDF file.
:param clobber: clobber existing dataset when creating NetCDF file.
:param uncert: Include uncertainty variable in the output NetCDF.
:param coreg: Co-register DEMs to an input DEM (given by a shapefile of tiles).
:param ref_tiles: Filename of input reference DEM tiles.
:param exc_mask: Filename of exclusion mask (i.e., glaciers) to use in co-registration
:param inc_mask: Filename of inclusion mask (i.e., land) to use in co-registration.
:param outdir: Output directory for temporary files.
:param filt_dem: Filename of DEM to filter elevation differences to.
:param add_ref: Add reference DEM as a stack variable
:param add_corr: Add correlation masks as a stack variable
:param latlontile_nodata: Apply nodata for a lat/lon tile footprint to avoid overlapping and simplify xarray merging
:param filt_mm_corr: Filter MMASTER DEM with correlation mask out of mmaster_tools when stacking (disk space),
:param l1a_zipped: Use if files have been zipped to save on space.
:param y0: Year 0 to reference NetCDF time variable to.
:param tmptag: string to append to temporary files.
:type filelist: array-like
:type extent: array-like
:type res: float
:type epsg: int
:type outfile: str
:type clobber: bool
:type uncert: bool
:type coreg: bool
:type ref_tiles: str
:type exc_mask: str
:type inc_mask: str
:type outdir: str
:type filt_dem: str
:type add_ref: bool
:type add_corr: bool
:type latlontile_nodata: str
:type filt_mm_corr: bool
:type l1a_zipped: bool
:type y0: float
:type tmptag: str
:returns nco: NetCDF Dataset of stacked DEMs.
"""
if extent is not None:
if type(extent) in [list, tuple]:
xmin, xmax, ymin, ymax = extent
elif type(extent) is Polygon:
x, y = extent.boundary.coords.xy
xmin, xmax = min(x), max(x)
ymin, ymax = min(y), max(y)
else:
raise ValueError(
'extent should be a list, tuple, or shapely.Polygon')
else:
xmin, xmax, ymin, ymax = get_common_bbox(filelist, epsg)
print('Searching for intersecting DEMs among the list of ' +
str(len(filelist)) + '...')
# check if each footprint falls within our given extent, and if not - remove from the list.
if l1a_zipped:
# if l1a are zipped, too long to extract archives and read extent from rasters ; so read metadata instead
l1a_filelist = [
fn for fn in filelist if os.path.basename(fn)[0:3] == 'AST'
]
rest_filelist = [fn for fn in filelist if fn not in l1a_filelist]
l1a_inters = get_footprints_inters_ext(l1a_filelist,
[xmin, ymin, xmax, ymax],
epsg,
use_l1a_met=True)
rest_inters = get_footprints_inters_ext(rest_filelist,
[xmin, ymin, xmax, ymax], epsg)
filelist = l1a_inters + rest_inters
else:
filelist = get_footprints_inters_ext(filelist,
[xmin, ymin, xmax, ymax], epsg)
print('Found ' + str(len(filelist)) + '.')
if len(filelist) == 0:
print('Found no DEMs intersecting extent to stack. Skipping...')
sys.exit()
datelist = np.array([parse_date(f) for f in filelist])
sorted_inds = np.argsort(datelist)
print(filelist[sorted_inds[0]])
if l1a_zipped and os.path.basename(filelist[sorted_inds[0]])[0:3] == 'AST':
tmp_zip = filelist[sorted_inds[0]]
z_name = '_'.join(
os.path.basename(tmp_zip).split('_')[0:3]) + '_Z_adj_XAJ_final.tif'
if tmptag is None:
fn_tmp = os.path.join(os.path.dirname(tmp_zip), 'tmp_out.tif')
else:
fn_tmp = os.path.join(os.path.dirname(tmp_zip),
'tmp_out_' + tmptag + '.tif')
mt.extract_file_from_zip(tmp_zip, z_name, fn_tmp)
tmp_img = GeoImg(fn_tmp)
else:
tmp_img = GeoImg(filelist[sorted_inds[0]])
if res is None:
res = np.round(
tmp_img.dx) # make sure that we have a nice resolution for gdal
if epsg is None:
epsg = tmp_img.epsg
# now, reproject the first image to the extent, resolution, and coordinate system needed.
dest = gdal.Warp('',
tmp_img.gd,
format='MEM',
dstSRS='EPSG:{}'.format(epsg),
xRes=res,
yRes=res,
outputBounds=(xmin, ymin, xmax, ymax),
resampleAlg=gdal.GRA_Bilinear)
if l1a_zipped and os.path.basename(filelist[sorted_inds[0]])[0:3] == 'AST':
os.remove(fn_tmp)
first_img = GeoImg(dest)
first_img.filename = filelist[sorted_inds[0]]
# NetCDF assumes that coordinates are the cell center
if first_img.is_area():
first_img.to_point()
# first_img.info()
nco, to, xo, yo = create_nc(first_img.img,
outfile=outfile,
clobber=clobber,
t0=np.datetime64('{}-01-01'.format(y0)))
create_crs_variable(first_img.epsg, nco)
# crso.GeoTransform = ' '.join([str(i) for i in first_img.gd.GetGeoTransform()])
# maxchar = max([len(f.rsplit('.tif', 1)[0]) for f in args.filelist])
go = nco.createVariable('dem_names', str, ('time', ))
go.long_name = 'Source DEM Filename'
zo = nco.createVariable('z',
'f4', ('time', 'y', 'x'),
fill_value=-9999,
zlib=True,
chunksizes=[
500,
min(150, first_img.npix_y),
min(150, first_img.npix_x)
])
zo.units = 'meters'
zo.long_name = 'Height above WGS84 ellipsoid'
zo.grid_mapping = 'crs'
zo.coordinates = 'x y'
zo.set_auto_mask(True)
if ref_tiles is not None:
if ref_tiles.endswith('.shp'):
master_tiles = gpd.read_file(ref_tiles)
s = STRtree([f for f in master_tiles['geometry'].values])
bounds = Polygon([(xmin, ymin), (xmax, ymin), (xmax, ymax),
(xmin, ymax)])
ref_vrt = get_tiles(bounds, master_tiles, s, outdir)
elif ref_tiles.endswith('.vrt') or ref_tiles.endswith('.tif'):
ref_vrt = ref_tiles
ref = GeoImg(ref_vrt)
if filt_dem is not None:
filt_dem_img = GeoImg(filt_dem)
filt_dem = filt_dem_img.reproject(first_img)
# 3 overlapping pixels on each side of the tile in case reprojection is necessary; will be removed when merging
if latlontile_nodata is not None and epsg is not None:
mask = binary_dilation(vt.latlontile_nodatamask(
first_img, latlontile_nodata),
iterations=3)
if uncert:
uo = nco.createVariable('uncert', 'f4', ('time', ))
uo.long_name = 'RMSE of stable terrain differences.'
uo.units = 'meters'
if add_ref and ref_tiles is not None:
ro = nco.createVariable('ref_z',
'f4', ('y', 'x'),
fill_value=-9999,
chunksizes=[
min(150, first_img.npix_y),
min(150, first_img.npix_x)
])
ro.units = 'meters'
ro.long_name = 'Height above WGS84 ellipsoid'
ro.grid_mapping = 'crs'
ro.coordinates = 'x y'
ro.set_auto_mask(True)
ref_img = ref.reproject(first_img).img
if latlontile_nodata is not None and epsg is not None:
ref_img[~mask] = np.nan
ro[:, :] = ref_img
if add_corr:
co = nco.createVariable('corr',
'i1', ('time', 'y', 'x'),
fill_value=-1,
zlib=True,
chunksizes=[
500,
min(150, first_img.npix_y),
min(150, first_img.npix_x)
])
co.units = 'percent'
co.long_name = 'MMASTER correlation'
co.grid_mapping = 'crs'
co.coordinates = 'x y'
co.set_auto_mask(True)
x, y = first_img.xy(grid=False)
xo[:] = x
yo[:] = y
# trying something else to speed up writting in compressed chunks
list_img, list_corr, list_uncert, list_dt, list_name = ([]
for i in range(5))
outind = 0
for ind in sorted_inds[0:]:
print(filelist[ind])
# get instrument
bname = os.path.splitext(os.path.basename(filelist[ind]))[0]
splitname = bname.split('_')
instru = splitname[0]
# special case for MMASTER outputs (for disk usage)
if instru == 'AST':
fn_z = '_'.join(splitname[0:3]) + '_Z_adj_XAJ_final.tif'
fn_corr = '_'.join(splitname[0:3]) + '_CORR_adj_final.tif'
# to avoid running into issues in parallel
if tmptag is None:
fn_z_tmp = os.path.join(os.path.dirname(filelist[ind]), fn_z)
fn_corr_tmp = os.path.join(os.path.dirname(filelist[ind]),
fn_corr)
else:
fn_z_tmp = os.path.join(
os.path.dirname(filelist[ind]),
os.path.splitext(fn_z)[0] + '_' + tmptag + '.tif')
fn_corr_tmp = os.path.join(
os.path.dirname(filelist[ind]),
os.path.splitext(fn_corr)[0] + '_' + tmptag + '.tif')
list_fn_rm = [fn_z_tmp, fn_corr_tmp]
# unzip if needed
if l1a_zipped:
mt.extract_file_from_zip(filelist[ind], fn_z, fn_z_tmp)
if filt_mm_corr or add_corr:
mt.extract_file_from_zip(filelist[ind], fn_corr,
fn_corr_tmp)
# open dem, filter with correlation mask if it comes out of MMASTER
if filt_mm_corr:
img = corr_filter_aster(fn_z_tmp, fn_corr_tmp, 70)
else:
img = GeoImg(fn_z_tmp)
else:
img = GeoImg(filelist[ind])
if img.is_area(): # netCDF assumes coordinates are the cell center
img.to_point()
if add_corr:
if instru == 'AST':
corr = GeoImg(fn_corr_tmp)
if corr.is_area():
corr.to_point()
if coreg:
try:
NDV = img.NDV
coreg_outdir = os.path.join(
outdir,
os.path.basename(filelist[ind]).rsplit('.tif', 1)[0])
_, img, _, stats_final = dem_coregistration(
ref,
img,
glaciermask=exc_mask,
landmask=inc_mask,
outdir=coreg_outdir,
inmem=True)
dest = gdal.Warp('',
img.gd,
format='MEM',
dstSRS='EPSG:{}'.format(epsg),
xRes=res,
yRes=res,
outputBounds=(xmin, ymin, xmax, ymax),
resampleAlg=gdal.GRA_Bilinear,
srcNodata=NDV,
dstNodata=-9999)
img = GeoImg(dest)
if add_corr:
if instru == 'AST':
corr = corr.reproject(img)
else:
corr = img.copy()
corr.img[:] = 100
co[outind, :, :] = corr.img.astype(np.int8)
if filt_dem is not None:
valid = np.logical_and(img.img - filt_dem.img > -400,
img.img - filt_dem.img < 1000)
img.img[~valid] = np.nan
if latlontile_nodata is not None and epsg is not None:
img.img[~mask] = np.nan
if add_corr:
corr.img[~mask] = -1
nvalid = np.count_nonzero(~np.isnan(img.img))
if nvalid == 0:
print('No valid pixel in the stack extent: skipping...')
if l1a_zipped and (instru == 'AST'):
for fn_rm in list_fn_rm:
if os.path.exists(fn_rm):
os.remove(fn_rm)
continue
zo[outind, :, :] = img.img
if uncert:
uo[outind] = stats_final[3]
print('Adding DEM that has ' + str(nvalid) +
' valid pixels in this extent, with a global RMSE of ' +
str(stats_final[3]))
except:
print('Coregistration failed: skipping...')
if l1a_zipped and (instru == 'AST'):
for fn_rm in list_fn_rm:
if os.path.exists(fn_rm):
os.remove(fn_rm)
continue
else:
img = img.reproject(first_img)
if add_corr:
if instru == 'AST':
corr = corr.reproject(first_img)
else:
corr = img.copy()
corr.img[:] = 100
# co[outind, :, :] = corr.img.astype(np.int8)
if filt_dem is not None:
valid = np.logical_and(img.img - filt_dem.img > -400,
img.img - filt_dem.img < 1000)
img.img[~valid] = np.nan
if latlontile_nodata is not None and epsg is not None:
img.img[~mask] = np.nan
if add_corr:
corr.img[~mask] = -1
nvalid = np.count_nonzero(~np.isnan(img.img))
if nvalid == 0:
print('No valid pixel in the stack extent: skipping...')
if l1a_zipped and (instru == 'AST'):
for fn_rm in list_fn_rm:
if os.path.exists(fn_rm):
os.remove(fn_rm)
continue
# zo[outind, :, :] = img.img
if uncert:
try:
stats = read_stats(os.path.dirname(filelist[ind]))
except:
stats = None
# uo[outind] = stats['RMSE']
# to[outind] = datelist[ind].toordinal() - dt.date(y0, 1, 1).toordinal()
# go[outind] = os.path.basename(filelist[ind]).rsplit('.tif', 1)[0]
if stats is None:
list_uncert.append(5.)
else:
try:
list_uncert.append(stats['RMSE'])
except KeyError:
print('KeyError for RMSE here:' + filelist[ind])
continue
list_img.append(img.img)
list_corr.append(corr.img.astype(np.int8))
list_dt.append(datelist[ind].toordinal() -
dt.date(y0, 1, 1).toordinal())
list_name.append(os.path.basename(filelist[ind]).rsplit('.tif', 1)[0])
outind += 1
if l1a_zipped and (instru == 'AST'):
for fn_rm in list_fn_rm:
if os.path.exists(fn_rm):
os.remove(fn_rm)
# then write all at once
zo[0:outind, :, :] = np.stack(list_img, axis=0)
co[0:outind, :, :] = np.stack(list_corr, axis=0)
uo[0:outind] = np.array(list_uncert)
to[0:outind] = np.array(list_dt)
go[0:outind] = np.array(list_name)
return nco
#and same for icebridge
#then
fn_count_icesat = '/home/atom/ongoing/work_worldwide/figures/fig3/icesat_count_800m.tif'
fn_sum_icesat = '/home/atom/ongoing/work_worldwide/figures/fig3/icesat_sum_800m.tif'
count_ics = GeoImg(fn_count_icesat)
sum_ics = GeoImg(fn_sum_icesat)
nodata = count_ics.img == 0.
mean_ics = np.zeros(np.shape(sum_ics.img)) * np.nan
mean_ics[~nodata] = sum_ics.img[~nodata] / count_ics.img[~nodata]
out = count_ics.copy()
out.img = np.abs(mean_ics)
out.write('/home/atom/ongoing/work_worldwide/figures/fig3/zsc_icesat_800m.tif')
fn_count_ib = '/home/atom/ongoing/work_worldwide/figures/fig3/ib_count_300m.tif'
fn_sum_ib = '/home/atom/ongoing/work_worldwide/figures/fig3/ib_sum_300m.tif'
count_ics = GeoImg(fn_count_ib)
sum_ics = GeoImg(fn_sum_ib)
nodata = count_ics.img == 0.
mean_ics = np.zeros(np.shape(sum_ics.img)) * np.nan
mean_ics[~nodata] = sum_ics.img[~nodata] / count_ics.img[~nodata]
out = count_ics.copy()
bedrx_poly_mask = odir+'bedrx_poly.shp'
# read in DEMs and project to same extents
dem1 = GeoImg(args.dem_1)
dem2 = GeoImg(args.dem_2)
x_res = dem1.dx # had to add this for calculating dv at the end, mask seems to distort
y_res = dem1.dy
#coregister
if coregFlag:
coreg.dem_coregistration(dem1,dem2,glaciermask=None,landmask=bedrx_poly_mask,outdir=odir,pts=False)
dem2 = GeoImg(args.dem_2[0:-4] + '_adj.tif')
# dh_dem
dem2_reproj = dem2.reproject(dem1) # reproject master DEM to slave DEM extent, cell size
dh_dem = dem2.copy(new_raster=dem2_reproj.img - dem1.img)
dh_bedrx = dem2.copy(new_raster=dem2_reproj.img - dem1.img)
# remove outliers
dem_rem = np.absolute(np.nanmean(dh_dem.img))+(np.nanstd(dh_dem.img)*5)
np.seterr(all='ignore')
dh_dem.img[dh_dem.img > dem_rem] = np.nan
dh_dem.img[dh_dem.img < (dem_rem*-1)] = np.nan
dh_bedrx.img[dh_bedrx.img > dem_rem] = np.nan
dh_bedrx.img[dh_bedrx.img < (dem_rem*-1)] = np.nan
# test coregistered bedrx error
bedrx_ma = imtool.create_mask_from_shapefile(dem1,bedrx_poly_mask,buffer=None)
bedrx_mask = np.invert(bedrx_ma) # inverting for mask
dh_bedrx.img = np.ma.masked_where(bedrx_mask, dh_bedrx.img) # masking
fig0 = dh_bedrx.display(fig=plt.figure(figsize=(8,8)), cmap='bwr_r', vmin=-10, vmax=10)