def SRTMGL1_V003_tile(dir_SRTMGL1,
tile_name,
dem_out,
filter_params=None,
format_out='SRTMHGT',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-32768,
interp_method=None,
geoid=True):
"""
:param dir_SRTMGL1: path to directory directly containing zip of SRTMGL1 and SRTMGL1N
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with SRTMGL1 NUM file using rastlib.filter_nanarray function
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return:
SRTMGL1: Shuttle Radar Topography Mission Global 1-arcsec V003 from NASA at: https://lpdaac.usgs.gov/sites/default/files/public/measures/docs/NASA_SRTM_V3.pdf
NUM file description
1 = Water-masked SRTM void *
2 = Water-masked SRTM non-void *
5 = GDEM elevation = 0 in SRTM void (helped correct ocean masking)
11 = NGA-interpolated SRTM (were very small voids in SRTMv1) 21 = GMTED2010 oversampled from 7.5 arc-second postings
25 = SRTM within GDEM **
31 = NGA fill of SRTM via GDEM***
51 = USGS NED
52 = USGS NED via GDEM
53 = Alaska USGS NED via GDEM
72 = Canadian Digital Elevation Data (CDED) via GDEM 101-200 = ASTER scene count (count limited to 100)
201-224 = SRTM swath count (non-voided swaths) Actual maximum = 24
OPTIMAL DIRECTORY ARCHITECTURE: SRTMGL1 and SRTMGL1N zip archives in the same directory
"""
#1/ LOCATE 1x1° tile
#nomenclature of SRTMGL1 and SRTMGL1N zip files
hgt_zip = os.path.join(dir_SRTMGL1, tile_name + '.SRTMGL1.hgt.zip')
num_zip = os.path.join(
dir_SRTMGL1, tile_name + '.SRTMGL1N.num.zip'
) #NOTE: nomenclature of V002 was SRTMGL1.num.zip for the num file
#2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
zip_ref = zipfile.ZipFile(hgt_zip, 'r')
zip_ref.extractall(tmp_dir)
zip_ref.close()
zip_ref = zipfile.ZipFile(num_zip, 'r')
zip_ref.extractall(tmp_dir)
zip_ref.close()
tmp_hgt = os.path.join(tmp_dir, tile_name + '.hgt')
tmp_num = os.path.join(tmp_dir, tile_name + '.num')
#3/ FILTER TILE
if filter_params is not None:
num = read_nanarray(tmp_num)
_, filt = filter_nanarray(num, filter_params[0], filter_params[1],
filter_params[2])
hgt = read_nanarray(tmp_hgt)
hgt_filtered = np.array(hgt)
hgt_filtered[filt] = np.NaN
update_nanarray(tmp_hgt, hgt_filtered)
#4/ REPROJECT TILE
# raw data is SRTMHGT, 4326, 1 arc-sec and -32768 nodata_out
if format_out == 'SRTMHGT' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32768:
tmp_hgt_proj = tmp_hgt
else:
tmp_hgt_proj = os.path.join(tmp_dir, tile_name + '_proj.tif')
warp_defaultUTM(tmp_hgt, tmp_hgt_proj, format_out, 4326, tgt_EPSG,
tgt_res, nodata_out, interp_method)
#5/ ELLIPSOID OR GEOID
# raw data is geoid EGM96
if not geoid:
geoid_to_ellipsoid(tmp_hgt_proj, dem_out)
else:
shutil.move(tmp_hgt_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def ASTGTM2_V002_tile(dir_ASTGTM2,
tile_name,
dem_out,
filter_params=None,
format_out='GTiff',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=True):
"""
:param dir_ASTGTM2: path to directory directly containing zip of ASTGTM2
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with ASTGTM2 NUM file using rastlib.filter_nanarray function
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return:
ASTGTM2: ASTER Global Digital Elevation Model V002 from NASA at: https://lpdaac.usgs.gov/node/1079
NUM file description
Fill value: -1 to -11
-1 SRTM3 V003
-2 SRTM3 V002
-5 NED
-6 CDED
-11 Alaska DEM
Valid range (ASTER count):
0 to 200
OPTIMAL DIRECTORY ARCHITECTURE: ASTGTM2 zip archives in the same directory
"""
#1/ LOCATE 1x1° tile
#nomenclature of ASTGTM2 zip files
tile_zip = os.path.join(dir_ASTGTM2, 'ASTGTM2_' + tile_name + '.zip')
dem_file = 'ASTGTM2_' + tile_name + '_dem.tif'
num_file = 'ASTGTM2_' + tile_name + '_num.tif'
#2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
tmp_dem = os.path.join(tmp_dir, dem_file)
tmp_num = os.path.join(tmp_dir, num_file)
extract_file_from_zip(tile_zip, dem_file, tmp_dem)
extract_file_from_zip(tile_zip, num_file, tmp_num)
#3/ FILTER TILE
if filter_params is not None:
num = read_nanarray(tmp_num)
_, filt = filter_nanarray(num, filter_params[0], filter_params[1],
filter_params[2])
dem = read_nanarray(tmp_dem)
dem_filtered = np.array(dem)
dem_filtered[filt] = np.NaN
update_nanarray(tmp_dem, dem_filtered)
#4/ REPROJECT TILE
# raw data is GTiff, 4326, 1 arc-sec and -32768 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -9999:
tmp_dem_proj = tmp_dem
else:
tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif')
warp_defaultUTM(tmp_dem, tmp_dem_proj, format_out, 4326, tgt_EPSG,
tgt_res, nodata_out, interp_method)
#5/ ELLIPSOID OR GEOID
# raw data is geoid EGM96
if not geoid:
geoid_to_ellipsoid(tmp_dem_proj, dem_out)
else:
shutil.move(tmp_dem_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def REMA_mosaic_r1_1_tile(dir_REMA,
tile_name,
dem_out,
filter_params=None,
format_out='GTiff',
tgt_EPSG=3031,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=False,
tag_lonlat_tile=False,
path_tile_index=None,
tag_merge=False,
tag_clip=False):
"""
:param dir_REMA: path to parent directory "8m" containing subdirectories of tar.gz archives (native FTP architecture)
:param tile_name: either REMA tile name or 1x1° lat/lon tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with REMA ERR file using rastlib.filter_nanarray function
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:param tag_lonlat_tile: True if tile_name follows SRTMGL1 tile naming, False if tile_name follows REMA tile naming
:param path_tile_index: if tile_name is REMA format, specify path to ESRI REMA Tile Index
:param tag_merge: if tile_name is REMA format, True to merge all ArcticDEM tiles to the 1x1° lat/lon extent
:param tag_clip: if tile_name is REMA format, True to clip the 5x5° tile to the 1x1° lat/lon extent of tile_name
:return:
REMA release 1.1 product: ref:https://www.pgc.umn.edu/data/rema/
Processing for 8m mosaic
(100m, 500m and 1km versions are bundled in one .tif file)
Tile name and processing is REMA tile naming convention by default
Provide path to ESRI tile index file to use 1x1° lat/lon tiles and SRTMGL1 naming convention
OPTIMAL DIRECTORY ARCHITECTURE: point to "8m" folder of similar architecture than: ftp://ftp.data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.0
"""
# 1/ LOCATE TILE
if not tag_lonlat_tile:
subtile_dir = os.path.join(dir_REMA, tile_name)
tile_tar_gz_list = [
os.path.join(subtile_dir, tar_file)
for tar_file in os.listdir(subtile_dir)
if tar_file.endswith('.tar.gz')
]
else:
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
extent = [lat_tile, lon_tile, lat_tile + 1, lon_tile + 1]
# feature name in REMA_Tile_Index_Rel1.1
feat_name = 'TILE'
subtile_name_list = list_shp_field_inters_extent(
path_tile_index, feat_name, extent, 4326)
subtile_dir_list = [
os.path.join(dir_REMA, tile) for tile in subtile_name_list
]
tile_tar_gz_list = []
for i in range(len(subtile_dir_list)):
tile_tar_gz_list = tile_tar_gz_list + [
os.path.join(subtile_dir_list[i], tar_file)
for tar_file in os.listdir(subtile_dir_list[i])
if tar_file.endswith('.tar.gz')
]
# 2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
list_tmp_dem = [
os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_dem.tif')
for tile_tar_gz in tile_tar_gz_list
]
for tile_tar_gz in tile_tar_gz_list:
extract_file_from_tar_gz(
tile_tar_gz,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_dem.tif',
list_tmp_dem[tile_tar_gz_list.index(tile_tar_gz)])
# list_tmp_err = [tmp_dir + os.path.splitext(os.path.basename(tile_tar_gz))[0]+'_err.tif' for tile_tar_gz in tile_tar_gz_list]
for tile_tar_gz in tile_tar_gz_list:
extract_file_from_tar_gz(
tile_tar_gz,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_err.tif',
list_tmp_dem[tile_tar_gz_list.index(tile_tar_gz)])
list_tmp_dem_tomerge = []
for tmp_dem in list_tmp_dem:
# 3/ FILTER TILE
if filter_params is not None:
tmp_err = tmp_dem[:-8] + '_err.tif'
err = read_nanarray(tmp_err)
_, filt = filter_nanarray(err, filter_params[0], filter_params[1],
filter_params[2])
dem = read_nanarray(tmp_dem)
dem_filtered = np.array(dem)
dem_filtered[filt] = np.NaN
update_nanarray(tmp_dem, dem_filtered)
# 4/ REPROJECT TILE
# raw data is GeoTiff, 3031, 1 arc-sec and -9999 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 3031 and tgt_res is None and nodata_out is -9999:
tmp_dem_proj = tmp_dem
else:
tmp_dem_proj = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tmp_dem))[0] + '_proj.tif')
warp_defaultUTM(tmp_dem, tmp_dem_proj, format_out, 3031, tgt_EPSG,
tgt_res, nodata_out, interp_method)
# 5/ ELLIPSOID OR GEOID
# raw data is ellipsoid WGS84
if geoid:
tmp_dem_geoid = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tmp_dem))[0] + '_geoid.tif')
ellipsoid_to_geoid(tmp_dem_proj, tmp_dem_geoid)
else:
tmp_dem_geoid = tmp_dem_proj
list_tmp_dem_tomerge.append(tmp_dem_geoid)
# 6/ MERGE ALL TILES
tmp_dem_merged = os.path.join(tmp_dir, tile_name + '_merged.tif')
if tag_merge:
merge_rast_list(list_tmp_dem_tomerge, tmp_dem_merged)
else:
shutil.copytree(
tmp_dir,
os.path.join(os.path.dirname(dem_out), tile_name + '_subtiles'))
# 7/ CLIP TO TILE EXTENT
if not tag_clip:
tmp_dem_clipped = os.path.join(tmp_dir, tile_name + '_clipped.tif')
lat, lon = SRTMGL1_naming_to_latlon(tile_name)
clip_rast_to_extent(tmp_dem_merged, tmp_dem_clipped,
[lat, lon, lat + 1, lon + 1], 4326)
else:
tmp_dem_clipped = tmp_dem_merged
shutil.move(tmp_dem_clipped, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def AW3D30_v2_1_tile(dir_ALOS,
tile_name,
dem_out,
bit_params=None,
format_out='GTiff',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=True):
"""
:param dir_ALOS: path to directory directly containing ALOS World 3D 30-m 5x5° tar.gz archives
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param bit_params: bit filtering for AW3D30 MSK file ; see rastlib.mask_bitarray function for an example
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return
ALOS World 3D-30m (AW3D30) Version 2.1 from JAXA at: https://www.eorc.jaxa.jp/ALOS/en/aw3d30/aw3d30v21_format_e.pdf
MSK file:
0000: Valid
0001: Cloud and show mask (invalid)
0010: Land water and low correlation mask of the 5 m resolution DSM (valid)
0011: Sea mask (valid)
0100: National Land Numerical Information 10 m DEM (by Geographical Survey Institute of Japan) (valid)
1000: Shuttle Radar Topography Mission (SRTM) SRTM-1 Version 3 （valid）
1100: PRISM DSM (valid)
STK, HDR, QAI, LST files not used in this function, could be extracted same manner as other in 2/
OPTIMAL DIRECTORY ARCHITECTURE: all 5x5 .tar.gz archives in the same directory
"""
#1/ LOCATE 1x1° TILE
#ALOS AW3D30 v2.1 5x5 archive naming convention
def latlon_to_ALOS_5x5tile_naming(lat, lon):
lat_5 = np.floor(lat / 5) * 5
lon_5 = np.floor(lon / 5) * 5
if lat_5 >= 0:
str_lat = 'N'
else:
str_lat = 'S'
if lon_5 >= 0:
str_lon = 'E'
else:
str_lon = 'W'
tile_name_5 = str_lat + str(lat_5).zfill(3) + str_lon + str(
lon_5).zfill(3)
return tile_name_5
#identify in which 5x5 tar.gz the 1x1 tile is located
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
name_5x5 = latlon_to_ALOS_5x5tile_naming(lat_tile, lon_tile)
list_5x5 = os.listdir(dir_ALOS)
list_5x5_prefix = [l[0:8] for l in list_5x5]
if name_5x5 in list_5x5_prefix:
tar_5x5 = list_5x5[list_5x5_prefix.index(name_5x5)]
else:
sys.exit(
'Could not find an ALOS AW3D30 5x5° archive containing the tile ' +
tile_name + ' in the directory ' + dir_ALOS)
#2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
alos_tile_name = tile_name[0:1] + str(0) + tile_name[
1:] #SRTMGL1 naming convention to ALOS 1x1 naming convention (3 digits for latitude)
#extract files of interest
dsm_file = alos_tile_name + '_AVE_DSM.tif'
msk_file = alos_tile_name + '_AVE_MSK.tif'
tmp_dsm = os.path.join(tmp_dir, dsm_file)
tmp_msk = os.path.join(tmp_dir, msk_file)
extract_file_from_tar_gz(os.path.join(dir_ALOS, tar_5x5), dsm_file,
tmp_dsm)
extract_file_from_tar_gz(os.path.join(dir_ALOS, tar_5x5), msk_file,
tmp_msk)
#3/ FILTER TILE
mask = read_nanarray(tmp_msk)
#no filtering by default
if bit_params is not None:
mask_goodval = mask_bitarray(mask, bit_params[0], bit_params[1])
dsm = read_nanarray(tmp_dsm)
dsm_filtered = np.array(dsm)
dsm_filtered[mask_goodval] = np.NaN
update_nanarray(tmp_dsm, dsm_filtered)
#4/ REPROJECT TILE:
#raw data is GeoTiff, 4326, 1 arc-sec and -9999 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -9999:
tmp_dsm_proj = tmp_dsm
else:
tmp_dsm_proj = os.path.join(tmp_dir, alos_tile_name + '_proj.tif')
warp_defaultUTM(tmp_dsm, tmp_dsm_proj, format_out, 4326, tgt_EPSG,
tgt_res, nodata_out, interp_method)
#5/ ELLIPSOID OR GEOID
#raw data is geoid EGM96
if not geoid:
geoid_to_ellipsoid(tmp_dsm_proj, dem_out)
else:
shutil.move(tmp_dsm_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def TDX_90m_tile(dir_TDX,
tile_name,
dem_out,
filter_params=None,
format_out='GTiff',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-32767,
interp_method=None,
geoid=False):
"""
:param dir_TDX: path to directory directly containing TanDEM-X zip archives
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with TDX HEM file ; see rastlib.filter_nanarray function for an example
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return:
TanDEM-X 90m product from DLR at: https://geoservice.dlr.de/web/dataguide/tdm90/pdfs/TD-GS-PS-0021_DEM-Product-Specification.pdf
Only HEM (Height Error Map) is extracted here
Using the same approach with binary/nanarray filters, 2/ can extract and 3/ can process the AMP, AM2, WAM, COV, COM and LSM files
OPTIMAL DIRECTORY ARCHITECTURE: all .zip archives in the same directory
"""
# 1/ LOCATE 1x1° TILE
def latlon_to_TDX_tile_naming(lat, lon):
if lat >= 80:
lon_sw = np.floor(lon / 4) * 4
elif lat >= 60:
lon_sw = np.floor(lon / 2) * 2
else:
lon_sw = lon
lat_sw = lat
if lat_sw >= 0:
str_lat = 'N'
else:
str_lat = 'S'
if lon_sw >= 0:
str_lon = 'E'
else:
str_lon = 'W'
tile_name_tdx = str_lat + str(int(
abs(lat_sw))).zfill(2) + str_lon + str(int(abs(lon_sw))).zfill(3)
return tile_name_tdx
#identify in which TDX tile the 1x1 tile is located
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
name_tdx = latlon_to_TDX_tile_naming(lat_tile, lon_tile)
list_tdx = os.listdir(dir_TDX)
list_prefix = [l[13:20] for l in list_tdx]
if name_tdx in list_prefix:
pass
# zip_tdx=list_prefix[list_prefix.index(name_tdx)]
else:
sys.exit('Could not find a TDX tile archive containing the tile ' +
name_tdx + ' in the directory ' + dir_TDX)
tile_name = name_tdx
# nomenclature of TDX 3 arc-sec files
tile_zip = os.path.join(dir_TDX, 'TDM1_DEM__30_' + name_tdx + '.zip')
dem_out = dem_out + '_' + name_tdx + '.tif'
# 2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
dem_file = 'TDM1_DEM__30_' + tile_name + '_DEM.tif'
hem_file = 'TDM1_DEM__30_' + tile_name + '_HEM.tif'
tmp_dem = os.path.join(tmp_dir, dem_file)
tmp_hem = os.path.join(tmp_dir, hem_file)
extract_file_from_zip(tile_zip, dem_file, tmp_dem)
extract_file_from_zip(tile_zip, hem_file, tmp_hem)
# 3/ FILTER TILE
if filter_params is not None:
hem = read_nanarray(tmp_hem)
_, filt = filter_nanarray(hem, filter_params[0], filter_params[1],
filter_params[2])
dem = read_nanarray(tmp_dem)
dem_filtered = np.array(dem)
dem_filtered[filt] = np.NaN
update_nanarray(tmp_dem, dem_filtered)
# 4/ REPROJECT TILE
# raw data is GTiff, 4326, 3 arc-sec and -32768 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32767:
tmp_dem_proj = tmp_dem
else:
tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif')
warp_defaultUTM(tmp_dem,
tmp_dem_proj,
format_out=format_out,
src_EPSG=4326,
tgt_EPSG=tgt_EPSG,
tgt_res=tgt_res,
nodata_in=-32767,
nodata_out=nodata_out,
interp_method=interp_method)
# 5/ ELLIPSOID OR GEOID
# raw data is ellipsoid WGS84
if geoid:
ellipsoid_to_geoid(tmp_dem_proj, dem_out)
else:
shutil.move(tmp_dem_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
if not idx_nonvoid[i]:
ddem_out[idx_orig] = final_mean[i]
return df, ddem_out
if __name__ == '__main__':
fn_ddem = '/home/atom/ongoing/std_err/data_vhr/etienne_mb/dh_MB_FULL_10m.tif'
fn_dem = '/home/atom/ongoing/std_err/data_vhr/etienne_mb/DEM_REF_MB_FULL_10m.tif'
fn_maskvoid = '/home/atom/ongoing/std_err/data_vhr/etienne_mb/mask_abla_void2.tif'
fn_mask = '/home/atom/ongoing/std_err/data_vhr/etienne_mb/mask_mdg.tif'
from rastlib import read_nanarray
ddem = read_nanarray(fn_ddem)
dem = read_nanarray(fn_dem)
maskvoid = (read_nanarray(fn_maskvoid) == 1)
mask = (read_nanarray(fn_mask) == 1)
ddem[np.absolute(ddem) > 50] = np.nan
ddem[~maskvoid] = np.nan
gsd = 10.
neff_geo = 33
neff_num = 20000
std_stable = 1
acc_dh = 1 / 300.
bin_type = 'fixed'
bin_val = 50.
filt_bin = '3NMAD'
def patches_method(fn_ddem,
fn_mask_stable,
perc_min_valid,
averaging_area,
method='circular',
nmax=1000.):
def create_circular_mask(h, w, center=None, radius=None):
if center is None: # use the middle of the image
center = [int(w / 2), int(h / 2)]
if radius is None: # use the smallest distance between the center and image walls
radius = min(center[0], center[1], w - center[0], h - center[1])
Y, X = np.ogrid[:h, :w]
dist_from_center = np.sqrt((X - center[0])**2 + (Y - center[1])**2)
mask = dist_from_center <= radius
return mask
ddem = read_nanarray(fn_ddem)
res = pixel_size(fn_ddem)
mask_stable = (read_nanarray(fn_mask_stable) > 0)
valid_mask = np.logical_and(np.isfinite(ddem), mask_stable)
ddem[~valid_mask] = np.nan
nx, ny = np.shape(ddem)
count = len(ddem[~np.isnan(ddem)])
print('Number of valid pixels: ' + str(count))
nb_cadrant = int(
np.floor(np.sqrt((count * res * res) / averaging_area) + 1))
#rectangular
nx_sub = int(np.floor((nx - 1) / nb_cadrant))
ny_sub = int(np.floor((ny - 1) / nb_cadrant))
#radius size for a circular patch
rad = int(np.floor(np.sqrt(averaging_area / res * res * np.pi)))
tile = []
mean_patch = []
med_patch = []
std_patch = []
nb_patch = []
list_cadrant = [[i, j] for i in range(nb_cadrant)
for j in range(nb_cadrant)]
u = 0
while len(list_cadrant) > 0 and u < nmax:
check = 0
while check == 0:
rand_cadrant = random.randint(0, len(list_cadrant) - 1)
i = list_cadrant[rand_cadrant][0]
j = list_cadrant[rand_cadrant][1]
check_x = int(np.floor(nx_sub * (i + 1 / 2)))
check_y = int(np.floor(ny_sub * (j + 1 / 2)))
if mask_stable[check_x, check_y]:
check = 1
list_cadrant.remove(list_cadrant[rand_cadrant])
tile.append(int(str(i) + str(j)))
if method == 'rectangular':
patch = ddem[nx_sub * i:nx_sub * (i + 1),
ny_sub * j:ny_sub * (j + 1)].flatten()
elif method == 'circular':
center_x = np.floor(nx_sub * (i + 1 / 2))
center_y = np.floor(ny_sub * (j + 1 / 2))
mask = create_circular_mask(nx,
ny,
center=[center_x, center_y],
radius=rad)
# patch = ddem[center_x-rad:center_x+rad,center_y-rad:center_y+rad]
patch = ddem[mask]
else:
print('Patch method must be rectangular or circular.')
sys.exit()
nb_pixel_total = len(patch)
nb_pixel_valid = len(patch[np.isfinite(patch)])
if nb_pixel_valid > np.ceil(perc_min_valid / 100. * nb_pixel_total):
print('Here : ' + str(u))
u = u + 1
mean_patch.append(np.nanmean(patch))
med_patch.append(np.nanmedian(patch))
std_patch.append(np.nanstd(patch))
nb_patch.append(nb_pixel_valid)
tile = np.array(tile)
mean_patch = np.array(mean_patch)
med_patch = np.array(med_patch)
std_patch = np.array(std_patch)
nb_patch = np.array(nb_patch)
return tile, mean_patch, med_patch, std_patch, nb_patch
def kernel_tvol_outline_adjust(fn_ddem,
fn_dem,
fn_outline_in,
fn_outline_out,
fn_mask_stable,
fn_mask_gla,
tolerance_factor=5.,
kernel_size=9,
only_thinning=True):
def gkern(kernlen=21):
#source: https://stackoverflow.com/questions/29731726/how-to-calculate-a-gaussian-kernel-matrix-efficiently-in-numpy
"""Returns a 2D Gaussian kernel."""
lim = kernlen // 2 + (kernlen % 2) / 2
x = np.linspace(-lim, lim, kernlen + 1)
kern1d = np.diff(st.norm.cdf(x))
kern2d = np.outer(kern1d, kern1d)
return kern2d / kern2d.sum()
#inside parameters based on prior knowledge:
tmp_dir = create_tmp_dir_for_outfile(fn_outline_out)
#read
ddem = read_nanarray(fn_ddem)
ddem[np.absolute(ddem) > 60] = np.nan
dem = read_nanarray(fn_dem)
mask_other_gla = (read_nanarray(fn_mask_stable) == 1)
mask_gla = (read_nanarray(fn_mask_gla) == 1)
mask_stable = np.invert(np.logical_or(mask_gla, mask_other_gla))
res = pixel_size(fn_dem)
elev, med, std, nmad = ddem_discrete_hypso(ddem,
dem,
mask_gla,
gsd=res,
bin_val=10)[1:5]
#first, keep stable terrain
keep_stable = np.logical_and(mask_stable, np.isfinite(ddem))
ddem_on_stable = ddem[keep_stable]
#get nmad as representative statistic of stable terrain
median_stable = np.nanmedian(ddem_on_stable)
mad_stable = np.nanmedian(
np.absolute(ddem_on_stable[~np.isnan(ddem_on_stable)] - median_stable))
nmad_stable = 1.4826 * mad_stable
# #filter heavy outliers to get a fair representation of overall stable terrain (without unmapped glaciers, landslides, etc...)
# ddem_on_stable[np.array(np.absolute(ddem_on_stable - median_stable) > 3 * nmad_stable)] = np.nan
#rasterize outline
# mask_outline = rasterize_shp(fn_outline_in,fn_ddem)
#get an idea of thinning rate and amplitude of adjustment
# dh_on_outline = ddem_poly_elev(ddem,dem,5,mask_outline)
outline_timelapse = 10
ddem_timelapse = 10
# detect elev above which thinning is not significant anymore so we don't care
critic_elev = np.nanmax(
elev[np.absolute(med) + 3 * np.absolute(nmad) > tolerance_factor *
nmad_stable])
filt_elev = (elev <= critic_elev)
# remove decreasing dh signal around the end of ablation area to assess effective dh with elevation
filt_diff_dh = (np.diff(med) > 0)
filt_diff_dh = np.append(filt_diff_dh, True)
filt = np.logical_and(filt_elev, filt_diff_dh)
new_elev = elev[filt]
new_med = med[filt]
#range of proximity value to search in as a function of elevation
critic_proximity = np.absolute(
new_med
) * outline_timelapse * ddem_timelapse #TODO: ideally, multiply that by a timelapse between outline and first DEM, make sure ddem is in m and not m.y-1, and then apply a factor with slope convolution in the proximity area
interp_critic_proximity = np.reshape(
np.interp(dem.flatten(),
new_elev,
critic_proximity,
right=0,
left=np.nanmax(critic_proximity)), np.shape(dem)
) #TODO: could extrapolate linearly with elevation to assess lower dh?
#define mask of areas for possible outline adjustment
# mask_surround_elev = (dem>min_elev-mvp_elev_value) & (dem<=critic_elev)
proximity_rast = proximity_shp(fn_outline_in, fn_ddem)
mask_proximity = proximity_rast < interp_critic_proximity
#define mask for neighboring glaciers
tmp_rast_othergla = os.path.join(tmp_dir, 'tmp_rast_othergla.tif')
tmp_prox_othergla = os.path.join(tmp_dir, 'tmp_prox_othergla.tif')
rast_othergla = np.ones(np.shape(mask_other_gla))
rast_othergla[~mask_other_gla] = 0
write_nanarray(tmp_rast_othergla, fn_dem, rast_othergla)
proximity_rast_fn(tmp_rast_othergla, tmp_prox_othergla)
prox_othergla = read_nanarray(tmp_prox_othergla)
max_prox_othergla = 4 * kernel_size * res
mask_max_prox_othergla = prox_othergla > max_prox_othergla
mask_maybe = np.logical_and(mask_proximity, mask_max_prox_othergla)
#get upper elevation/center of glacier by using intersect with original outline with buffer/polygonize excluding proximity mask
tmp_buffered = os.path.join(tmp_dir, 'tmp_buff.shp')
buffer_shp_fn(fn_outline_in, tmp_buffered, 3 * res)
buff_rast = rasterize_shp(tmp_buffered, fn_ddem)
buff_rast[mask_maybe] = 0
buff_mask = np.ones(np.shape(buff_rast))
buff_mask[~buff_rast] = -9999
tmp_buffered_rast = os.path.join(tmp_dir, 'tmp_buff_mask.tif')
tmp_upper_outline = os.path.join(tmp_dir, 'tmp_upper.shp')
write_nanarray(tmp_buffered_rast, fn_ddem, buff_mask)
#if there is an area untouched that is kept to union with adjusted polygon at the end
nb_pixel_kept = len(buff_mask[buff_mask == 1])
if nb_pixel_kept > 0:
polygonize_fn(tmp_buffered_rast, tmp_upper_outline)
tmp_inters = os.path.join(tmp_dir, 'tmp_inters.shp')
# inters_shp_fn(fn_outline_in,tmp_upper_outline,tmp_inters)
#get a common area to merge with final polygon by buffering previous outline
tmp_upper_buff = os.path.join(tmp_dir, 'tmp_buff_upper.shp')
tmp_upper_buff_inters = os.path.join(tmp_dir,
'tmp_buff_upper_inters.shp')
buffer_shp_fn(tmp_upper_outline, tmp_upper_buff, 2 * kernel_size * res)
inters_shp_fn(fn_outline_in, tmp_upper_buff, tmp_upper_buff_inters)
mask_upper_buff = rasterize_shp(tmp_upper_buff_inters, fn_dem)
#if there is no area untouched, find pole of inaccessibility
else:
tmp_poi = os.path.join(tmp_dir, 'poi.shp')
poi_polygon(fn_outline_in, tmp_poi, res)
mask_upper_buff = rasterize_shp(tmp_poi, fn_dem, all_touched=True)
ddem_maybe = ddem
ddem_maybe[~mask_maybe] = np.nan
if only_thinning:
ddem_maybe[ddem_maybe > 5 * nmad_stable] = np.nan
#use kernel to ensure volume change at a given pixel is significant
conv = convolve(ddem_maybe, gkern(kernel_size), mode='nearest')
if only_thinning:
mask_tvol = (conv < -tolerance_factor * nmad_stable)
else:
mask_tvol = (np.absolute(conv) > tolerance_factor * nmad_stable)
#use filling algorithm from common mask to get only spatially continuous adjusted area with interior of glacier
mask_common = np.logical_and(mask_upper_buff, mask_tvol)
mask_flood = floodfill_discontinuous(mask_tvol, mask_start=mask_common)
mask_flood = mask_flood.astype('float')
mask_flood[mask_flood == 0] = -9999
fn_mask_flood = os.path.join(tmp_dir, 'mask_flood.tif')
fn_shp_flood = os.path.join(tmp_dir, 'shp_flood.shp')
fn_smooth_flood = os.path.join(tmp_dir, 'shp_flood_smooth.shp')
write_nanarray(fn_mask_flood, fn_ddem, mask_flood)
#polygonize final mask
polygonize_fn(fn_mask_flood, fn_shp_flood)
#smooth polygon
simplify_shp_fn(fn_shp_flood, fn_smooth_flood, tolerance=res)
#union with inside
if nb_pixel_kept > 0 and not isempty_firstfeat(fn_smooth_flood):
union_shp_fn(fn_smooth_flood, tmp_upper_buff_inters, fn_outline_out)
elif not isempty_firstfeat(fn_smooth_flood):
copy_shp_fn(fn_smooth_flood, fn_outline_out)
else:
#TODO: actually need to change projection in this step or it stays that of the input shapefile
copy_shp_fn(fn_outline_in, fn_outline_out)
def get_geophys_var_hypso(fn_ddem, fn_dem, fn_shp, out_dir, path_to_r_geophys):
pp = PdfPages(os.path.join(out_dir, 'hypsometric_fit_results.pdf'))
ddem = read_nanarray(fn_ddem)
ddem[np.absolute(ddem) > 60] = np.nan
# ddem = ddem*12.
dem = read_nanarray(fn_dem)
mask = rasterize_shp(fn_shp, fn_dem)
gsd = pixel_size(fn_ddem)
fn_proxi = os.path.join(out_dir, 'proxi.tif')
proxi = proximity_shp(fn_shp, fn_ddem, type_prox='interior')
#first get residuals of poly fit
res, res_stdized, elev, med, std, nmad, area_tot, area_meas, prox = ddem_med_hypso(
ddem, dem, mask, gsd, pp=pp, proxi=proxi, get_elev_residual=True)
plt.close('all')
fn_mask = os.path.join(out_dir, 'mask.tif')
write_nanarray(fn_mask, fn_ddem, mask)
fn_res = os.path.join(out_dir, 'residual.tif')
fn_res_stdized = os.path.join(out_dir, 'residual_standardized.tif')
write_nanarray(fn_res, fn_ddem, res)
write_nanarray(fn_res_stdized, fn_ddem, res_stdized)
mask_geo = GeoImg(fn_mask)
res_geo = GeoImg(fn_res)
res_stdized_geo = GeoImg(fn_res_stdized)
ddem_geo = GeoImg(fn_ddem)
dem_geo = GeoImg(fn_dem)
# res_geo.img[np.invert(mask)] = np.nan
extent = extent_shp_ref(fn_shp, fn_dem)
crop_res = res_geo.crop_to_extent(
[extent[0], extent[2], extent[1], extent[3]])
crop_res_stdized = res_stdized_geo.crop_to_extent(
[extent[0], extent[2], extent[1], extent[3]])
crop_ddem = ddem_geo.crop_to_extent(
[extent[0], extent[2], extent[1], extent[3]])
crop_mask = mask_geo.crop_to_extent(
[extent[0], extent[2], extent[1], extent[3]])
crop_dem = dem_geo.crop_to_extent(
[extent[0], extent[2], extent[1], extent[3]])
fn_crop_res_stdized = os.path.join(out_dir, 'res_stdized_cropped.tif')
fn_crop_mask = os.path.join(out_dir, 'mask_cropped.tif')
fn_crop_dem = os.path.join(out_dir, 'dem_cropped.tif')
crop_res_stdized.img[crop_mask.img != 1] = np.nan
crop_res_stdized.write(fn_crop_res_stdized)
crop_mask.write(fn_crop_mask)
crop_dem.write(fn_crop_dem)
crop_res.img[crop_mask.img != 1] = np.nan
crop_res_stdized.img[crop_mask.img != 1] = np.nan
# crop_ddem.img = 12*crop_ddem.img
clim_ddem_raw = np.nanmax(np.absolute(med))
outline_gla = gpd.read_file(fn_shp)
fig, _ = plot_polygon_df(outline_gla, edgecolor='k', lw=2, alpha=0.5)
plt.title('Outline')
pp.savefig(fig, dpi=300)
fig = plot_ddem_results(crop_ddem,
clim=(-clim_ddem_raw, clim_ddem_raw),
colormap='Spectral')[0]
plt.title('Elevation change [m] (Large scale)')
pp.savefig(fig, dpi=300)
fig = plot_ddem_results(crop_ddem, clim=(-3, 3), colormap='Spectral')[0]
plt.title('Elevation change [m] (Thin scale)')
pp.savefig(fig, dpi=300)
clim_res = np.nanmean(np.absolute(nmad))
fig = plot_ddem_results(crop_res,
clim=(-clim_res, clim_res),
colormap='Spectral')[0]
plt.title(
'Hypsometric residual of elevation change [m] \n (Elevation change minus hypsometric median)'
)
pp.savefig(fig, dpi=300)
fig = plot_ddem_results(crop_res_stdized,
clim=(-1, 1),
colormap='Spectral')[0]
plt.title(
'Standardized hypsometric residual of elevation change [no unit] \n (Elevation change minus hypsometric median divided by hypsometric nmad)'
)
pp.savefig(fig, dpi=300)
pp.close()
plt.close('all')
os.remove(fn_res)
os.remove(fn_mask)
os.remove(fn_res_stdized)
#normalize elevation
max_elev = np.nanmax(elev)
min_elev = np.nanmin(elev)
elev_n = (elev - min_elev) / (max_elev - min_elev)
#normalize dh
max_dh = np.nanmax(med)
min_dh = np.nanmin(med)
accu_elev = min_elev + 80 * (max_elev - min_elev) / 100
tmp_max_dh = np.nanmean(
med[elev > accu_elev]) #use mean of accumulation instead of max
if np.abs((np.nanmax(med) - tmp_max_dh) / (max_dh - min_dh)) < 0.3:
max_dh = tmp_max_dh
med_n = (min_dh - med) / (max_dh - min_dh)
std_n = std / (max_dh - min_dh)
nmad_n = nmad / (max_dh - min_dh)
#write normalized data
elev_rs = np.arange(0, 1, 0.01)
med_rs = np.interp(elev_rs, elev_n, med_n)
std_rs = np.interp(elev_rs, elev_n, std_n)
nmad_rs = np.interp(elev_rs, elev_n, nmad_n)
area_rs = np.interp(elev_rs, elev_n, area_tot)
df = pd.DataFrame()
df = df.assign(norm_elev=elev_rs,
norm_med_dh=med_rs,
norm_std_dh=std_rs,
norm_nmad_rs=nmad_rs,
area_rs=area_rs)
df_raw = pd.DataFrame()
df_raw = df_raw.assign(elev=elev,
med_dh=med,
std_dh=std,
nmad_dh=nmad,
area_tot=area_tot,
area_meas=area_meas,
prox=prox)
df.to_csv(os.path.join(out_dir, 'df_norm_dh_elev.csv'))
df_raw.to_csv(os.path.join(out_dir, 'df_raw_dh_elev.csv'))
ddem = dem = mask = res = res_stdized = crop_mask = crop_res_stdized = crop_res = crop_ddem = crop_dem = ddem_geo = dem_geo = res_geo = res_stdized_geo = None
#get variogram with moving elevation window from R
# cmd = 'Rscript '+path_to_r_geophys+' -d '+fn_crop_dem+' -r '+fn_crop_res_stdized+' -m '+fn_crop_mask+' -v Exp -o '+out_dir
# fn_log = os.path.join(out_dir,'r_geophys.log')
# log=open(fn_log,'w')
# p=Popen(cmd,stdout=log,stderr=log,shell=True)
# p.wait()
# log.close()
os.remove(fn_crop_dem)
os.remove(fn_crop_res_stdized)
os.remove(fn_crop_mask)