def REMA_mosaic_r1_1_tile(dir_REMA,
tile_name,
dem_out,
filter_params=None,
format_out='GTiff',
tgt_EPSG=3031,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=False,
tag_lonlat_tile=False,
path_tile_index=None,
tag_merge=False,
tag_clip=False):
"""
:param dir_REMA: path to parent directory "8m" containing subdirectories of tar.gz archives (native FTP architecture)
:param tile_name: either REMA tile name or 1x1° lat/lon tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with REMA ERR file using rastlib.filter_nanarray function
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:param tag_lonlat_tile: True if tile_name follows SRTMGL1 tile naming, False if tile_name follows REMA tile naming
:param path_tile_index: if tile_name is REMA format, specify path to ESRI REMA Tile Index
:param tag_merge: if tile_name is REMA format, True to merge all ArcticDEM tiles to the 1x1° lat/lon extent
:param tag_clip: if tile_name is REMA format, True to clip the 5x5° tile to the 1x1° lat/lon extent of tile_name
:return:
REMA release 1.1 product: ref:https://www.pgc.umn.edu/data/rema/
Processing for 8m mosaic
(100m, 500m and 1km versions are bundled in one .tif file)
Tile name and processing is REMA tile naming convention by default
Provide path to ESRI tile index file to use 1x1° lat/lon tiles and SRTMGL1 naming convention
OPTIMAL DIRECTORY ARCHITECTURE: point to "8m" folder of similar architecture than: ftp://ftp.data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.0
"""
# 1/ LOCATE TILE
if not tag_lonlat_tile:
subtile_dir = os.path.join(dir_REMA, tile_name)
tile_tar_gz_list = [
os.path.join(subtile_dir, tar_file)
for tar_file in os.listdir(subtile_dir)
if tar_file.endswith('.tar.gz')
]
else:
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
extent = [lat_tile, lon_tile, lat_tile + 1, lon_tile + 1]
# feature name in REMA_Tile_Index_Rel1.1
feat_name = 'TILE'
subtile_name_list = list_shp_field_inters_extent(
path_tile_index, feat_name, extent, 4326)
subtile_dir_list = [
os.path.join(dir_REMA, tile) for tile in subtile_name_list
]
tile_tar_gz_list = []
for i in range(len(subtile_dir_list)):
tile_tar_gz_list = tile_tar_gz_list + [
os.path.join(subtile_dir_list[i], tar_file)
for tar_file in os.listdir(subtile_dir_list[i])
if tar_file.endswith('.tar.gz')
]
# 2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
list_tmp_dem = [
os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_dem.tif')
for tile_tar_gz in tile_tar_gz_list
]
for tile_tar_gz in tile_tar_gz_list:
extract_file_from_tar_gz(
tile_tar_gz,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_dem.tif',
list_tmp_dem[tile_tar_gz_list.index(tile_tar_gz)])
# list_tmp_err = [tmp_dir + os.path.splitext(os.path.basename(tile_tar_gz))[0]+'_err.tif' for tile_tar_gz in tile_tar_gz_list]
for tile_tar_gz in tile_tar_gz_list:
extract_file_from_tar_gz(
tile_tar_gz,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_err.tif',
list_tmp_dem[tile_tar_gz_list.index(tile_tar_gz)])
list_tmp_dem_tomerge = []
for tmp_dem in list_tmp_dem:
# 3/ FILTER TILE
if filter_params is not None:
tmp_err = tmp_dem[:-8] + '_err.tif'
err = read_nanarray(tmp_err)
_, filt = filter_nanarray(err, filter_params[0], filter_params[1],
filter_params[2])
dem = read_nanarray(tmp_dem)
dem_filtered = np.array(dem)
dem_filtered[filt] = np.NaN
update_nanarray(tmp_dem, dem_filtered)
# 4/ REPROJECT TILE
# raw data is GeoTiff, 3031, 1 arc-sec and -9999 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 3031 and tgt_res is None and nodata_out is -9999:
tmp_dem_proj = tmp_dem
else:
tmp_dem_proj = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tmp_dem))[0] + '_proj.tif')
warp_defaultUTM(tmp_dem, tmp_dem_proj, format_out, 3031, tgt_EPSG,
tgt_res, nodata_out, interp_method)
# 5/ ELLIPSOID OR GEOID
# raw data is ellipsoid WGS84
if geoid:
tmp_dem_geoid = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tmp_dem))[0] + '_geoid.tif')
ellipsoid_to_geoid(tmp_dem_proj, tmp_dem_geoid)
else:
tmp_dem_geoid = tmp_dem_proj
list_tmp_dem_tomerge.append(tmp_dem_geoid)
# 6/ MERGE ALL TILES
tmp_dem_merged = os.path.join(tmp_dir, tile_name + '_merged.tif')
if tag_merge:
merge_rast_list(list_tmp_dem_tomerge, tmp_dem_merged)
else:
shutil.copytree(
tmp_dir,
os.path.join(os.path.dirname(dem_out), tile_name + '_subtiles'))
# 7/ CLIP TO TILE EXTENT
if not tag_clip:
tmp_dem_clipped = os.path.join(tmp_dir, tile_name + '_clipped.tif')
lat, lon = SRTMGL1_naming_to_latlon(tile_name)
clip_rast_to_extent(tmp_dem_merged, tmp_dem_clipped,
[lat, lon, lat + 1, lon + 1], 4326)
else:
tmp_dem_clipped = tmp_dem_merged
shutil.move(tmp_dem_clipped, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def REMA_strip_r1_1(dir_REMA,
tile_name,
dem_dir_out,
mosaic_coreg_segm=True,
filter_params=None,
format_out='GTiff',
tgt_EPSG=3031,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=False,
rm_tar=False,
downsample=True):
print('--REMA release 1.1 strip processing--')
# 1/ LOCATE STRIPS
print('Searching for tile ' + tile_name + ' in folder ' + dir_REMA + '...')
subtile_dir = os.path.join(dir_REMA, tile_name)
seg_tar_gz_list = [
os.path.join(subtile_dir, tar_file)
for tar_file in os.listdir(subtile_dir) if tar_file.endswith('.tar.gz')
]
print('Found ' + str(len(seg_tar_gz_list)) + ' segments in tile folder.')
# 2/ EXTRACT ALL STRIPS
tmp_dir = create_tmp_dir_for_outfile(
os.path.join(dem_dir_out, 'all_strips'))
list_tmp_dem = [
os.path.join(
tmp_dir,
os.path.splitext(
os.path.splitext(os.path.basename(seg_tar_gz))[0])[0] +
'_dem.tif') for seg_tar_gz in seg_tar_gz_list
]
for seg_tar_gz in seg_tar_gz_list:
print('Extracting dem file of segment ' +
str(seg_tar_gz_list.index(seg_tar_gz) + 1) + ' out of ' +
str(len(seg_tar_gz_list)))
extract_file_from_tar_gz(
seg_tar_gz,
os.path.splitext(
os.path.splitext(os.path.basename(seg_tar_gz))[0])[0] +
'_dem.tif', list_tmp_dem[seg_tar_gz_list.index(seg_tar_gz)])
if rm_tar:
# once extracted, remove original data to avoid taking up too much space
shutil.rmtree(subtile_dir)
if downsample:
list_ds = []
for tmp_dem in list_tmp_dem:
tmp_dem_ds = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tmp_dem))[0] + '_ds.tif')
list_ds.append(tmp_dem_ds)
translate(tmp_dem,
tmp_dem_ds,
format_out='GTiff',
tgt_res=[8, -8],
interp_method='bilinear')
os.remove(tmp_dem)
list_tmp_dem = list_ds
# 2.5/ MOSAIC/COREGISTER SETSM SEGMENTS ITERATIVELY INTO STRIPS
if mosaic_coreg_segm:
max_coreg_rmse = 10.
nb_init_pixel_min = 1000
if not os.path.exists(os.path.join(dem_dir_out, 'coreg_results')):
os.mkdir(os.path.join(dem_dir_out, 'coreg_results'))
if not os.path.exists(
os.path.join(dem_dir_out, 'coreg_results', 'good')):
os.mkdir(os.path.join(dem_dir_out, 'coreg_results', 'good'))
if not os.path.exists(
os.path.join(dem_dir_out, 'coreg_results', 'breaks')):
os.mkdir(os.path.join(dem_dir_out, 'coreg_results', 'breaks'))
summary = os.path.join(dem_dir_out,
'mosaic_coreg_summary_' + tile_name + '.csv')
with open(summary, 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow([
'scene', 'segment', 'master', 'dx', 'dy', 'dz', 'RMSE',
'mosaic_nb'
])
# create segment lists
list_seg = list(list_tmp_dem)
list_seg_id = []
for seg in list_seg:
seg_id_split = os.path.basename(seg).split('_')
seg_id = '_'.join(seg_id_split[0:5])
seg_suf = '_'.join(seg_id_split[6:])
# seg_nb=int(seg_id_split[5][3:])
if seg_id not in list_seg_id:
list_seg_id.append(seg_id)
# list all list of scenes acquired the same day
list_list_by_seg_id = []
for seg_id in list_seg_id:
tmp_list = [seg for seg in list_seg if seg_id in seg]
tmp_list = sorted(
tmp_list,
key=lambda x: x.split('/')[-1].split('_')[5][3:].zfill(2))
list_list_by_seg_id.append(tmp_list)
# coregister in series
list_final_strip = []
print('List of list of segments:')
print(list_list_by_seg_id)
# loop through same day scenes
for list_by_seg_id in list_list_by_seg_id:
print(list_by_seg_id)
# bother to coreg only if there is more than 1 segment
if len(list_by_seg_id) > 1:
tmp_dir_coreg = create_tmp_dir_for_outfile(
os.path.join(tmp_dir, 'coreg'))
# master dem is the first segment
dem_master = list_by_seg_id[0]
k = 0
# for csv summary: first segment
dem_master_id_split = os.path.basename(dem_master).split(
'_')
dem_master_id = '_'.join(dem_master_id_split[0:5])
dem_master_seg_nb = int(dem_master_id_split[5][3:])
writer.writerow([
dem_master_id, dem_master_seg_nb, dem_master_seg_nb,
'first_seg', 'first_seg', 'first_seg', 'first_seg',
str(k + 1)
])
for i in range(len(list_by_seg_id) - 1):
dem_in = list_by_seg_id[i + 1]
print('Master segment is ' + dem_master)
print('Slave segment is ' + dem_in)
# for csv summary: all segments
dem_in_id_split = os.path.basename(dem_in).split('_')
dem_in_id = '_'.join(dem_in_id_split[0:5])
dem_in_seg_nb = int(dem_in_id_split[5][3:])
dem_master_id_split = os.path.basename(
dem_master).split('_')
dem_master_seg_nb = int(dem_master_id_split[5][3:])
k0 = k
# deriving intersection
extent = inters_raster(dem_in, dem_master)
nb_init_pixel = valid_points_coreg(dem_master, dem_in)
# try coreg only if intersection is non void
if extent is not None and nb_init_pixel > nb_init_pixel_min:
tmp_dir_seg = create_tmp_dir_for_outfile(dem_in)
# do coreg, save results
try:
out_offs, final_rmse = dem_coregistration_custom(
dem_master, dem_in,
outdir=tmp_dir_seg)[2:4]
except Exception:
out_offs = None
final_rmse = 999
if final_rmse < max_coreg_rmse:
str_out = 'good'
else:
str_out = 'breaks'
shutil.move(
os.path.join(tmp_dir_seg,
'CoRegistration_Results.pdf'),
os.path.join(
dem_dir_out, 'coreg_results', str_out,
os.path.splitext(
os.path.basename(dem_in))[0] + '_' +
os.path.splitext(
os.path.basename(dem_master))[0] +
'_coreg_results.pdf'))
shutil.move(
os.path.join(tmp_dir_seg, 'coreg_params.txt'),
os.path.join(
dem_dir_out, 'coreg_results', str_out,
os.path.splitext(
os.path.basename(dem_in))[0] + '_' +
os.path.splitext(
os.path.basename(dem_master))[0] +
'_coreg_params.txt'))
print('Final coregistration RMSE is:' +
str(final_rmse))
# if rmse is good, keep doing mosaics with next segment: mosaic is new master
if final_rmse < max_coreg_rmse:
print(
'All good. Doing mosaic and iterating to next segment...'
)
tmp_mosaic_list = [
dem_master,
os.path.join(
tmp_dir_seg,
os.path.splitext(
os.path.basename(dem_in))[0] +
'_adj.tif')
]
tmp_dem_coreg = os.path.join(
tmp_dir_coreg,
os.path.basename(list_by_seg_id[i + 1]))
dem_mosaic(tmp_mosaic_list,
tmp_dem_coreg,
erode_length=0,
hole_fill_length=0,
blending_method='mean')
dem_master = tmp_dem_coreg
# if this is the last segment, save mosaic and add to list of finalized mosaics
if i == (len(list_by_seg_id) - 2):
k = k + 1
final_strip = os.path.join(
tmp_dir,
list_seg_id[list_list_by_seg_id.index(
list_by_seg_id)] + '_mosaic' +
str(k) + '_' + seg_suf)
shutil.copyfile(dem_master, final_strip)
list_final_strip.append(final_strip)
# otherwise, break the iterative mosaic-ing
else:
# if this is the last segment
if i == (len(list_by_seg_id) - 2):
# if master is the previous segment alone
if dem_master == list_by_seg_id[i]:
print(
'RMSE is larger than limit of ' +
str(max_coreg_rmse) +
'm: saving last segments independently...'
)
list_final_strip.append(dem_in)
list_final_strip.append(dem_master)
# if master is a previous ongoing mosaic
else:
k = k + 1
final_strip = os.path.join(
tmp_dir,
list_seg_id[list_list_by_seg_id.
index(list_by_seg_id)]
+ '_mosaic' + str(k) + '_' +
seg_suf)
shutil.copyfile(
dem_master, final_strip)
print(
'RMSE is larger than limit of ' +
str(max_coreg_rmse) +
'm: saving previous mosaic and last segment...'
)
list_final_strip.append(dem_in)
list_final_strip.append(final_strip)
# if there is still segments to go
else:
# save current mosaic
k = k + 1
final_strip = os.path.join(
tmp_dir,
list_seg_id[list_list_by_seg_id.index(
list_by_seg_id)] + '_mosaic' +
str(k) + '_' + seg_suf)
shutil.copyfile(dem_master, final_strip)
print(
'RMSE is larger than limit of ' +
str(max_coreg_rmse) +
'm: saving previous mosaic, slave is new master...'
)
list_final_strip.append(final_strip)
# new master is the segment that did not mosaic
dem_master = dem_in
remove_tmp_dir_for_outfile(dem_in)
# if no intersection and last segment
elif i == (len(list_by_seg_id) - 2):
out_offs = None
final_rmse = None
if dem_master == list_by_seg_id[i]:
print(
'No intersection between master and slave: saving last segments independently...'
)
list_final_strip.append(dem_in)
list_final_strip.append(dem_master)
# if master is a previous ongoing mosaic
else:
k = k + 1
final_strip = os.path.join(
tmp_dir,
list_seg_id[list_list_by_seg_id.index(
list_by_seg_id)] + '_mosaic' + str(k) +
'_' + seg_suf)
shutil.copyfile(dem_master, final_strip)
print(
'No intersection between master and slave: saving previous mosaic and last segment...'
)
list_final_strip.append(dem_in)
list_final_strip.append(final_strip)
# if no intersection and still segments to go
else:
out_offs = None
final_rmse = None
# save current mosaic
k = k + 1
final_strip = os.path.join(
tmp_dir, list_seg_id[list_list_by_seg_id.index(
list_by_seg_id)] + '_mosaic' + str(k) +
'_' + seg_suf)
shutil.copyfile(dem_master, final_strip)
print(
'No intersection between master and slave: saving previous mosaic, slave is new master...'
)
list_final_strip.append(final_strip)
# new master is the segment that did not mosaic
dem_master = dem_in
# write data in summary
if out_offs is None:
dx = 'no_inters'
dy = 'no_inters'
dz = 'no_inters'
rmse = 'no_inters'
else:
dx = str(out_offs[0])
dy = str(out_offs[1])
dz = str(out_offs[2])
rmse = str(final_rmse)
writer.writerow([
dem_in_id, dem_in_seg_nb, dem_master_seg_nb, dx,
dy, dz, rmse,
str(k0 + 1)
])
remove_tmp_dir_for_outfile(os.path.join(tmp_dir, 'coreg'))
else:
list_final_strip.append(list_by_seg_id[0])
dem_master_id_split = os.path.basename(
list_by_seg_id[0]).split('_')
dem_master_id = '_'.join(dem_master_id_split[0:5])
dem_master_seg_nb = int(dem_master_id_split[5][3:])
writer.writerow([
dem_master_id, dem_master_seg_nb, dem_master_seg_nb,
'only_seg', 'only_seg', 'only_seg', 'only_seg',
'only_seg'
])
else:
list_final_strip = list_tmp_dem
# 3/ PROCESSING OF FINAL STRIPS
final_dem = []
for final_strip in list_final_strip:
# 3.1/ FILTER STRIPS
if filter_params is not None:
sys.exit('No filter pre-defined for this DEM product.')
# 3.2/ REPROJECT STRIPS
# raw data is GeoTiff, 3413, 1 arc-sec and -9999 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 3031 and tgt_res is None and nodata_out is -9999:
tmp_dem_proj = final_strip
else:
tmp_dem_proj = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(final_strip))[0] +
'_proj.tif')
warp_defaultUTM(final_strip, tmp_dem_proj, format_out, 3031,
tgt_EPSG, tgt_res, nodata_out, interp_method)
# 3.3/ ELLIPSOID OR GEOID STRIPS
# raw data is ellipsoid WGS84
if geoid:
tmp_dem_geoid = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(final_strip))[0] +
'_geoid.tif')
ellipsoid_to_geoid(tmp_dem_proj, tmp_dem_geoid)
else:
tmp_dem_geoid = tmp_dem_proj
final_dem.append(tmp_dem_geoid)
for dem in final_dem:
shutil.copyfile(dem, os.path.join(dem_dir_out, os.path.basename(dem)))
remove_tmp_dir_for_outfile(os.path.join(dem_dir_out, 'all_strips'))
print('Fin.')
def TDX_90m_tile(dir_TDX,
tile_name,
dem_out,
filter_params=None,
format_out='GTiff',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-32767,
interp_method=None,
geoid=False):
"""
:param dir_TDX: path to directory directly containing TanDEM-X zip archives
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with TDX HEM file ; see rastlib.filter_nanarray function for an example
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return:
TanDEM-X 90m product from DLR at: https://geoservice.dlr.de/web/dataguide/tdm90/pdfs/TD-GS-PS-0021_DEM-Product-Specification.pdf
Only HEM (Height Error Map) is extracted here
Using the same approach with binary/nanarray filters, 2/ can extract and 3/ can process the AMP, AM2, WAM, COV, COM and LSM files
OPTIMAL DIRECTORY ARCHITECTURE: all .zip archives in the same directory
"""
# 1/ LOCATE 1x1° TILE
def latlon_to_TDX_tile_naming(lat, lon):
if lat >= 80:
lon_sw = np.floor(lon / 4) * 4
elif lat >= 60:
lon_sw = np.floor(lon / 2) * 2
else:
lon_sw = lon
lat_sw = lat
if lat_sw >= 0:
str_lat = 'N'
else:
str_lat = 'S'
if lon_sw >= 0:
str_lon = 'E'
else:
str_lon = 'W'
tile_name_tdx = str_lat + str(int(
abs(lat_sw))).zfill(2) + str_lon + str(int(abs(lon_sw))).zfill(3)
return tile_name_tdx
#identify in which TDX tile the 1x1 tile is located
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
name_tdx = latlon_to_TDX_tile_naming(lat_tile, lon_tile)
list_tdx = os.listdir(dir_TDX)
list_prefix = [l[13:20] for l in list_tdx]
if name_tdx in list_prefix:
pass
# zip_tdx=list_prefix[list_prefix.index(name_tdx)]
else:
sys.exit('Could not find a TDX tile archive containing the tile ' +
name_tdx + ' in the directory ' + dir_TDX)
tile_name = name_tdx
# nomenclature of TDX 3 arc-sec files
tile_zip = os.path.join(dir_TDX, 'TDM1_DEM__30_' + name_tdx + '.zip')
dem_out = dem_out + '_' + name_tdx + '.tif'
# 2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
dem_file = 'TDM1_DEM__30_' + tile_name + '_DEM.tif'
hem_file = 'TDM1_DEM__30_' + tile_name + '_HEM.tif'
tmp_dem = os.path.join(tmp_dir, dem_file)
tmp_hem = os.path.join(tmp_dir, hem_file)
extract_file_from_zip(tile_zip, dem_file, tmp_dem)
extract_file_from_zip(tile_zip, hem_file, tmp_hem)
# 3/ FILTER TILE
if filter_params is not None:
hem = read_nanarray(tmp_hem)
_, filt = filter_nanarray(hem, filter_params[0], filter_params[1],
filter_params[2])
dem = read_nanarray(tmp_dem)
dem_filtered = np.array(dem)
dem_filtered[filt] = np.NaN
update_nanarray(tmp_dem, dem_filtered)
# 4/ REPROJECT TILE
# raw data is GTiff, 4326, 3 arc-sec and -32768 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32767:
tmp_dem_proj = tmp_dem
else:
tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif')
warp_defaultUTM(tmp_dem,
tmp_dem_proj,
format_out=format_out,
src_EPSG=4326,
tgt_EPSG=tgt_EPSG,
tgt_res=tgt_res,
nodata_in=-32767,
nodata_out=nodata_out,
interp_method=interp_method)
# 5/ ELLIPSOID OR GEOID
# raw data is ellipsoid WGS84
if geoid:
ellipsoid_to_geoid(tmp_dem_proj, dem_out)
else:
shutil.move(tmp_dem_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)