def clip_rast_to_shp_sql(raster_in, raster_out, shp_in, sql_req):
#reproject shp to similar proj if needed
rast_proj = read_proj_rast(raster_in)
shp_proj = read_proj_shp(shp_in)
comp_proj, rast_epsg, _ = compare_proj_wkt_or_EPSG(True, rast_proj, True,
shp_proj)
if not comp_proj:
tmp_dir = create_tmp_dir_for_outfile(raster_out)
tmp_shp = tmp_dir + os.path.splitext(
os.path.basename(shp_in))[0] + '_tmp.shp'
os.system('ogr2ogr -t_srs ' + rast_epsg + ' ' + tmp_shp + ' ' + shp_in)
final_shp = tmp_shp
else:
final_shp = shp_in
#example of sql request
#sql_req = "SELECT * FROM your_shapefile"
gdal.Warp(raster_out,
raster_in,
cutlineDSName=final_shp,
cutlineSQL=sql_req,
cropToCutline=True)
remove_tmp_dir_for_outfile(raster_out)
def saga_aspect_slope_curvature(dem_in,topo_out,method_nb=5):
"""
:param dem_in: dem input
:param topo_out: raster out (3 bands: aspect, slope, curvature)
:param method_nb: algorithm used, see function description
:return:
requirement: SAGA 2.X with X>3
#ref:http://www.saga-gis.org/saga_tool_doc/2.2.3/ta_morphometry_0.html
aspect, slope, curvature methods
methods number
[0] maximum slope (Travis et al. 1975)
[1] maximum triangle slope (Tarboton 1997)
[2] least squares fitted plane (Horn 1981, Costa-Cabral & Burgess 1996)
[3] 6 parameter 2nd order polynom (Evans 1979)
[4] 6 parameter 2nd order polynom (Heerdegen & Beran 1982)
[5] 6 parameter 2nd order polynom (Bauer, Rohdenburg, Bork 1985)
[6] 9 parameter 2nd order polynom (Zevenbergen & Thorne 1987)
[7] 10 parameter 3rd order polynom (Haralick 1983)
unit slope 0=rad, 1=deg, 2=percent
unit aspect 0=rad, 1=deg
"""
tmp_dir=create_tmp_dir_for_outfile(topo_out)
saga_elev = tmp_dir + 'elev_temp.sgrd'
saga_slope = tmp_dir + 'slope_temp.sgrd'
saga_aspect = tmp_dir + 'aspect_temp.sgrd'
saga_max_curv = tmp_dir + 'max_curv_temp.sgrd'
tif_slope = tmp_dir + 'slope_temp.tif'
tif_aspect = tmp_dir + 'aspect_temp.tif'
tif_max_curv = tmp_dir + 'max_curv_temp.tif'
output_vrt = tmp_dir + 'stack.vrt'
os.system('saga_cmd io_gdal 0 -GRIDS ' + saga_elev + ' -FILES ' + dem_in)
os.system('saga_cmd ta_morphometry 0 -ELEVATION ' + saga_elev + ' -SLOPE ' + saga_slope + ' -ASPECT ' + saga_aspect + ' -C_MAXI ' + saga_max_curv + ' -METHOD ' +str(method_nb)+' -UNIT_SLOPE 1 -UNIT_ASPECT 1')
os.system('saga_cmd io_gdal 2 -GRIDS ' + saga_slope + ' -FILE ' + tif_slope)
os.system('saga_cmd io_gdal 2 -GRIDS ' + saga_aspect + ' -FILE ' + tif_aspect)
os.system('saga_cmd io_gdal 2 -GRIDS ' + saga_max_curv + ' -FILE ' + tif_max_curv)
os.system('gdalbuildvrt -separate -overwrite ' + output_vrt + ' ' + tif_slope + ' ' + tif_aspect + ' ' + tif_max_curv)
os.system('gdal_translate ' + output_vrt + ' ' + topo_out)
remove_tmp_dir_for_outfile(topo_out)
def crop_raster_nodata(raster_in, raster_out):
tmp_dir = create_tmp_dir_for_outfile(raster_out)
tmp_raster = os.path.join(
tmp_dir, 'tmp' + os.path.splitext(os.path.basename(raster_in))[0])
tmp_nodataextent = os.path.join(tmp_dir, 'tmp.shp')
os.system('gdal_calc.py -A ' + raster_in + ' --outfile ' + tmp_raster +
' --NoDataValue 0 --calc="1*(A>0)"')
os.system('gdal_polygonize.py -8 -b 1 -f "ESRI Shapefile" ' + tmp_raster +
' ' + tmp_nodataextent)
clip_rast_to_shp_sql(raster_in, raster_out, tmp_nodataextent, None)
remove_tmp_dir_for_outfile(raster_out)
def merge_rast_list(list_raster,
raster_out,
tgt_EPSG=None,
nodata_in=None,
nodata_out=None):
#reproject to similar proj if specified
if tgt_EPSG is not None:
tmp_dir = create_tmp_dir_for_outfile(raster_out)
list_raster_proj = []
for rast in list_raster:
print(rast)
ind = list_raster.index(rast)
rast_proj = tmp_dir + os.path.splitext(
os.path.basename(rast))[0] + '_' + str(ind) + '_proj.tif'
warp_defaultUTM(rast,
rast_proj,
tgt_EPSG=tgt_EPSG,
nodata_out=nodata_out)
list_raster_proj.append(rast_proj)
final_list_raster = list_raster_proj
else:
final_list_raster = list_raster
if nodata_in is None:
str_nodata_in = ''
else:
str_nodata_in = ' -n ' + str(nodata_in)
if nodata_out is None:
str_nodata_out = ''
else:
str_nodata_out = ' -a_nodata ' + str(nodata_out)
if os.path.exists(raster_out):
os.remove(raster_out)
os.system('gdal_merge.py -o ' + raster_out + str_nodata_in +
str_nodata_out + ' ' + " ".join(final_list_raster))
remove_tmp_dir_for_outfile(raster_out)
def REMA_strip_r1_1(dir_REMA,
tile_name,
dem_dir_out,
mosaic_coreg_segm=True,
filter_params=None,
format_out='GTiff',
tgt_EPSG=3031,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=False,
rm_tar=False,
downsample=True):
print('--REMA release 1.1 strip processing--')
# 1/ LOCATE STRIPS
print('Searching for tile ' + tile_name + ' in folder ' + dir_REMA + '...')
subtile_dir = os.path.join(dir_REMA, tile_name)
seg_tar_gz_list = [
os.path.join(subtile_dir, tar_file)
for tar_file in os.listdir(subtile_dir) if tar_file.endswith('.tar.gz')
]
print('Found ' + str(len(seg_tar_gz_list)) + ' segments in tile folder.')
# 2/ EXTRACT ALL STRIPS
tmp_dir = create_tmp_dir_for_outfile(
os.path.join(dem_dir_out, 'all_strips'))
list_tmp_dem = [
os.path.join(
tmp_dir,
os.path.splitext(
os.path.splitext(os.path.basename(seg_tar_gz))[0])[0] +
'_dem.tif') for seg_tar_gz in seg_tar_gz_list
]
for seg_tar_gz in seg_tar_gz_list:
print('Extracting dem file of segment ' +
str(seg_tar_gz_list.index(seg_tar_gz) + 1) + ' out of ' +
str(len(seg_tar_gz_list)))
extract_file_from_tar_gz(
seg_tar_gz,
os.path.splitext(
os.path.splitext(os.path.basename(seg_tar_gz))[0])[0] +
'_dem.tif', list_tmp_dem[seg_tar_gz_list.index(seg_tar_gz)])
if rm_tar:
# once extracted, remove original data to avoid taking up too much space
shutil.rmtree(subtile_dir)
if downsample:
list_ds = []
for tmp_dem in list_tmp_dem:
tmp_dem_ds = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tmp_dem))[0] + '_ds.tif')
list_ds.append(tmp_dem_ds)
translate(tmp_dem,
tmp_dem_ds,
format_out='GTiff',
tgt_res=[8, -8],
interp_method='bilinear')
os.remove(tmp_dem)
list_tmp_dem = list_ds
# 2.5/ MOSAIC/COREGISTER SETSM SEGMENTS ITERATIVELY INTO STRIPS
if mosaic_coreg_segm:
max_coreg_rmse = 10.
nb_init_pixel_min = 1000
if not os.path.exists(os.path.join(dem_dir_out, 'coreg_results')):
os.mkdir(os.path.join(dem_dir_out, 'coreg_results'))
if not os.path.exists(
os.path.join(dem_dir_out, 'coreg_results', 'good')):
os.mkdir(os.path.join(dem_dir_out, 'coreg_results', 'good'))
if not os.path.exists(
os.path.join(dem_dir_out, 'coreg_results', 'breaks')):
os.mkdir(os.path.join(dem_dir_out, 'coreg_results', 'breaks'))
summary = os.path.join(dem_dir_out,
'mosaic_coreg_summary_' + tile_name + '.csv')
with open(summary, 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow([
'scene', 'segment', 'master', 'dx', 'dy', 'dz', 'RMSE',
'mosaic_nb'
])
# create segment lists
list_seg = list(list_tmp_dem)
list_seg_id = []
for seg in list_seg:
seg_id_split = os.path.basename(seg).split('_')
seg_id = '_'.join(seg_id_split[0:5])
seg_suf = '_'.join(seg_id_split[6:])
# seg_nb=int(seg_id_split[5][3:])
if seg_id not in list_seg_id:
list_seg_id.append(seg_id)
# list all list of scenes acquired the same day
list_list_by_seg_id = []
for seg_id in list_seg_id:
tmp_list = [seg for seg in list_seg if seg_id in seg]
tmp_list = sorted(
tmp_list,
key=lambda x: x.split('/')[-1].split('_')[5][3:].zfill(2))
list_list_by_seg_id.append(tmp_list)
# coregister in series
list_final_strip = []
print('List of list of segments:')
print(list_list_by_seg_id)
# loop through same day scenes
for list_by_seg_id in list_list_by_seg_id:
print(list_by_seg_id)
# bother to coreg only if there is more than 1 segment
if len(list_by_seg_id) > 1:
tmp_dir_coreg = create_tmp_dir_for_outfile(
os.path.join(tmp_dir, 'coreg'))
# master dem is the first segment
dem_master = list_by_seg_id[0]
k = 0
# for csv summary: first segment
dem_master_id_split = os.path.basename(dem_master).split(
'_')
dem_master_id = '_'.join(dem_master_id_split[0:5])
dem_master_seg_nb = int(dem_master_id_split[5][3:])
writer.writerow([
dem_master_id, dem_master_seg_nb, dem_master_seg_nb,
'first_seg', 'first_seg', 'first_seg', 'first_seg',
str(k + 1)
])
for i in range(len(list_by_seg_id) - 1):
dem_in = list_by_seg_id[i + 1]
print('Master segment is ' + dem_master)
print('Slave segment is ' + dem_in)
# for csv summary: all segments
dem_in_id_split = os.path.basename(dem_in).split('_')
dem_in_id = '_'.join(dem_in_id_split[0:5])
dem_in_seg_nb = int(dem_in_id_split[5][3:])
dem_master_id_split = os.path.basename(
dem_master).split('_')
dem_master_seg_nb = int(dem_master_id_split[5][3:])
k0 = k
# deriving intersection
extent = inters_raster(dem_in, dem_master)
nb_init_pixel = valid_points_coreg(dem_master, dem_in)
# try coreg only if intersection is non void
if extent is not None and nb_init_pixel > nb_init_pixel_min:
tmp_dir_seg = create_tmp_dir_for_outfile(dem_in)
# do coreg, save results
try:
out_offs, final_rmse = dem_coregistration_custom(
dem_master, dem_in,
outdir=tmp_dir_seg)[2:4]
except Exception:
out_offs = None
final_rmse = 999
if final_rmse < max_coreg_rmse:
str_out = 'good'
else:
str_out = 'breaks'
shutil.move(
os.path.join(tmp_dir_seg,
'CoRegistration_Results.pdf'),
os.path.join(
dem_dir_out, 'coreg_results', str_out,
os.path.splitext(
os.path.basename(dem_in))[0] + '_' +
os.path.splitext(
os.path.basename(dem_master))[0] +
'_coreg_results.pdf'))
shutil.move(
os.path.join(tmp_dir_seg, 'coreg_params.txt'),
os.path.join(
dem_dir_out, 'coreg_results', str_out,
os.path.splitext(
os.path.basename(dem_in))[0] + '_' +
os.path.splitext(
os.path.basename(dem_master))[0] +
'_coreg_params.txt'))
print('Final coregistration RMSE is:' +
str(final_rmse))
# if rmse is good, keep doing mosaics with next segment: mosaic is new master
if final_rmse < max_coreg_rmse:
print(
'All good. Doing mosaic and iterating to next segment...'
)
tmp_mosaic_list = [
dem_master,
os.path.join(
tmp_dir_seg,
os.path.splitext(
os.path.basename(dem_in))[0] +
'_adj.tif')
]
tmp_dem_coreg = os.path.join(
tmp_dir_coreg,
os.path.basename(list_by_seg_id[i + 1]))
dem_mosaic(tmp_mosaic_list,
tmp_dem_coreg,
erode_length=0,
hole_fill_length=0,
blending_method='mean')
dem_master = tmp_dem_coreg
# if this is the last segment, save mosaic and add to list of finalized mosaics
if i == (len(list_by_seg_id) - 2):
k = k + 1
final_strip = os.path.join(
tmp_dir,
list_seg_id[list_list_by_seg_id.index(
list_by_seg_id)] + '_mosaic' +
str(k) + '_' + seg_suf)
shutil.copyfile(dem_master, final_strip)
list_final_strip.append(final_strip)
# otherwise, break the iterative mosaic-ing
else:
# if this is the last segment
if i == (len(list_by_seg_id) - 2):
# if master is the previous segment alone
if dem_master == list_by_seg_id[i]:
print(
'RMSE is larger than limit of ' +
str(max_coreg_rmse) +
'm: saving last segments independently...'
)
list_final_strip.append(dem_in)
list_final_strip.append(dem_master)
# if master is a previous ongoing mosaic
else:
k = k + 1
final_strip = os.path.join(
tmp_dir,
list_seg_id[list_list_by_seg_id.
index(list_by_seg_id)]
+ '_mosaic' + str(k) + '_' +
seg_suf)
shutil.copyfile(
dem_master, final_strip)
print(
'RMSE is larger than limit of ' +
str(max_coreg_rmse) +
'm: saving previous mosaic and last segment...'
)
list_final_strip.append(dem_in)
list_final_strip.append(final_strip)
# if there is still segments to go
else:
# save current mosaic
k = k + 1
final_strip = os.path.join(
tmp_dir,
list_seg_id[list_list_by_seg_id.index(
list_by_seg_id)] + '_mosaic' +
str(k) + '_' + seg_suf)
shutil.copyfile(dem_master, final_strip)
print(
'RMSE is larger than limit of ' +
str(max_coreg_rmse) +
'm: saving previous mosaic, slave is new master...'
)
list_final_strip.append(final_strip)
# new master is the segment that did not mosaic
dem_master = dem_in
remove_tmp_dir_for_outfile(dem_in)
# if no intersection and last segment
elif i == (len(list_by_seg_id) - 2):
out_offs = None
final_rmse = None
if dem_master == list_by_seg_id[i]:
print(
'No intersection between master and slave: saving last segments independently...'
)
list_final_strip.append(dem_in)
list_final_strip.append(dem_master)
# if master is a previous ongoing mosaic
else:
k = k + 1
final_strip = os.path.join(
tmp_dir,
list_seg_id[list_list_by_seg_id.index(
list_by_seg_id)] + '_mosaic' + str(k) +
'_' + seg_suf)
shutil.copyfile(dem_master, final_strip)
print(
'No intersection between master and slave: saving previous mosaic and last segment...'
)
list_final_strip.append(dem_in)
list_final_strip.append(final_strip)
# if no intersection and still segments to go
else:
out_offs = None
final_rmse = None
# save current mosaic
k = k + 1
final_strip = os.path.join(
tmp_dir, list_seg_id[list_list_by_seg_id.index(
list_by_seg_id)] + '_mosaic' + str(k) +
'_' + seg_suf)
shutil.copyfile(dem_master, final_strip)
print(
'No intersection between master and slave: saving previous mosaic, slave is new master...'
)
list_final_strip.append(final_strip)
# new master is the segment that did not mosaic
dem_master = dem_in
# write data in summary
if out_offs is None:
dx = 'no_inters'
dy = 'no_inters'
dz = 'no_inters'
rmse = 'no_inters'
else:
dx = str(out_offs[0])
dy = str(out_offs[1])
dz = str(out_offs[2])
rmse = str(final_rmse)
writer.writerow([
dem_in_id, dem_in_seg_nb, dem_master_seg_nb, dx,
dy, dz, rmse,
str(k0 + 1)
])
remove_tmp_dir_for_outfile(os.path.join(tmp_dir, 'coreg'))
else:
list_final_strip.append(list_by_seg_id[0])
dem_master_id_split = os.path.basename(
list_by_seg_id[0]).split('_')
dem_master_id = '_'.join(dem_master_id_split[0:5])
dem_master_seg_nb = int(dem_master_id_split[5][3:])
writer.writerow([
dem_master_id, dem_master_seg_nb, dem_master_seg_nb,
'only_seg', 'only_seg', 'only_seg', 'only_seg',
'only_seg'
])
else:
list_final_strip = list_tmp_dem
# 3/ PROCESSING OF FINAL STRIPS
final_dem = []
for final_strip in list_final_strip:
# 3.1/ FILTER STRIPS
if filter_params is not None:
sys.exit('No filter pre-defined for this DEM product.')
# 3.2/ REPROJECT STRIPS
# raw data is GeoTiff, 3413, 1 arc-sec and -9999 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 3031 and tgt_res is None and nodata_out is -9999:
tmp_dem_proj = final_strip
else:
tmp_dem_proj = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(final_strip))[0] +
'_proj.tif')
warp_defaultUTM(final_strip, tmp_dem_proj, format_out, 3031,
tgt_EPSG, tgt_res, nodata_out, interp_method)
# 3.3/ ELLIPSOID OR GEOID STRIPS
# raw data is ellipsoid WGS84
if geoid:
tmp_dem_geoid = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(final_strip))[0] +
'_geoid.tif')
ellipsoid_to_geoid(tmp_dem_proj, tmp_dem_geoid)
else:
tmp_dem_geoid = tmp_dem_proj
final_dem.append(tmp_dem_geoid)
for dem in final_dem:
shutil.copyfile(dem, os.path.join(dem_dir_out, os.path.basename(dem)))
remove_tmp_dir_for_outfile(os.path.join(dem_dir_out, 'all_strips'))
print('Fin.')
def ASTGTM2_V002_tile(dir_ASTGTM2,
tile_name,
dem_out,
filter_params=None,
format_out='GTiff',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=True):
"""
:param dir_ASTGTM2: path to directory directly containing zip of ASTGTM2
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with ASTGTM2 NUM file using rastlib.filter_nanarray function
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return:
ASTGTM2: ASTER Global Digital Elevation Model V002 from NASA at: https://lpdaac.usgs.gov/node/1079
NUM file description
Fill value: -1 to -11
-1 SRTM3 V003
-2 SRTM3 V002
-5 NED
-6 CDED
-11 Alaska DEM
Valid range (ASTER count):
0 to 200
OPTIMAL DIRECTORY ARCHITECTURE: ASTGTM2 zip archives in the same directory
"""
#1/ LOCATE 1x1° tile
#nomenclature of ASTGTM2 zip files
tile_zip = os.path.join(dir_ASTGTM2, 'ASTGTM2_' + tile_name + '.zip')
dem_file = 'ASTGTM2_' + tile_name + '_dem.tif'
num_file = 'ASTGTM2_' + tile_name + '_num.tif'
#2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
tmp_dem = os.path.join(tmp_dir, dem_file)
tmp_num = os.path.join(tmp_dir, num_file)
extract_file_from_zip(tile_zip, dem_file, tmp_dem)
extract_file_from_zip(tile_zip, num_file, tmp_num)
#3/ FILTER TILE
if filter_params is not None:
num = read_nanarray(tmp_num)
_, filt = filter_nanarray(num, filter_params[0], filter_params[1],
filter_params[2])
dem = read_nanarray(tmp_dem)
dem_filtered = np.array(dem)
dem_filtered[filt] = np.NaN
update_nanarray(tmp_dem, dem_filtered)
#4/ REPROJECT TILE
# raw data is GTiff, 4326, 1 arc-sec and -32768 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -9999:
tmp_dem_proj = tmp_dem
else:
tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif')
warp_defaultUTM(tmp_dem, tmp_dem_proj, format_out, 4326, tgt_EPSG,
tgt_res, nodata_out, interp_method)
#5/ ELLIPSOID OR GEOID
# raw data is geoid EGM96
if not geoid:
geoid_to_ellipsoid(tmp_dem_proj, dem_out)
else:
shutil.move(tmp_dem_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def SRTMGL1_V003_tile(dir_SRTMGL1,
tile_name,
dem_out,
filter_params=None,
format_out='SRTMHGT',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-32768,
interp_method=None,
geoid=True):
"""
:param dir_SRTMGL1: path to directory directly containing zip of SRTMGL1 and SRTMGL1N
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with SRTMGL1 NUM file using rastlib.filter_nanarray function
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return:
SRTMGL1: Shuttle Radar Topography Mission Global 1-arcsec V003 from NASA at: https://lpdaac.usgs.gov/sites/default/files/public/measures/docs/NASA_SRTM_V3.pdf
NUM file description
1 = Water-masked SRTM void *
2 = Water-masked SRTM non-void *
5 = GDEM elevation = 0 in SRTM void (helped correct ocean masking)
11 = NGA-interpolated SRTM (were very small voids in SRTMv1) 21 = GMTED2010 oversampled from 7.5 arc-second postings
25 = SRTM within GDEM **
31 = NGA fill of SRTM via GDEM***
51 = USGS NED
52 = USGS NED via GDEM
53 = Alaska USGS NED via GDEM
72 = Canadian Digital Elevation Data (CDED) via GDEM 101-200 = ASTER scene count (count limited to 100)
201-224 = SRTM swath count (non-voided swaths) Actual maximum = 24
OPTIMAL DIRECTORY ARCHITECTURE: SRTMGL1 and SRTMGL1N zip archives in the same directory
"""
#1/ LOCATE 1x1° tile
#nomenclature of SRTMGL1 and SRTMGL1N zip files
hgt_zip = os.path.join(dir_SRTMGL1, tile_name + '.SRTMGL1.hgt.zip')
num_zip = os.path.join(
dir_SRTMGL1, tile_name + '.SRTMGL1N.num.zip'
) #NOTE: nomenclature of V002 was SRTMGL1.num.zip for the num file
#2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
zip_ref = zipfile.ZipFile(hgt_zip, 'r')
zip_ref.extractall(tmp_dir)
zip_ref.close()
zip_ref = zipfile.ZipFile(num_zip, 'r')
zip_ref.extractall(tmp_dir)
zip_ref.close()
tmp_hgt = os.path.join(tmp_dir, tile_name + '.hgt')
tmp_num = os.path.join(tmp_dir, tile_name + '.num')
#3/ FILTER TILE
if filter_params is not None:
num = read_nanarray(tmp_num)
_, filt = filter_nanarray(num, filter_params[0], filter_params[1],
filter_params[2])
hgt = read_nanarray(tmp_hgt)
hgt_filtered = np.array(hgt)
hgt_filtered[filt] = np.NaN
update_nanarray(tmp_hgt, hgt_filtered)
#4/ REPROJECT TILE
# raw data is SRTMHGT, 4326, 1 arc-sec and -32768 nodata_out
if format_out == 'SRTMHGT' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32768:
tmp_hgt_proj = tmp_hgt
else:
tmp_hgt_proj = os.path.join(tmp_dir, tile_name + '_proj.tif')
warp_defaultUTM(tmp_hgt, tmp_hgt_proj, format_out, 4326, tgt_EPSG,
tgt_res, nodata_out, interp_method)
#5/ ELLIPSOID OR GEOID
# raw data is geoid EGM96
if not geoid:
geoid_to_ellipsoid(tmp_hgt_proj, dem_out)
else:
shutil.move(tmp_hgt_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def AW3D30_v2_1_tile(dir_ALOS,
tile_name,
dem_out,
bit_params=None,
format_out='GTiff',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=True):
"""
:param dir_ALOS: path to directory directly containing ALOS World 3D 30-m 5x5° tar.gz archives
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param bit_params: bit filtering for AW3D30 MSK file ; see rastlib.mask_bitarray function for an example
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return
ALOS World 3D-30m (AW3D30) Version 2.1 from JAXA at: https://www.eorc.jaxa.jp/ALOS/en/aw3d30/aw3d30v21_format_e.pdf
MSK file:
0000: Valid
0001: Cloud and show mask (invalid)
0010: Land water and low correlation mask of the 5 m resolution DSM (valid)
0011: Sea mask (valid)
0100: National Land Numerical Information 10 m DEM (by Geographical Survey Institute of Japan) (valid)
1000: Shuttle Radar Topography Mission (SRTM) SRTM-1 Version 3 （valid）
1100: PRISM DSM (valid)
STK, HDR, QAI, LST files not used in this function, could be extracted same manner as other in 2/
OPTIMAL DIRECTORY ARCHITECTURE: all 5x5 .tar.gz archives in the same directory
"""
#1/ LOCATE 1x1° TILE
#ALOS AW3D30 v2.1 5x5 archive naming convention
def latlon_to_ALOS_5x5tile_naming(lat, lon):
lat_5 = np.floor(lat / 5) * 5
lon_5 = np.floor(lon / 5) * 5
if lat_5 >= 0:
str_lat = 'N'
else:
str_lat = 'S'
if lon_5 >= 0:
str_lon = 'E'
else:
str_lon = 'W'
tile_name_5 = str_lat + str(lat_5).zfill(3) + str_lon + str(
lon_5).zfill(3)
return tile_name_5
#identify in which 5x5 tar.gz the 1x1 tile is located
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
name_5x5 = latlon_to_ALOS_5x5tile_naming(lat_tile, lon_tile)
list_5x5 = os.listdir(dir_ALOS)
list_5x5_prefix = [l[0:8] for l in list_5x5]
if name_5x5 in list_5x5_prefix:
tar_5x5 = list_5x5[list_5x5_prefix.index(name_5x5)]
else:
sys.exit(
'Could not find an ALOS AW3D30 5x5° archive containing the tile ' +
tile_name + ' in the directory ' + dir_ALOS)
#2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
alos_tile_name = tile_name[0:1] + str(0) + tile_name[
1:] #SRTMGL1 naming convention to ALOS 1x1 naming convention (3 digits for latitude)
#extract files of interest
dsm_file = alos_tile_name + '_AVE_DSM.tif'
msk_file = alos_tile_name + '_AVE_MSK.tif'
tmp_dsm = os.path.join(tmp_dir, dsm_file)
tmp_msk = os.path.join(tmp_dir, msk_file)
extract_file_from_tar_gz(os.path.join(dir_ALOS, tar_5x5), dsm_file,
tmp_dsm)
extract_file_from_tar_gz(os.path.join(dir_ALOS, tar_5x5), msk_file,
tmp_msk)
#3/ FILTER TILE
mask = read_nanarray(tmp_msk)
#no filtering by default
if bit_params is not None:
mask_goodval = mask_bitarray(mask, bit_params[0], bit_params[1])
dsm = read_nanarray(tmp_dsm)
dsm_filtered = np.array(dsm)
dsm_filtered[mask_goodval] = np.NaN
update_nanarray(tmp_dsm, dsm_filtered)
#4/ REPROJECT TILE:
#raw data is GeoTiff, 4326, 1 arc-sec and -9999 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -9999:
tmp_dsm_proj = tmp_dsm
else:
tmp_dsm_proj = os.path.join(tmp_dir, alos_tile_name + '_proj.tif')
warp_defaultUTM(tmp_dsm, tmp_dsm_proj, format_out, 4326, tgt_EPSG,
tgt_res, nodata_out, interp_method)
#5/ ELLIPSOID OR GEOID
#raw data is geoid EGM96
if not geoid:
geoid_to_ellipsoid(tmp_dsm_proj, dem_out)
else:
shutil.move(tmp_dsm_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def REMA_mosaic_r1_1_tile(dir_REMA,
tile_name,
dem_out,
filter_params=None,
format_out='GTiff',
tgt_EPSG=3031,
tgt_res=None,
nodata_out=-9999,
interp_method=None,
geoid=False,
tag_lonlat_tile=False,
path_tile_index=None,
tag_merge=False,
tag_clip=False):
"""
:param dir_REMA: path to parent directory "8m" containing subdirectories of tar.gz archives (native FTP architecture)
:param tile_name: either REMA tile name or 1x1° lat/lon tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with REMA ERR file using rastlib.filter_nanarray function
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:param tag_lonlat_tile: True if tile_name follows SRTMGL1 tile naming, False if tile_name follows REMA tile naming
:param path_tile_index: if tile_name is REMA format, specify path to ESRI REMA Tile Index
:param tag_merge: if tile_name is REMA format, True to merge all ArcticDEM tiles to the 1x1° lat/lon extent
:param tag_clip: if tile_name is REMA format, True to clip the 5x5° tile to the 1x1° lat/lon extent of tile_name
:return:
REMA release 1.1 product: ref:https://www.pgc.umn.edu/data/rema/
Processing for 8m mosaic
(100m, 500m and 1km versions are bundled in one .tif file)
Tile name and processing is REMA tile naming convention by default
Provide path to ESRI tile index file to use 1x1° lat/lon tiles and SRTMGL1 naming convention
OPTIMAL DIRECTORY ARCHITECTURE: point to "8m" folder of similar architecture than: ftp://ftp.data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.0
"""
# 1/ LOCATE TILE
if not tag_lonlat_tile:
subtile_dir = os.path.join(dir_REMA, tile_name)
tile_tar_gz_list = [
os.path.join(subtile_dir, tar_file)
for tar_file in os.listdir(subtile_dir)
if tar_file.endswith('.tar.gz')
]
else:
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
extent = [lat_tile, lon_tile, lat_tile + 1, lon_tile + 1]
# feature name in REMA_Tile_Index_Rel1.1
feat_name = 'TILE'
subtile_name_list = list_shp_field_inters_extent(
path_tile_index, feat_name, extent, 4326)
subtile_dir_list = [
os.path.join(dir_REMA, tile) for tile in subtile_name_list
]
tile_tar_gz_list = []
for i in range(len(subtile_dir_list)):
tile_tar_gz_list = tile_tar_gz_list + [
os.path.join(subtile_dir_list[i], tar_file)
for tar_file in os.listdir(subtile_dir_list[i])
if tar_file.endswith('.tar.gz')
]
# 2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
list_tmp_dem = [
os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_dem.tif')
for tile_tar_gz in tile_tar_gz_list
]
for tile_tar_gz in tile_tar_gz_list:
extract_file_from_tar_gz(
tile_tar_gz,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_dem.tif',
list_tmp_dem[tile_tar_gz_list.index(tile_tar_gz)])
# list_tmp_err = [tmp_dir + os.path.splitext(os.path.basename(tile_tar_gz))[0]+'_err.tif' for tile_tar_gz in tile_tar_gz_list]
for tile_tar_gz in tile_tar_gz_list:
extract_file_from_tar_gz(
tile_tar_gz,
os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_err.tif',
list_tmp_dem[tile_tar_gz_list.index(tile_tar_gz)])
list_tmp_dem_tomerge = []
for tmp_dem in list_tmp_dem:
# 3/ FILTER TILE
if filter_params is not None:
tmp_err = tmp_dem[:-8] + '_err.tif'
err = read_nanarray(tmp_err)
_, filt = filter_nanarray(err, filter_params[0], filter_params[1],
filter_params[2])
dem = read_nanarray(tmp_dem)
dem_filtered = np.array(dem)
dem_filtered[filt] = np.NaN
update_nanarray(tmp_dem, dem_filtered)
# 4/ REPROJECT TILE
# raw data is GeoTiff, 3031, 1 arc-sec and -9999 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 3031 and tgt_res is None and nodata_out is -9999:
tmp_dem_proj = tmp_dem
else:
tmp_dem_proj = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tmp_dem))[0] + '_proj.tif')
warp_defaultUTM(tmp_dem, tmp_dem_proj, format_out, 3031, tgt_EPSG,
tgt_res, nodata_out, interp_method)
# 5/ ELLIPSOID OR GEOID
# raw data is ellipsoid WGS84
if geoid:
tmp_dem_geoid = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(tmp_dem))[0] + '_geoid.tif')
ellipsoid_to_geoid(tmp_dem_proj, tmp_dem_geoid)
else:
tmp_dem_geoid = tmp_dem_proj
list_tmp_dem_tomerge.append(tmp_dem_geoid)
# 6/ MERGE ALL TILES
tmp_dem_merged = os.path.join(tmp_dir, tile_name + '_merged.tif')
if tag_merge:
merge_rast_list(list_tmp_dem_tomerge, tmp_dem_merged)
else:
shutil.copytree(
tmp_dir,
os.path.join(os.path.dirname(dem_out), tile_name + '_subtiles'))
# 7/ CLIP TO TILE EXTENT
if not tag_clip:
tmp_dem_clipped = os.path.join(tmp_dir, tile_name + '_clipped.tif')
lat, lon = SRTMGL1_naming_to_latlon(tile_name)
clip_rast_to_extent(tmp_dem_merged, tmp_dem_clipped,
[lat, lon, lat + 1, lon + 1], 4326)
else:
tmp_dem_clipped = tmp_dem_merged
shutil.move(tmp_dem_clipped, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def SRTMv4_1_tile(dir_SRTMv4_1,
tile_name,
dem_out,
tag_clip=False,
filter_params=None,
format_out='GTiff',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-32768,
interp_method=None,
geoid=True):
"""
:param dir_SRTMv4_1: path to parent directory containing "6_5x5_TIFS" directory of zip archives (bulk downloading)
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param tag_clip: True to clip the 5x5° tile to the 1x1° extent of tile_name
:param filter_params: no filtering for this product
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return:
SRTM 90m Digital Elevation Database v4.1 by NASA/CGIARCSI at: https://cgiarcsi.community/data/srtm-90m-digital-elevation-database-v4-1/
Product Description temporarily not available (web issue)
OPTIMAL DIRECTORY ARCHITECTURE: bulk download format, point to the directoy containing 6_5x5_TIFS directory
"""
def latlon_to_SRTMv4_1_5x5tile_naming(lat, lon):
ind_lon = np.floor(lon + 180 / 5) + 1
ind_lat = np.floor(lat + 90 / 5) + 1
return ind_lat, ind_lon
# 1/ LOCATE 5x5° tile
#read lat, lon according to SRTMv4.1 naming convention
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
ind_lat_4_1, ind_lon_4_1 = latlon_to_SRTMv4_1_5x5tile_naming(
lat_tile, lon_tile)
# nomenclature of SRTMv4.1 archives
tile_zip = os.path.join(
dir_SRTMv4_1, '6_5x5_TIFS', 'srtm_' + str(ind_lon_4_1).zfill(2) + '_' +
str(ind_lat_4_1).zfill(2) + '.zip')
# 2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
dem_file = 'srtm_' + str(ind_lon_4_1).zfill(2) + '_' + str(
ind_lat_4_1).zfill(2) + '.tif'
tmp_dem = os.path.join(tmp_dir, dem_file)
extract_file_from_zip(tile_zip, dem_file, tmp_dem)
#2.5/ CLIP TO TILE EXTENT
if not tag_clip:
tmp_dem_clipped = os.path.join(
tmp_dir, 'srtm_' + str(ind_lon_4_1).zfill(2) + '_' +
str(ind_lat_4_1).zfill(2) + '_clipped.tif')
clip_rast_to_extent(tmp_dem, tmp_dem_clipped,
[lat_tile, lon_tile, lat_tile + 1, lon_tile + 1],
4326)
else:
tmp_dem_clipped = tmp_dem
# 3/ FILTER TILE
if filter_params is not None:
sys.exit('No filter pre-defined for this DEM product.')
# 4/ REPROJECT TILE
# raw data is GeoTiff, 4326, 3 arc-sec and -32768 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32768 and tag_clip:
tmp_dem_proj = tmp_dem_clipped
else:
tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif')
warp_defaultUTM(tmp_dem_clipped, tmp_dem_proj, format_out, 4326,
tgt_EPSG, tgt_res, nodata_out, interp_method)
# 5/ ELLIPSOID OR GEOID
# raw data is geoid EGM96
if not geoid:
geoid_to_ellipsoid(tmp_dem_proj, dem_out)
else:
shutil.move(tmp_dem_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
def TDX_90m_tile(dir_TDX,
tile_name,
dem_out,
filter_params=None,
format_out='GTiff',
tgt_EPSG=4326,
tgt_res=None,
nodata_out=-32767,
interp_method=None,
geoid=False):
"""
:param dir_TDX: path to directory directly containing TanDEM-X zip archives
:param tile_name: 1x1° tile name (SRTMGL1/classic naming convention)
:param dem_out: path to DEM out file
:param filter_params: filtering with TDX HEM file ; see rastlib.filter_nanarray function for an example
:param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html
:param tgt_EPSG: EPSG of output projection
:param tgt_res: output resolution, GDAL naming [xres, yres]
:param nodata_out: output no-data value
:param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc..
:param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid
:return:
TanDEM-X 90m product from DLR at: https://geoservice.dlr.de/web/dataguide/tdm90/pdfs/TD-GS-PS-0021_DEM-Product-Specification.pdf
Only HEM (Height Error Map) is extracted here
Using the same approach with binary/nanarray filters, 2/ can extract and 3/ can process the AMP, AM2, WAM, COV, COM and LSM files
OPTIMAL DIRECTORY ARCHITECTURE: all .zip archives in the same directory
"""
# 1/ LOCATE 1x1° TILE
def latlon_to_TDX_tile_naming(lat, lon):
if lat >= 80:
lon_sw = np.floor(lon / 4) * 4
elif lat >= 60:
lon_sw = np.floor(lon / 2) * 2
else:
lon_sw = lon
lat_sw = lat
if lat_sw >= 0:
str_lat = 'N'
else:
str_lat = 'S'
if lon_sw >= 0:
str_lon = 'E'
else:
str_lon = 'W'
tile_name_tdx = str_lat + str(int(
abs(lat_sw))).zfill(2) + str_lon + str(int(abs(lon_sw))).zfill(3)
return tile_name_tdx
#identify in which TDX tile the 1x1 tile is located
lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name)
name_tdx = latlon_to_TDX_tile_naming(lat_tile, lon_tile)
list_tdx = os.listdir(dir_TDX)
list_prefix = [l[13:20] for l in list_tdx]
if name_tdx in list_prefix:
pass
# zip_tdx=list_prefix[list_prefix.index(name_tdx)]
else:
sys.exit('Could not find a TDX tile archive containing the tile ' +
name_tdx + ' in the directory ' + dir_TDX)
tile_name = name_tdx
# nomenclature of TDX 3 arc-sec files
tile_zip = os.path.join(dir_TDX, 'TDM1_DEM__30_' + name_tdx + '.zip')
dem_out = dem_out + '_' + name_tdx + '.tif'
# 2/ EXTRACT TILE
tmp_dir = create_tmp_dir_for_outfile(dem_out)
dem_file = 'TDM1_DEM__30_' + tile_name + '_DEM.tif'
hem_file = 'TDM1_DEM__30_' + tile_name + '_HEM.tif'
tmp_dem = os.path.join(tmp_dir, dem_file)
tmp_hem = os.path.join(tmp_dir, hem_file)
extract_file_from_zip(tile_zip, dem_file, tmp_dem)
extract_file_from_zip(tile_zip, hem_file, tmp_hem)
# 3/ FILTER TILE
if filter_params is not None:
hem = read_nanarray(tmp_hem)
_, filt = filter_nanarray(hem, filter_params[0], filter_params[1],
filter_params[2])
dem = read_nanarray(tmp_dem)
dem_filtered = np.array(dem)
dem_filtered[filt] = np.NaN
update_nanarray(tmp_dem, dem_filtered)
# 4/ REPROJECT TILE
# raw data is GTiff, 4326, 3 arc-sec and -32768 nodata_out
if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32767:
tmp_dem_proj = tmp_dem
else:
tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif')
warp_defaultUTM(tmp_dem,
tmp_dem_proj,
format_out=format_out,
src_EPSG=4326,
tgt_EPSG=tgt_EPSG,
tgt_res=tgt_res,
nodata_in=-32767,
nodata_out=nodata_out,
interp_method=interp_method)
# 5/ ELLIPSOID OR GEOID
# raw data is ellipsoid WGS84
if geoid:
ellipsoid_to_geoid(tmp_dem_proj, dem_out)
else:
shutil.move(tmp_dem_proj, dem_out)
remove_tmp_dir_for_outfile(dem_out)
