def clip_rast_to_shp_sql(raster_in, raster_out, shp_in, sql_req): #reproject shp to similar proj if needed rast_proj = read_proj_rast(raster_in) shp_proj = read_proj_shp(shp_in) comp_proj, rast_epsg, _ = compare_proj_wkt_or_EPSG(True, rast_proj, True, shp_proj) if not comp_proj: tmp_dir = create_tmp_dir_for_outfile(raster_out) tmp_shp = tmp_dir + os.path.splitext( os.path.basename(shp_in))[0] + '_tmp.shp' os.system('ogr2ogr -t_srs ' + rast_epsg + ' ' + tmp_shp + ' ' + shp_in) final_shp = tmp_shp else: final_shp = shp_in #example of sql request #sql_req = "SELECT * FROM your_shapefile" gdal.Warp(raster_out, raster_in, cutlineDSName=final_shp, cutlineSQL=sql_req, cropToCutline=True) remove_tmp_dir_for_outfile(raster_out)
def saga_aspect_slope_curvature(dem_in,topo_out,method_nb=5): """ :param dem_in: dem input :param topo_out: raster out (3 bands: aspect, slope, curvature) :param method_nb: algorithm used, see function description :return: requirement: SAGA 2.X with X>3 #ref:http://www.saga-gis.org/saga_tool_doc/2.2.3/ta_morphometry_0.html aspect, slope, curvature methods methods number [0] maximum slope (Travis et al. 1975) [1] maximum triangle slope (Tarboton 1997) [2] least squares fitted plane (Horn 1981, Costa-Cabral & Burgess 1996) [3] 6 parameter 2nd order polynom (Evans 1979) [4] 6 parameter 2nd order polynom (Heerdegen & Beran 1982) [5] 6 parameter 2nd order polynom (Bauer, Rohdenburg, Bork 1985) [6] 9 parameter 2nd order polynom (Zevenbergen & Thorne 1987) [7] 10 parameter 3rd order polynom (Haralick 1983) unit slope 0=rad, 1=deg, 2=percent unit aspect 0=rad, 1=deg """ tmp_dir=create_tmp_dir_for_outfile(topo_out) saga_elev = tmp_dir + 'elev_temp.sgrd' saga_slope = tmp_dir + 'slope_temp.sgrd' saga_aspect = tmp_dir + 'aspect_temp.sgrd' saga_max_curv = tmp_dir + 'max_curv_temp.sgrd' tif_slope = tmp_dir + 'slope_temp.tif' tif_aspect = tmp_dir + 'aspect_temp.tif' tif_max_curv = tmp_dir + 'max_curv_temp.tif' output_vrt = tmp_dir + 'stack.vrt' os.system('saga_cmd io_gdal 0 -GRIDS ' + saga_elev + ' -FILES ' + dem_in) os.system('saga_cmd ta_morphometry 0 -ELEVATION ' + saga_elev + ' -SLOPE ' + saga_slope + ' -ASPECT ' + saga_aspect + ' -C_MAXI ' + saga_max_curv + ' -METHOD ' +str(method_nb)+' -UNIT_SLOPE 1 -UNIT_ASPECT 1') os.system('saga_cmd io_gdal 2 -GRIDS ' + saga_slope + ' -FILE ' + tif_slope) os.system('saga_cmd io_gdal 2 -GRIDS ' + saga_aspect + ' -FILE ' + tif_aspect) os.system('saga_cmd io_gdal 2 -GRIDS ' + saga_max_curv + ' -FILE ' + tif_max_curv) os.system('gdalbuildvrt -separate -overwrite ' + output_vrt + ' ' + tif_slope + ' ' + tif_aspect + ' ' + tif_max_curv) os.system('gdal_translate ' + output_vrt + ' ' + topo_out) remove_tmp_dir_for_outfile(topo_out)
def crop_raster_nodata(raster_in, raster_out): tmp_dir = create_tmp_dir_for_outfile(raster_out) tmp_raster = os.path.join( tmp_dir, 'tmp' + os.path.splitext(os.path.basename(raster_in))[0]) tmp_nodataextent = os.path.join(tmp_dir, 'tmp.shp') os.system('gdal_calc.py -A ' + raster_in + ' --outfile ' + tmp_raster + ' --NoDataValue 0 --calc="1*(A>0)"') os.system('gdal_polygonize.py -8 -b 1 -f "ESRI Shapefile" ' + tmp_raster + ' ' + tmp_nodataextent) clip_rast_to_shp_sql(raster_in, raster_out, tmp_nodataextent, None) remove_tmp_dir_for_outfile(raster_out)
def merge_rast_list(list_raster, raster_out, tgt_EPSG=None, nodata_in=None, nodata_out=None): #reproject to similar proj if specified if tgt_EPSG is not None: tmp_dir = create_tmp_dir_for_outfile(raster_out) list_raster_proj = [] for rast in list_raster: print(rast) ind = list_raster.index(rast) rast_proj = tmp_dir + os.path.splitext( os.path.basename(rast))[0] + '_' + str(ind) + '_proj.tif' warp_defaultUTM(rast, rast_proj, tgt_EPSG=tgt_EPSG, nodata_out=nodata_out) list_raster_proj.append(rast_proj) final_list_raster = list_raster_proj else: final_list_raster = list_raster if nodata_in is None: str_nodata_in = '' else: str_nodata_in = ' -n ' + str(nodata_in) if nodata_out is None: str_nodata_out = '' else: str_nodata_out = ' -a_nodata ' + str(nodata_out) if os.path.exists(raster_out): os.remove(raster_out) os.system('gdal_merge.py -o ' + raster_out + str_nodata_in + str_nodata_out + ' ' + " ".join(final_list_raster)) remove_tmp_dir_for_outfile(raster_out)
def REMA_strip_r1_1(dir_REMA, tile_name, dem_dir_out, mosaic_coreg_segm=True, filter_params=None, format_out='GTiff', tgt_EPSG=3031, tgt_res=None, nodata_out=-9999, interp_method=None, geoid=False, rm_tar=False, downsample=True): print('--REMA release 1.1 strip processing--') # 1/ LOCATE STRIPS print('Searching for tile ' + tile_name + ' in folder ' + dir_REMA + '...') subtile_dir = os.path.join(dir_REMA, tile_name) seg_tar_gz_list = [ os.path.join(subtile_dir, tar_file) for tar_file in os.listdir(subtile_dir) if tar_file.endswith('.tar.gz') ] print('Found ' + str(len(seg_tar_gz_list)) + ' segments in tile folder.') # 2/ EXTRACT ALL STRIPS tmp_dir = create_tmp_dir_for_outfile( os.path.join(dem_dir_out, 'all_strips')) list_tmp_dem = [ os.path.join( tmp_dir, os.path.splitext( os.path.splitext(os.path.basename(seg_tar_gz))[0])[0] + '_dem.tif') for seg_tar_gz in seg_tar_gz_list ] for seg_tar_gz in seg_tar_gz_list: print('Extracting dem file of segment ' + str(seg_tar_gz_list.index(seg_tar_gz) + 1) + ' out of ' + str(len(seg_tar_gz_list))) extract_file_from_tar_gz( seg_tar_gz, os.path.splitext( os.path.splitext(os.path.basename(seg_tar_gz))[0])[0] + '_dem.tif', list_tmp_dem[seg_tar_gz_list.index(seg_tar_gz)]) if rm_tar: # once extracted, remove original data to avoid taking up too much space shutil.rmtree(subtile_dir) if downsample: list_ds = [] for tmp_dem in list_tmp_dem: tmp_dem_ds = os.path.join( tmp_dir, os.path.splitext(os.path.basename(tmp_dem))[0] + '_ds.tif') list_ds.append(tmp_dem_ds) translate(tmp_dem, tmp_dem_ds, format_out='GTiff', tgt_res=[8, -8], interp_method='bilinear') os.remove(tmp_dem) list_tmp_dem = list_ds # 2.5/ MOSAIC/COREGISTER SETSM SEGMENTS ITERATIVELY INTO STRIPS if mosaic_coreg_segm: max_coreg_rmse = 10. nb_init_pixel_min = 1000 if not os.path.exists(os.path.join(dem_dir_out, 'coreg_results')): os.mkdir(os.path.join(dem_dir_out, 'coreg_results')) if not os.path.exists( os.path.join(dem_dir_out, 'coreg_results', 'good')): os.mkdir(os.path.join(dem_dir_out, 'coreg_results', 'good')) if not os.path.exists( os.path.join(dem_dir_out, 'coreg_results', 'breaks')): os.mkdir(os.path.join(dem_dir_out, 'coreg_results', 'breaks')) summary = os.path.join(dem_dir_out, 'mosaic_coreg_summary_' + tile_name + '.csv') with open(summary, 'w') as csvfile: writer = csv.writer(csvfile, delimiter=',') writer.writerow([ 'scene', 'segment', 'master', 'dx', 'dy', 'dz', 'RMSE', 'mosaic_nb' ]) # create segment lists list_seg = list(list_tmp_dem) list_seg_id = [] for seg in list_seg: seg_id_split = os.path.basename(seg).split('_') seg_id = '_'.join(seg_id_split[0:5]) seg_suf = '_'.join(seg_id_split[6:]) # seg_nb=int(seg_id_split[5][3:]) if seg_id not in list_seg_id: list_seg_id.append(seg_id) # list all list of scenes acquired the same day list_list_by_seg_id = [] for seg_id in list_seg_id: tmp_list = [seg for seg in list_seg if seg_id in seg] tmp_list = sorted( tmp_list, key=lambda x: x.split('/')[-1].split('_')[5][3:].zfill(2)) list_list_by_seg_id.append(tmp_list) # coregister in series list_final_strip = [] print('List of list of segments:') print(list_list_by_seg_id) # loop through same day scenes for list_by_seg_id in list_list_by_seg_id: print(list_by_seg_id) # bother to coreg only if there is more than 1 segment if len(list_by_seg_id) > 1: tmp_dir_coreg = create_tmp_dir_for_outfile( os.path.join(tmp_dir, 'coreg')) # master dem is the first segment dem_master = list_by_seg_id[0] k = 0 # for csv summary: first segment dem_master_id_split = os.path.basename(dem_master).split( '_') dem_master_id = '_'.join(dem_master_id_split[0:5]) dem_master_seg_nb = int(dem_master_id_split[5][3:]) writer.writerow([ dem_master_id, dem_master_seg_nb, dem_master_seg_nb, 'first_seg', 'first_seg', 'first_seg', 'first_seg', str(k + 1) ]) for i in range(len(list_by_seg_id) - 1): dem_in = list_by_seg_id[i + 1] print('Master segment is ' + dem_master) print('Slave segment is ' + dem_in) # for csv summary: all segments dem_in_id_split = os.path.basename(dem_in).split('_') dem_in_id = '_'.join(dem_in_id_split[0:5]) dem_in_seg_nb = int(dem_in_id_split[5][3:]) dem_master_id_split = os.path.basename( dem_master).split('_') dem_master_seg_nb = int(dem_master_id_split[5][3:]) k0 = k # deriving intersection extent = inters_raster(dem_in, dem_master) nb_init_pixel = valid_points_coreg(dem_master, dem_in) # try coreg only if intersection is non void if extent is not None and nb_init_pixel > nb_init_pixel_min: tmp_dir_seg = create_tmp_dir_for_outfile(dem_in) # do coreg, save results try: out_offs, final_rmse = dem_coregistration_custom( dem_master, dem_in, outdir=tmp_dir_seg)[2:4] except Exception: out_offs = None final_rmse = 999 if final_rmse < max_coreg_rmse: str_out = 'good' else: str_out = 'breaks' shutil.move( os.path.join(tmp_dir_seg, 'CoRegistration_Results.pdf'), os.path.join( dem_dir_out, 'coreg_results', str_out, os.path.splitext( os.path.basename(dem_in))[0] + '_' + os.path.splitext( os.path.basename(dem_master))[0] + '_coreg_results.pdf')) shutil.move( os.path.join(tmp_dir_seg, 'coreg_params.txt'), os.path.join( dem_dir_out, 'coreg_results', str_out, os.path.splitext( os.path.basename(dem_in))[0] + '_' + os.path.splitext( os.path.basename(dem_master))[0] + '_coreg_params.txt')) print('Final coregistration RMSE is:' + str(final_rmse)) # if rmse is good, keep doing mosaics with next segment: mosaic is new master if final_rmse < max_coreg_rmse: print( 'All good. Doing mosaic and iterating to next segment...' ) tmp_mosaic_list = [ dem_master, os.path.join( tmp_dir_seg, os.path.splitext( os.path.basename(dem_in))[0] + '_adj.tif') ] tmp_dem_coreg = os.path.join( tmp_dir_coreg, os.path.basename(list_by_seg_id[i + 1])) dem_mosaic(tmp_mosaic_list, tmp_dem_coreg, erode_length=0, hole_fill_length=0, blending_method='mean') dem_master = tmp_dem_coreg # if this is the last segment, save mosaic and add to list of finalized mosaics if i == (len(list_by_seg_id) - 2): k = k + 1 final_strip = os.path.join( tmp_dir, list_seg_id[list_list_by_seg_id.index( list_by_seg_id)] + '_mosaic' + str(k) + '_' + seg_suf) shutil.copyfile(dem_master, final_strip) list_final_strip.append(final_strip) # otherwise, break the iterative mosaic-ing else: # if this is the last segment if i == (len(list_by_seg_id) - 2): # if master is the previous segment alone if dem_master == list_by_seg_id[i]: print( 'RMSE is larger than limit of ' + str(max_coreg_rmse) + 'm: saving last segments independently...' ) list_final_strip.append(dem_in) list_final_strip.append(dem_master) # if master is a previous ongoing mosaic else: k = k + 1 final_strip = os.path.join( tmp_dir, list_seg_id[list_list_by_seg_id. index(list_by_seg_id)] + '_mosaic' + str(k) + '_' + seg_suf) shutil.copyfile( dem_master, final_strip) print( 'RMSE is larger than limit of ' + str(max_coreg_rmse) + 'm: saving previous mosaic and last segment...' ) list_final_strip.append(dem_in) list_final_strip.append(final_strip) # if there is still segments to go else: # save current mosaic k = k + 1 final_strip = os.path.join( tmp_dir, list_seg_id[list_list_by_seg_id.index( list_by_seg_id)] + '_mosaic' + str(k) + '_' + seg_suf) shutil.copyfile(dem_master, final_strip) print( 'RMSE is larger than limit of ' + str(max_coreg_rmse) + 'm: saving previous mosaic, slave is new master...' ) list_final_strip.append(final_strip) # new master is the segment that did not mosaic dem_master = dem_in remove_tmp_dir_for_outfile(dem_in) # if no intersection and last segment elif i == (len(list_by_seg_id) - 2): out_offs = None final_rmse = None if dem_master == list_by_seg_id[i]: print( 'No intersection between master and slave: saving last segments independently...' ) list_final_strip.append(dem_in) list_final_strip.append(dem_master) # if master is a previous ongoing mosaic else: k = k + 1 final_strip = os.path.join( tmp_dir, list_seg_id[list_list_by_seg_id.index( list_by_seg_id)] + '_mosaic' + str(k) + '_' + seg_suf) shutil.copyfile(dem_master, final_strip) print( 'No intersection between master and slave: saving previous mosaic and last segment...' ) list_final_strip.append(dem_in) list_final_strip.append(final_strip) # if no intersection and still segments to go else: out_offs = None final_rmse = None # save current mosaic k = k + 1 final_strip = os.path.join( tmp_dir, list_seg_id[list_list_by_seg_id.index( list_by_seg_id)] + '_mosaic' + str(k) + '_' + seg_suf) shutil.copyfile(dem_master, final_strip) print( 'No intersection between master and slave: saving previous mosaic, slave is new master...' ) list_final_strip.append(final_strip) # new master is the segment that did not mosaic dem_master = dem_in # write data in summary if out_offs is None: dx = 'no_inters' dy = 'no_inters' dz = 'no_inters' rmse = 'no_inters' else: dx = str(out_offs[0]) dy = str(out_offs[1]) dz = str(out_offs[2]) rmse = str(final_rmse) writer.writerow([ dem_in_id, dem_in_seg_nb, dem_master_seg_nb, dx, dy, dz, rmse, str(k0 + 1) ]) remove_tmp_dir_for_outfile(os.path.join(tmp_dir, 'coreg')) else: list_final_strip.append(list_by_seg_id[0]) dem_master_id_split = os.path.basename( list_by_seg_id[0]).split('_') dem_master_id = '_'.join(dem_master_id_split[0:5]) dem_master_seg_nb = int(dem_master_id_split[5][3:]) writer.writerow([ dem_master_id, dem_master_seg_nb, dem_master_seg_nb, 'only_seg', 'only_seg', 'only_seg', 'only_seg', 'only_seg' ]) else: list_final_strip = list_tmp_dem # 3/ PROCESSING OF FINAL STRIPS final_dem = [] for final_strip in list_final_strip: # 3.1/ FILTER STRIPS if filter_params is not None: sys.exit('No filter pre-defined for this DEM product.') # 3.2/ REPROJECT STRIPS # raw data is GeoTiff, 3413, 1 arc-sec and -9999 nodata_out if format_out == 'GTiff' and tgt_EPSG == 3031 and tgt_res is None and nodata_out is -9999: tmp_dem_proj = final_strip else: tmp_dem_proj = os.path.join( tmp_dir, os.path.splitext(os.path.basename(final_strip))[0] + '_proj.tif') warp_defaultUTM(final_strip, tmp_dem_proj, format_out, 3031, tgt_EPSG, tgt_res, nodata_out, interp_method) # 3.3/ ELLIPSOID OR GEOID STRIPS # raw data is ellipsoid WGS84 if geoid: tmp_dem_geoid = os.path.join( tmp_dir, os.path.splitext(os.path.basename(final_strip))[0] + '_geoid.tif') ellipsoid_to_geoid(tmp_dem_proj, tmp_dem_geoid) else: tmp_dem_geoid = tmp_dem_proj final_dem.append(tmp_dem_geoid) for dem in final_dem: shutil.copyfile(dem, os.path.join(dem_dir_out, os.path.basename(dem))) remove_tmp_dir_for_outfile(os.path.join(dem_dir_out, 'all_strips')) print('Fin.')
def ASTGTM2_V002_tile(dir_ASTGTM2, tile_name, dem_out, filter_params=None, format_out='GTiff', tgt_EPSG=4326, tgt_res=None, nodata_out=-9999, interp_method=None, geoid=True): """ :param dir_ASTGTM2: path to directory directly containing zip of ASTGTM2 :param tile_name: 1x1° tile name (SRTMGL1/classic naming convention) :param dem_out: path to DEM out file :param filter_params: filtering with ASTGTM2 NUM file using rastlib.filter_nanarray function :param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html :param tgt_EPSG: EPSG of output projection :param tgt_res: output resolution, GDAL naming [xres, yres] :param nodata_out: output no-data value :param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc.. :param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid :return: ASTGTM2: ASTER Global Digital Elevation Model V002 from NASA at: https://lpdaac.usgs.gov/node/1079 NUM file description Fill value: -1 to -11 -1 SRTM3 V003 -2 SRTM3 V002 -5 NED -6 CDED -11 Alaska DEM Valid range (ASTER count): 0 to 200 OPTIMAL DIRECTORY ARCHITECTURE: ASTGTM2 zip archives in the same directory """ #1/ LOCATE 1x1° tile #nomenclature of ASTGTM2 zip files tile_zip = os.path.join(dir_ASTGTM2, 'ASTGTM2_' + tile_name + '.zip') dem_file = 'ASTGTM2_' + tile_name + '_dem.tif' num_file = 'ASTGTM2_' + tile_name + '_num.tif' #2/ EXTRACT TILE tmp_dir = create_tmp_dir_for_outfile(dem_out) tmp_dem = os.path.join(tmp_dir, dem_file) tmp_num = os.path.join(tmp_dir, num_file) extract_file_from_zip(tile_zip, dem_file, tmp_dem) extract_file_from_zip(tile_zip, num_file, tmp_num) #3/ FILTER TILE if filter_params is not None: num = read_nanarray(tmp_num) _, filt = filter_nanarray(num, filter_params[0], filter_params[1], filter_params[2]) dem = read_nanarray(tmp_dem) dem_filtered = np.array(dem) dem_filtered[filt] = np.NaN update_nanarray(tmp_dem, dem_filtered) #4/ REPROJECT TILE # raw data is GTiff, 4326, 1 arc-sec and -32768 nodata_out if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -9999: tmp_dem_proj = tmp_dem else: tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif') warp_defaultUTM(tmp_dem, tmp_dem_proj, format_out, 4326, tgt_EPSG, tgt_res, nodata_out, interp_method) #5/ ELLIPSOID OR GEOID # raw data is geoid EGM96 if not geoid: geoid_to_ellipsoid(tmp_dem_proj, dem_out) else: shutil.move(tmp_dem_proj, dem_out) remove_tmp_dir_for_outfile(dem_out)
def SRTMGL1_V003_tile(dir_SRTMGL1, tile_name, dem_out, filter_params=None, format_out='SRTMHGT', tgt_EPSG=4326, tgt_res=None, nodata_out=-32768, interp_method=None, geoid=True): """ :param dir_SRTMGL1: path to directory directly containing zip of SRTMGL1 and SRTMGL1N :param tile_name: 1x1° tile name (SRTMGL1/classic naming convention) :param dem_out: path to DEM out file :param filter_params: filtering with SRTMGL1 NUM file using rastlib.filter_nanarray function :param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html :param tgt_EPSG: EPSG of output projection :param tgt_res: output resolution, GDAL naming [xres, yres] :param nodata_out: output no-data value :param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc.. :param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid :return: SRTMGL1: Shuttle Radar Topography Mission Global 1-arcsec V003 from NASA at: https://lpdaac.usgs.gov/sites/default/files/public/measures/docs/NASA_SRTM_V3.pdf NUM file description 1 = Water-masked SRTM void * 2 = Water-masked SRTM non-void * 5 = GDEM elevation = 0 in SRTM void (helped correct ocean masking) 11 = NGA-interpolated SRTM (were very small voids in SRTMv1) 21 = GMTED2010 oversampled from 7.5 arc-second postings 25 = SRTM within GDEM ** 31 = NGA fill of SRTM via GDEM*** 51 = USGS NED 52 = USGS NED via GDEM 53 = Alaska USGS NED via GDEM 72 = Canadian Digital Elevation Data (CDED) via GDEM 101-200 = ASTER scene count (count limited to 100) 201-224 = SRTM swath count (non-voided swaths) Actual maximum = 24 OPTIMAL DIRECTORY ARCHITECTURE: SRTMGL1 and SRTMGL1N zip archives in the same directory """ #1/ LOCATE 1x1° tile #nomenclature of SRTMGL1 and SRTMGL1N zip files hgt_zip = os.path.join(dir_SRTMGL1, tile_name + '.SRTMGL1.hgt.zip') num_zip = os.path.join( dir_SRTMGL1, tile_name + '.SRTMGL1N.num.zip' ) #NOTE: nomenclature of V002 was SRTMGL1.num.zip for the num file #2/ EXTRACT TILE tmp_dir = create_tmp_dir_for_outfile(dem_out) zip_ref = zipfile.ZipFile(hgt_zip, 'r') zip_ref.extractall(tmp_dir) zip_ref.close() zip_ref = zipfile.ZipFile(num_zip, 'r') zip_ref.extractall(tmp_dir) zip_ref.close() tmp_hgt = os.path.join(tmp_dir, tile_name + '.hgt') tmp_num = os.path.join(tmp_dir, tile_name + '.num') #3/ FILTER TILE if filter_params is not None: num = read_nanarray(tmp_num) _, filt = filter_nanarray(num, filter_params[0], filter_params[1], filter_params[2]) hgt = read_nanarray(tmp_hgt) hgt_filtered = np.array(hgt) hgt_filtered[filt] = np.NaN update_nanarray(tmp_hgt, hgt_filtered) #4/ REPROJECT TILE # raw data is SRTMHGT, 4326, 1 arc-sec and -32768 nodata_out if format_out == 'SRTMHGT' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32768: tmp_hgt_proj = tmp_hgt else: tmp_hgt_proj = os.path.join(tmp_dir, tile_name + '_proj.tif') warp_defaultUTM(tmp_hgt, tmp_hgt_proj, format_out, 4326, tgt_EPSG, tgt_res, nodata_out, interp_method) #5/ ELLIPSOID OR GEOID # raw data is geoid EGM96 if not geoid: geoid_to_ellipsoid(tmp_hgt_proj, dem_out) else: shutil.move(tmp_hgt_proj, dem_out) remove_tmp_dir_for_outfile(dem_out)
def AW3D30_v2_1_tile(dir_ALOS, tile_name, dem_out, bit_params=None, format_out='GTiff', tgt_EPSG=4326, tgt_res=None, nodata_out=-9999, interp_method=None, geoid=True): """ :param dir_ALOS: path to directory directly containing ALOS World 3D 30-m 5x5° tar.gz archives :param tile_name: 1x1° tile name (SRTMGL1/classic naming convention) :param dem_out: path to DEM out file :param bit_params: bit filtering for AW3D30 MSK file ; see rastlib.mask_bitarray function for an example :param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html :param tgt_EPSG: EPSG of output projection :param tgt_res: output resolution, GDAL naming [xres, yres] :param nodata_out: output no-data value :param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc.. :param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid :return ALOS World 3D-30m (AW3D30) Version 2.1 from JAXA at: https://www.eorc.jaxa.jp/ALOS/en/aw3d30/aw3d30v21_format_e.pdf MSK file: 0000: Valid 0001: Cloud and show mask (invalid) 0010: Land water and low correlation mask of the 5 m resolution DSM (valid) 0011: Sea mask (valid) 0100: National Land Numerical Information 10 m DEM (by Geographical Survey Institute of Japan) (valid) 1000: Shuttle Radar Topography Mission (SRTM) SRTM-1 Version 3 (valid) 1100: PRISM DSM (valid) STK, HDR, QAI, LST files not used in this function, could be extracted same manner as other in 2/ OPTIMAL DIRECTORY ARCHITECTURE: all 5x5 .tar.gz archives in the same directory """ #1/ LOCATE 1x1° TILE #ALOS AW3D30 v2.1 5x5 archive naming convention def latlon_to_ALOS_5x5tile_naming(lat, lon): lat_5 = np.floor(lat / 5) * 5 lon_5 = np.floor(lon / 5) * 5 if lat_5 >= 0: str_lat = 'N' else: str_lat = 'S' if lon_5 >= 0: str_lon = 'E' else: str_lon = 'W' tile_name_5 = str_lat + str(lat_5).zfill(3) + str_lon + str( lon_5).zfill(3) return tile_name_5 #identify in which 5x5 tar.gz the 1x1 tile is located lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name) name_5x5 = latlon_to_ALOS_5x5tile_naming(lat_tile, lon_tile) list_5x5 = os.listdir(dir_ALOS) list_5x5_prefix = [l[0:8] for l in list_5x5] if name_5x5 in list_5x5_prefix: tar_5x5 = list_5x5[list_5x5_prefix.index(name_5x5)] else: sys.exit( 'Could not find an ALOS AW3D30 5x5° archive containing the tile ' + tile_name + ' in the directory ' + dir_ALOS) #2/ EXTRACT TILE tmp_dir = create_tmp_dir_for_outfile(dem_out) alos_tile_name = tile_name[0:1] + str(0) + tile_name[ 1:] #SRTMGL1 naming convention to ALOS 1x1 naming convention (3 digits for latitude) #extract files of interest dsm_file = alos_tile_name + '_AVE_DSM.tif' msk_file = alos_tile_name + '_AVE_MSK.tif' tmp_dsm = os.path.join(tmp_dir, dsm_file) tmp_msk = os.path.join(tmp_dir, msk_file) extract_file_from_tar_gz(os.path.join(dir_ALOS, tar_5x5), dsm_file, tmp_dsm) extract_file_from_tar_gz(os.path.join(dir_ALOS, tar_5x5), msk_file, tmp_msk) #3/ FILTER TILE mask = read_nanarray(tmp_msk) #no filtering by default if bit_params is not None: mask_goodval = mask_bitarray(mask, bit_params[0], bit_params[1]) dsm = read_nanarray(tmp_dsm) dsm_filtered = np.array(dsm) dsm_filtered[mask_goodval] = np.NaN update_nanarray(tmp_dsm, dsm_filtered) #4/ REPROJECT TILE: #raw data is GeoTiff, 4326, 1 arc-sec and -9999 nodata_out if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -9999: tmp_dsm_proj = tmp_dsm else: tmp_dsm_proj = os.path.join(tmp_dir, alos_tile_name + '_proj.tif') warp_defaultUTM(tmp_dsm, tmp_dsm_proj, format_out, 4326, tgt_EPSG, tgt_res, nodata_out, interp_method) #5/ ELLIPSOID OR GEOID #raw data is geoid EGM96 if not geoid: geoid_to_ellipsoid(tmp_dsm_proj, dem_out) else: shutil.move(tmp_dsm_proj, dem_out) remove_tmp_dir_for_outfile(dem_out)
def REMA_mosaic_r1_1_tile(dir_REMA, tile_name, dem_out, filter_params=None, format_out='GTiff', tgt_EPSG=3031, tgt_res=None, nodata_out=-9999, interp_method=None, geoid=False, tag_lonlat_tile=False, path_tile_index=None, tag_merge=False, tag_clip=False): """ :param dir_REMA: path to parent directory "8m" containing subdirectories of tar.gz archives (native FTP architecture) :param tile_name: either REMA tile name or 1x1° lat/lon tile name (SRTMGL1/classic naming convention) :param dem_out: path to DEM out file :param filter_params: filtering with REMA ERR file using rastlib.filter_nanarray function :param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html :param tgt_EPSG: EPSG of output projection :param tgt_res: output resolution, GDAL naming [xres, yres] :param nodata_out: output no-data value :param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc.. :param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid :param tag_lonlat_tile: True if tile_name follows SRTMGL1 tile naming, False if tile_name follows REMA tile naming :param path_tile_index: if tile_name is REMA format, specify path to ESRI REMA Tile Index :param tag_merge: if tile_name is REMA format, True to merge all ArcticDEM tiles to the 1x1° lat/lon extent :param tag_clip: if tile_name is REMA format, True to clip the 5x5° tile to the 1x1° lat/lon extent of tile_name :return: REMA release 1.1 product: ref:https://www.pgc.umn.edu/data/rema/ Processing for 8m mosaic (100m, 500m and 1km versions are bundled in one .tif file) Tile name and processing is REMA tile naming convention by default Provide path to ESRI tile index file to use 1x1° lat/lon tiles and SRTMGL1 naming convention OPTIMAL DIRECTORY ARCHITECTURE: point to "8m" folder of similar architecture than: ftp://ftp.data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.0 """ # 1/ LOCATE TILE if not tag_lonlat_tile: subtile_dir = os.path.join(dir_REMA, tile_name) tile_tar_gz_list = [ os.path.join(subtile_dir, tar_file) for tar_file in os.listdir(subtile_dir) if tar_file.endswith('.tar.gz') ] else: lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name) extent = [lat_tile, lon_tile, lat_tile + 1, lon_tile + 1] # feature name in REMA_Tile_Index_Rel1.1 feat_name = 'TILE' subtile_name_list = list_shp_field_inters_extent( path_tile_index, feat_name, extent, 4326) subtile_dir_list = [ os.path.join(dir_REMA, tile) for tile in subtile_name_list ] tile_tar_gz_list = [] for i in range(len(subtile_dir_list)): tile_tar_gz_list = tile_tar_gz_list + [ os.path.join(subtile_dir_list[i], tar_file) for tar_file in os.listdir(subtile_dir_list[i]) if tar_file.endswith('.tar.gz') ] # 2/ EXTRACT TILE tmp_dir = create_tmp_dir_for_outfile(dem_out) list_tmp_dem = [ os.path.join( tmp_dir, os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_dem.tif') for tile_tar_gz in tile_tar_gz_list ] for tile_tar_gz in tile_tar_gz_list: extract_file_from_tar_gz( tile_tar_gz, os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_dem.tif', list_tmp_dem[tile_tar_gz_list.index(tile_tar_gz)]) # list_tmp_err = [tmp_dir + os.path.splitext(os.path.basename(tile_tar_gz))[0]+'_err.tif' for tile_tar_gz in tile_tar_gz_list] for tile_tar_gz in tile_tar_gz_list: extract_file_from_tar_gz( tile_tar_gz, os.path.splitext(os.path.basename(tile_tar_gz))[0] + '_err.tif', list_tmp_dem[tile_tar_gz_list.index(tile_tar_gz)]) list_tmp_dem_tomerge = [] for tmp_dem in list_tmp_dem: # 3/ FILTER TILE if filter_params is not None: tmp_err = tmp_dem[:-8] + '_err.tif' err = read_nanarray(tmp_err) _, filt = filter_nanarray(err, filter_params[0], filter_params[1], filter_params[2]) dem = read_nanarray(tmp_dem) dem_filtered = np.array(dem) dem_filtered[filt] = np.NaN update_nanarray(tmp_dem, dem_filtered) # 4/ REPROJECT TILE # raw data is GeoTiff, 3031, 1 arc-sec and -9999 nodata_out if format_out == 'GTiff' and tgt_EPSG == 3031 and tgt_res is None and nodata_out is -9999: tmp_dem_proj = tmp_dem else: tmp_dem_proj = os.path.join( tmp_dir, os.path.splitext(os.path.basename(tmp_dem))[0] + '_proj.tif') warp_defaultUTM(tmp_dem, tmp_dem_proj, format_out, 3031, tgt_EPSG, tgt_res, nodata_out, interp_method) # 5/ ELLIPSOID OR GEOID # raw data is ellipsoid WGS84 if geoid: tmp_dem_geoid = os.path.join( tmp_dir, os.path.splitext(os.path.basename(tmp_dem))[0] + '_geoid.tif') ellipsoid_to_geoid(tmp_dem_proj, tmp_dem_geoid) else: tmp_dem_geoid = tmp_dem_proj list_tmp_dem_tomerge.append(tmp_dem_geoid) # 6/ MERGE ALL TILES tmp_dem_merged = os.path.join(tmp_dir, tile_name + '_merged.tif') if tag_merge: merge_rast_list(list_tmp_dem_tomerge, tmp_dem_merged) else: shutil.copytree( tmp_dir, os.path.join(os.path.dirname(dem_out), tile_name + '_subtiles')) # 7/ CLIP TO TILE EXTENT if not tag_clip: tmp_dem_clipped = os.path.join(tmp_dir, tile_name + '_clipped.tif') lat, lon = SRTMGL1_naming_to_latlon(tile_name) clip_rast_to_extent(tmp_dem_merged, tmp_dem_clipped, [lat, lon, lat + 1, lon + 1], 4326) else: tmp_dem_clipped = tmp_dem_merged shutil.move(tmp_dem_clipped, dem_out) remove_tmp_dir_for_outfile(dem_out)
def SRTMv4_1_tile(dir_SRTMv4_1, tile_name, dem_out, tag_clip=False, filter_params=None, format_out='GTiff', tgt_EPSG=4326, tgt_res=None, nodata_out=-32768, interp_method=None, geoid=True): """ :param dir_SRTMv4_1: path to parent directory containing "6_5x5_TIFS" directory of zip archives (bulk downloading) :param tile_name: 1x1° tile name (SRTMGL1/classic naming convention) :param dem_out: path to DEM out file :param tag_clip: True to clip the 5x5° tile to the 1x1° extent of tile_name :param filter_params: no filtering for this product :param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html :param tgt_EPSG: EPSG of output projection :param tgt_res: output resolution, GDAL naming [xres, yres] :param nodata_out: output no-data value :param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc.. :param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid :return: SRTM 90m Digital Elevation Database v4.1 by NASA/CGIARCSI at: https://cgiarcsi.community/data/srtm-90m-digital-elevation-database-v4-1/ Product Description temporarily not available (web issue) OPTIMAL DIRECTORY ARCHITECTURE: bulk download format, point to the directoy containing 6_5x5_TIFS directory """ def latlon_to_SRTMv4_1_5x5tile_naming(lat, lon): ind_lon = np.floor(lon + 180 / 5) + 1 ind_lat = np.floor(lat + 90 / 5) + 1 return ind_lat, ind_lon # 1/ LOCATE 5x5° tile #read lat, lon according to SRTMv4.1 naming convention lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name) ind_lat_4_1, ind_lon_4_1 = latlon_to_SRTMv4_1_5x5tile_naming( lat_tile, lon_tile) # nomenclature of SRTMv4.1 archives tile_zip = os.path.join( dir_SRTMv4_1, '6_5x5_TIFS', 'srtm_' + str(ind_lon_4_1).zfill(2) + '_' + str(ind_lat_4_1).zfill(2) + '.zip') # 2/ EXTRACT TILE tmp_dir = create_tmp_dir_for_outfile(dem_out) dem_file = 'srtm_' + str(ind_lon_4_1).zfill(2) + '_' + str( ind_lat_4_1).zfill(2) + '.tif' tmp_dem = os.path.join(tmp_dir, dem_file) extract_file_from_zip(tile_zip, dem_file, tmp_dem) #2.5/ CLIP TO TILE EXTENT if not tag_clip: tmp_dem_clipped = os.path.join( tmp_dir, 'srtm_' + str(ind_lon_4_1).zfill(2) + '_' + str(ind_lat_4_1).zfill(2) + '_clipped.tif') clip_rast_to_extent(tmp_dem, tmp_dem_clipped, [lat_tile, lon_tile, lat_tile + 1, lon_tile + 1], 4326) else: tmp_dem_clipped = tmp_dem # 3/ FILTER TILE if filter_params is not None: sys.exit('No filter pre-defined for this DEM product.') # 4/ REPROJECT TILE # raw data is GeoTiff, 4326, 3 arc-sec and -32768 nodata_out if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32768 and tag_clip: tmp_dem_proj = tmp_dem_clipped else: tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif') warp_defaultUTM(tmp_dem_clipped, tmp_dem_proj, format_out, 4326, tgt_EPSG, tgt_res, nodata_out, interp_method) # 5/ ELLIPSOID OR GEOID # raw data is geoid EGM96 if not geoid: geoid_to_ellipsoid(tmp_dem_proj, dem_out) else: shutil.move(tmp_dem_proj, dem_out) remove_tmp_dir_for_outfile(dem_out)
def TDX_90m_tile(dir_TDX, tile_name, dem_out, filter_params=None, format_out='GTiff', tgt_EPSG=4326, tgt_res=None, nodata_out=-32767, interp_method=None, geoid=False): """ :param dir_TDX: path to directory directly containing TanDEM-X zip archives :param tile_name: 1x1° tile name (SRTMGL1/classic naming convention) :param dem_out: path to DEM out file :param filter_params: filtering with TDX HEM file ; see rastlib.filter_nanarray function for an example :param format_out: output format, GDAL naming (e.g.: 'GTiff','HDF4', ...) ; see: https://www.gdal.org/formats_list.html :param tgt_EPSG: EPSG of output projection :param tgt_res: output resolution, GDAL naming [xres, yres] :param nodata_out: output no-data value :param interp_method: resampling method, GDAL naming 'bilinear', 'neir', 'cubic', etc.. :param geoid: True, converts to geoid if is ellipsoid; False converts to ellipsoid if is geoid :return: TanDEM-X 90m product from DLR at: https://geoservice.dlr.de/web/dataguide/tdm90/pdfs/TD-GS-PS-0021_DEM-Product-Specification.pdf Only HEM (Height Error Map) is extracted here Using the same approach with binary/nanarray filters, 2/ can extract and 3/ can process the AMP, AM2, WAM, COV, COM and LSM files OPTIMAL DIRECTORY ARCHITECTURE: all .zip archives in the same directory """ # 1/ LOCATE 1x1° TILE def latlon_to_TDX_tile_naming(lat, lon): if lat >= 80: lon_sw = np.floor(lon / 4) * 4 elif lat >= 60: lon_sw = np.floor(lon / 2) * 2 else: lon_sw = lon lat_sw = lat if lat_sw >= 0: str_lat = 'N' else: str_lat = 'S' if lon_sw >= 0: str_lon = 'E' else: str_lon = 'W' tile_name_tdx = str_lat + str(int( abs(lat_sw))).zfill(2) + str_lon + str(int(abs(lon_sw))).zfill(3) return tile_name_tdx #identify in which TDX tile the 1x1 tile is located lat_tile, lon_tile = SRTMGL1_naming_to_latlon(tile_name) name_tdx = latlon_to_TDX_tile_naming(lat_tile, lon_tile) list_tdx = os.listdir(dir_TDX) list_prefix = [l[13:20] for l in list_tdx] if name_tdx in list_prefix: pass # zip_tdx=list_prefix[list_prefix.index(name_tdx)] else: sys.exit('Could not find a TDX tile archive containing the tile ' + name_tdx + ' in the directory ' + dir_TDX) tile_name = name_tdx # nomenclature of TDX 3 arc-sec files tile_zip = os.path.join(dir_TDX, 'TDM1_DEM__30_' + name_tdx + '.zip') dem_out = dem_out + '_' + name_tdx + '.tif' # 2/ EXTRACT TILE tmp_dir = create_tmp_dir_for_outfile(dem_out) dem_file = 'TDM1_DEM__30_' + tile_name + '_DEM.tif' hem_file = 'TDM1_DEM__30_' + tile_name + '_HEM.tif' tmp_dem = os.path.join(tmp_dir, dem_file) tmp_hem = os.path.join(tmp_dir, hem_file) extract_file_from_zip(tile_zip, dem_file, tmp_dem) extract_file_from_zip(tile_zip, hem_file, tmp_hem) # 3/ FILTER TILE if filter_params is not None: hem = read_nanarray(tmp_hem) _, filt = filter_nanarray(hem, filter_params[0], filter_params[1], filter_params[2]) dem = read_nanarray(tmp_dem) dem_filtered = np.array(dem) dem_filtered[filt] = np.NaN update_nanarray(tmp_dem, dem_filtered) # 4/ REPROJECT TILE # raw data is GTiff, 4326, 3 arc-sec and -32768 nodata_out if format_out == 'GTiff' and tgt_EPSG == 4326 and tgt_res is None and nodata_out is -32767: tmp_dem_proj = tmp_dem else: tmp_dem_proj = os.path.join(tmp_dir, tile_name + '_proj.tif') warp_defaultUTM(tmp_dem, tmp_dem_proj, format_out=format_out, src_EPSG=4326, tgt_EPSG=tgt_EPSG, tgt_res=tgt_res, nodata_in=-32767, nodata_out=nodata_out, interp_method=interp_method) # 5/ ELLIPSOID OR GEOID # raw data is ellipsoid WGS84 if geoid: ellipsoid_to_geoid(tmp_dem_proj, dem_out) else: shutil.move(tmp_dem_proj, dem_out) remove_tmp_dir_for_outfile(dem_out)