def process_arcticDEM_tiles(tar_list,
save_dir,
inter_format,
resample_method,
o_res,
extent_poly,
extent_id,
pre_name,
b_rm_inter=True):
'''
process the mosaic (not multi-temporal) version of ArcticDEM
:param tar_list:
:param save_dir:
:param inter_format:
:param resample_method:
:param o_res: output resolution
:param extent_poly: extent polygons, in the same projection of ArcticDEM
:param extent_id: extent id
:param pre_name:
:param b_rm_inter:
:return:
'''
# unpackage and crop to extent
dem_tif_list, tar_folders = process_dem_tarball(tar_list,
save_dir,
inter_format,
o_res,
extent_poly=extent_poly,
poly_id=extent_id,
process_num=4)
if len(dem_tif_list) < 1:
raise ValueError('No DEM extracted from tarballs')
dem_name = os.path.basename(tar_folders[0])[-7:]
save_path = os.path.join(
save_dir,
pre_name + '_' + dem_name + '_ArcticTileDEM_sub_%d.tif' % extent_id)
RSImageProcess.mosaic_crop_images_gdalwarp(
dem_tif_list,
save_path,
resampling_method=resample_method,
o_format=inter_format,
xres=o_res,
yres=o_res,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
# remove intermediate files
if b_rm_inter:
basic.outputlogMessage('remove intermediate files')
for folder in tar_folders:
io_function.delete_file_or_dir(folder)
return True
def create_a_mosaic(pre_name, extent_id, save_dir, extent_poly, tile_list):
# create mosaic
tif_list = []
for idx, tile_num in enumerate(tile_list):
dem_name = tile_num + '_' + res + '_v3.0'
tiff_name = dem_name + '_reg_dem.tif'
tiff_path = os.path.join(dem_tif_dir, tiff_name)
if os.path.isfile(tiff_path) is False:
raise ValueError('%s not exists' % tiff_path)
tif_list.append(tiff_path)
thread_num = 8
output_mosaic = os.path.join(
save_dir, pre_name + '_ArcticDEM_mosaic_%d' % extent_id + '.tif')
# create mosaic, can handle only input one file, but is slow
if os.path.isfile(output_mosaic) is False:
result = RSImageProcess.mosaic_crop_images_gdalwarp(
tif_list,
output_mosaic,
resampling_method='average',
o_format='GTiff',
compress='lzw',
tiled='yes',
bigtiff='if_safer',
thread_num=thread_num)
else:
print('mosaic: %s exist, skip' % output_mosaic)
# crop
output_crop = os.path.join(
save_dir, pre_name + '_ArcticDEM_mosaic_%d_crop' % extent_id + '.tif')
if os.path.isfile(output_crop) is False:
RSImageProcess.subset_image_by_polygon_box_image_min(
output_crop,
output_mosaic,
extent_poly,
resample_m='average',
o_format='GTiff',
xres=32,
yres=32,
compress='lzw',
tiled='yes',
bigtiff='if_safer',
thread_num=thread_num)
else:
print('Crop: %s exist, skip' % output_crop)
def mosaic_dem_list(key, dem_list, save_tif_dir, resample_method, save_source,
o_format):
save_mosaic = os.path.join(save_tif_dir, key + '.tif')
# check file existence
# if os.path.isfile(save_mosaic):
b_save_mosaic = io_function.is_file_exist_subfolder(
save_tif_dir, key + '.tif')
if b_save_mosaic is not False:
basic.outputlogMessage('warning, mosaic file: %s exist, skip' %
b_save_mosaic)
return b_save_mosaic
# mosaic_list.append(b_save_mosaic)
# continue
# save the source file for producing the mosaic
if save_source:
save_mosaic_source_txt = os.path.join(save_tif_dir, key + '_src.txt')
io_function.save_list_to_txt(save_mosaic_source_txt, dem_list)
# if len(demTif_groups[key]) == 1:
# io_function.copy_file_to_dst(demTif_groups[key][0],save_mosaic)
# else:
# # RSImageProcess.mosaics_images(dem_groups[key],save_mosaic)
# RSImageProcess.mosaic_crop_images_gdalwarp(demTif_groups[key],save_mosaic,resampling_method=resample_method,o_format=o_format)
# create mosaic, can handle only input one file
result = RSImageProcess.mosaic_crop_images_gdalwarp(
dem_list,
save_mosaic,
resampling_method=resample_method,
o_format=o_format,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
if result is False:
return False
return save_mosaic
def mosaic_dem_list(key, dem_list, save_tif_dir,resample_method,save_source, o_format, thread_num=1):
# print('\n\n os.fork \n\n', os.fork())
# if os.fork()==0:
# proc_id = multiprocessing.current_process().pid
# basic.setlogfile('log_file_pid_%d.txt'%proc_id)
save_mosaic = os.path.join(save_tif_dir, key + '.tif')
# check file existence
# if os.path.isfile(save_mosaic):
b_save_mosaic = io_function.is_file_exist_subfolder(save_tif_dir, key + '.tif')
if b_save_mosaic is not False:
basic.outputlogMessage('warning, mosaic file: %s exist, skip' % save_mosaic)
return save_mosaic
# mosaic_list.append(b_save_mosaic)
# continue
# save the source file for producing the mosaic
if save_source:
save_mosaic_source_txt = os.path.join(save_tif_dir, key + '_src.txt')
io_function.save_list_to_txt(save_mosaic_source_txt, dem_list)
# if only one dem, then copy it if it's not VRT format
if len(dem_list) == 1:
if raster_io.get_driver_format(dem_list[0]) != 'VRT':
io_function.copy_file_to_dst(dem_list[0], save_mosaic)
return save_mosaic
# create mosaic, can handle only input one file, but is slow
result = RSImageProcess.mosaic_crop_images_gdalwarp(dem_list, save_mosaic, resampling_method=resample_method,
o_format=o_format,
compress='lzw', tiled='yes', bigtiff='if_safer',thread_num=thread_num)
if result is False:
sys.exit(1)
# return False
return save_mosaic
def crop_mosaic_reproject_dem_diff(grid_dem_tifs,
pre_name,
extent_poly,
o_res,
new_prj,
b_mosaic=False):
# crop
crop_tif_list = []
# crop to the same extent
crop_tif_dir = os.path.join('dem_diff_crop')
if os.path.isdir(crop_tif_dir) is False:
io_function.mkdir(crop_tif_dir)
for tif in grid_dem_tifs:
save_crop_path = os.path.join(
crop_tif_dir,
os.path.basename(io_function.get_name_by_adding_tail(tif, 'sub')))
if os.path.isfile(save_crop_path):
basic.outputlogMessage('%s exists, skip cropping' % save_crop_path)
crop_tif_list.append(save_crop_path)
else:
crop_tif = subset_image_by_polygon_box(tif,
save_crop_path,
extent_poly,
resample_m='near',
o_format='VRT',
out_res=o_res,
same_extent=True,
thread_num=2)
if crop_tif is False:
raise ValueError('warning, crop %s failed' % tif)
crop_tif_list.append(crop_tif)
grid_dem_tifs = crop_tif_list
# mosaic
if b_mosaic:
save_mosaic = pre_name + '_DEM_diff.tif'
resample_method = 'average'
# create mosaic, can handle only input one file, but is slow
result = RSImageProcess.mosaic_crop_images_gdalwarp(
grid_dem_tifs,
save_mosaic,
resampling_method=resample_method,
o_format='Gtiff',
compress='lzw',
tiled='yes',
bigtiff='if_safer',
thread_num=2)
if result is False:
sys.exit(1)
grid_dem_tifs = [save_mosaic]
# reproject
for tif in grid_dem_tifs:
t_file = io_function.get_name_by_adding_tail(tif, 'prj')
map_projection.transforms_raster_srs(tif,
new_prj,
t_file,
o_res,
o_res,
resample_m='bilinear',
o_format='GTiff',
compress='lzw',
tiled='yes',
bigtiff='if_safer')
pass
def mask_dem_by_surface_water(crop_dem_list, extent_poly, extent_id, crop_tif_dir, o_res, process_num):
# get list of the ArcticDEM mosaic
water_mask_tifs = io_function.get_file_list_by_ext('.tif',mask_water_dir,bsub_folder=False)
water_mask_ext_polys = get_dem_tif_ext_polygons(water_mask_tifs)
overlap_index = vector_gpd.get_poly_index_within_extent(water_mask_ext_polys,extent_poly)
#### crop and mosaic water mask
sub_mosaic_dem_tifs = [water_mask_tifs[item] for item in overlap_index]
water_mask_crop_tif_list = []
for tif in sub_mosaic_dem_tifs:
save_crop_path = os.path.join(crop_tif_dir, os.path.basename(io_function.get_name_by_adding_tail(tif, 'sub_poly_%d' % extent_id)) )
if os.path.isfile(save_crop_path):
basic.outputlogMessage('%s exists, skip' % save_crop_path)
water_mask_crop_tif_list.append(save_crop_path)
else:
crop_tif = subset_image_by_polygon_box(tif, save_crop_path, extent_poly, resample_m='near',
o_format='VRT',out_res=o_res, same_extent=True,thread_num=process_num) #
if crop_tif is False:
raise ValueError('warning, crop %s failed' % tif)
water_mask_crop_tif_list.append(crop_tif)
if len(water_mask_crop_tif_list) < 1:
basic.outputlogMessage('No water mask for %d grid'%extent_id)
save_id_grid_no_watermask(extent_id)
return None
# create mosaic, can handle only input one file, but is slow
save_water_mask_mosaic = os.path.join(crop_tif_dir, 'global_surface_water_grid%d.tif'%extent_id)
result = RSImageProcess.mosaic_crop_images_gdalwarp(water_mask_crop_tif_list, save_water_mask_mosaic, resampling_method='average',o_format='GTiff',
compress='lzw', tiled='yes', bigtiff='if_safer',thread_num=process_num)
if result is False:
return False
# because the resolution of dem and water mask is different, so we polygonize the watermask, then burn into the dem
water_mask_shp = os.path.join(crop_tif_dir, 'global_surface_water_grid%d.shp'%extent_id)
if os.path.isfile(water_mask_shp):
basic.outputlogMessage('%s exists, skip cropping' % water_mask_shp)
else:
# set 0 as nodata
if raster_io.set_nodata_to_raster_metadata(save_water_mask_mosaic,0) is False:
return False
if vector_gpd.raster2shapefile(save_water_mask_mosaic,water_mask_shp,connect8=True) is None:
return False
# masking the strip version of DEMs
mask_dem_list = []
for idx, strip_dem in enumerate(crop_dem_list):
save_path = io_function.get_name_by_adding_tail(strip_dem, 'maskWater')
if os.path.isfile(save_path):
basic.outputlogMessage('%s exist, skip'%save_path)
mask_dem_list.append(save_path)
continue
io_function.copy_file_to_dst(strip_dem,save_path,overwrite=True)
nodata = raster_io.get_nodata(save_path)
if raster_io.burn_polygon_to_raster_oneband(save_path,water_mask_shp,nodata) is False:
continue
mask_dem_list.append(save_path)
return mask_dem_list
def mask_strip_dem_outlier_by_ArcticDEM_mosaic(crop_strip_dem_list, extent_poly, extent_id, crop_tif_dir, o_res, process_num):
# get list of the ArcticDEM mosaic
arcticDEM_mosaic_reg_tifs = io_function.get_file_list_by_ext('.tif',arcticDEM_tile_reg_tif_dir,bsub_folder=False)
mosaic_dem_ext_polys = get_dem_tif_ext_polygons(arcticDEM_mosaic_reg_tifs)
overlap_index = vector_gpd.get_poly_index_within_extent(mosaic_dem_ext_polys,extent_poly)
#### crop and mosaic mosaic_reg_tifs
sub_mosaic_dem_tifs = [arcticDEM_mosaic_reg_tifs[item] for item in overlap_index]
mosaic_crop_tif_list = []
for tif in sub_mosaic_dem_tifs:
save_crop_path = os.path.join(crop_tif_dir, os.path.basename(io_function.get_name_by_adding_tail(tif, 'sub_poly_%d' % extent_id)) )
if os.path.isfile(save_crop_path):
basic.outputlogMessage('%s exists, skip cropping' % save_crop_path)
mosaic_crop_tif_list.append(save_crop_path)
else:
crop_tif = subset_image_by_polygon_box(tif, save_crop_path, extent_poly, resample_m='near',
o_format='VRT', out_res=o_res,same_extent=True,thread_num=process_num)
if crop_tif is False:
raise ValueError('warning, crop %s failed' % tif)
mosaic_crop_tif_list.append(crop_tif)
if len(mosaic_crop_tif_list) < 1:
basic.outputlogMessage('No mosaic version of ArcticDEM for %d grid, skip mask_strip_dem_outlier_by_ArcticDEM_mosaic'%extent_id)
return False
# create mosaic, can handle only input one file, but is slow
save_dem_mosaic = os.path.join(crop_tif_dir, 'ArcticDEM_tiles_grid%d.tif'%extent_id)
result = RSImageProcess.mosaic_crop_images_gdalwarp(mosaic_crop_tif_list, save_dem_mosaic, resampling_method='average',o_format='GTiff',
compress='lzw', tiled='yes', bigtiff='if_safer',thread_num=process_num)
if result is False:
return False
height_tileDEM, width_tileDEM, count_tileDEM, dtype_tileDEM = raster_io.get_height_width_bandnum_dtype(save_dem_mosaic)
tileDEM_data, tileDEM_nodata = raster_io.read_raster_one_band_np(save_dem_mosaic)
# masking the strip version of DEMs
mask_strip_dem_list = []
for idx, strip_dem in enumerate(crop_strip_dem_list):
save_path = io_function.get_name_by_adding_tail(strip_dem, 'maskOutlier')
if os.path.isfile(save_path):
basic.outputlogMessage('%s exist, skip'%save_path)
mask_strip_dem_list.append(save_path)
continue
# check band, with, height
height, width, count, dtype = raster_io.get_height_width_bandnum_dtype(strip_dem)
if height_tileDEM != height or width_tileDEM != width or count_tileDEM != count:
raise ValueError('size different between %s and %s' % (strip_dem, save_dem_mosaic))
if count != 1:
raise ValueError('DEM and Matchtag should only have one band')
try:
dem_data, nodata = raster_io.read_raster_one_band_np(strip_dem)
except:
basic.outputlogMessage(' invalid tif file: %s'%strip_dem)
continue
nodata_loc = np.where(dem_data == nodata)
diff = dem_data - tileDEM_data
# mask as nodata
dem_data[np.abs(diff) > 50 ] = nodata # ignore greater than 50 m
dem_data[ nodata_loc ] = nodata # may change some nodata pixel, change them back
# save to file
raster_io.save_numpy_array_to_rasterfile(dem_data, save_path, strip_dem, compress='lzw', tiled='yes',
bigtiff='if_safer')
mask_strip_dem_list.append(save_path)
return mask_strip_dem_list