def remove_based_slope(in_shp, output_shp,slope_files, max_slope,process_num):
# not too many polygons, not sure if slope info has been calculate, so just let it calculate agian.
# if os.path.isfile(output_shp):
# basic.outputlogMessage('%s exists, skip'%output_shp)
# return output_shp
# calcuate slope info
raster_statistic.zonal_stats_multiRasters(in_shp, slope_files, tile_min_overlap=tile_min_overlap,
stats=['mean', 'std', 'count'], prefix='slope',process_num=process_num)
# remove sloep greater than max_slope
bsmaller = False
if vector_gpd.remove_polygons(in_shp,'slope_mean',max_slope,bsmaller,output_shp) is False:
return False
return output_shp
def get_surrounding_polygons(remain_polyons,surrounding_shp,wkt, dem_diff_tif,buffer_surrounding,process_num):
if os.path.isfile(surrounding_shp):
# also check the file is complete
surr_polys, surr_demD = vector_gpd.read_polygons_attributes_list(surrounding_shp,'demD_mean')
if len(surr_polys) < len(remain_polyons) or surr_demD is None or len(surr_demD) < len(remain_polyons):
basic.outputlogMessage('%s already exists, but not complete, will be overwritten'%surrounding_shp)
else:
basic.outputlogMessage('%s already exists, skip'%surrounding_shp)
return surrounding_shp
# based on the merged polygons, calculate the relative dem_diff
surrounding_polygons = vector_gpd.get_surrounding_polygons(remain_polyons, buffer_surrounding)
surr_pd = pd.DataFrame({'Polygon': surrounding_polygons})
vector_gpd.save_polygons_to_files(surr_pd, 'Polygon', wkt, surrounding_shp)
raster_statistic.zonal_stats_multiRasters(surrounding_shp, dem_diff_tif, tile_min_overlap=tile_min_overlap,
stats=['mean', 'std', 'count'],prefix='demD', process_num=process_num)
return surrounding_shp
def polygonize_one_label(idx,
label_path,
org_raster,
stats,
prefix,
b_remove_nodata,
process_num=1):
save_dir = os.path.dirname(label_path)
out_pre = io_function.get_name_no_ext(label_path)
label_shp_path = os.path.join(save_dir, out_pre + '.shp')
if os.path.isfile(label_shp_path):
print('%s exist, skip' % label_shp_path)
return idx, label_shp_path
if b_remove_nodata is True:
# remove nodato (it was copy from the input image)
command_str = 'gdal_edit.py -unsetnodata ' + label_path
res = os.system(command_str)
if res != 0:
return None, None
# convert the label to shapefile
command_string = 'gdal_polygonize.py -8 %s -b 1 -f "ESRI Shapefile" %s' % (
label_path, label_shp_path)
res = os.system(command_string)
if res != 0:
return None, None
if org_raster is not None and stats is not None and prefix is not None:
# get dem elevation information for each polygon,
raster_statistic.zonal_stats_multiRasters(label_shp_path,
org_raster,
stats=stats,
prefix=prefix,
process_num=process_num)
return idx, label_shp_path
def calculate_polygon_topography(polygons_shp,
para_file,
dem_files,
slope_files,
aspect_files=None,
dem_diffs=None):
"""
calculate the topography information such elevation and slope of each polygon
Args:
polygons_shp: input shapfe file
dem_files: DEM raster file or tiles, should have the same projection of shapefile
slope_files: slope raster file or tiles (can be drived from dem file by using QGIS or ArcGIS)
aspect_files: aspect raster file or tiles (can be drived from dem file by using QGIS or ArcGIS)
Returns: True if successful, False Otherwise
"""
if io_function.is_file_exist(polygons_shp) is False:
return False
operation_obj = shape_opeation()
## calculate the topography information from the buffer area
# the para file was set in parameters.set_saved_parafile_path(options.para_file)
b_use_buffer_area = parameters.get_bool_parameters(
para_file, 'b_topo_use_buffer_area')
if b_use_buffer_area is True:
b_buffer_size = 5 # meters (the same as the shape file)
basic.outputlogMessage(
"info: calculate the topography information from the buffer area")
buffer_polygon_shp = io_function.get_name_by_adding_tail(
polygons_shp, 'buffer')
# if os.path.isfile(buffer_polygon_shp) is False:
if vector_features.get_buffer_polygons(
polygons_shp, buffer_polygon_shp, b_buffer_size) is False:
basic.outputlogMessage(
"error, failed in producing the buffer_polygon_shp")
return False
# else:
# basic.outputlogMessage("warning, buffer_polygon_shp already exist, skip producing it")
# replace the polygon shape file
polygons_shp_backup = polygons_shp
polygons_shp = buffer_polygon_shp
else:
basic.outputlogMessage(
"info: calculate the topography information from the inside of each polygon"
)
# all_touched: bool, optional
# Whether to include every raster cell touched by a geometry, or only
# those having a center point within the polygon.
# defaults to `False`
# Since the dem usually is coarser, so we set all_touched = True
all_touched = True
process_num = 4
# #DEM
if dem_files is not None:
stats_list = ['min', 'max', 'mean', 'median',
'std'] #['min', 'max', 'mean', 'count','median','std']
# if operation_obj.add_fields_from_raster(polygons_shp, dem_file, "dem", band=1,stats_list=stats_list,all_touched=all_touched) is False:
# return False
if zonal_stats_multiRasters(polygons_shp,
dem_files,
stats=stats_list,
prefix='dem',
band=1,
all_touched=all_touched,
process_num=process_num) is False:
return False
else:
basic.outputlogMessage(
"warning, DEM file not exist, skip the calculation of DEM information"
)
# #slope
if slope_files is not None:
stats_list = ['min', 'max', 'mean', 'median', 'std']
if zonal_stats_multiRasters(polygons_shp,
slope_files,
stats=stats_list,
prefix='slo',
band=1,
all_touched=all_touched,
process_num=process_num) is False:
return False
else:
basic.outputlogMessage(
"warning, slope file not exist, skip the calculation of slope information"
)
# #aspect
if aspect_files is not None:
stats_list = ['min', 'max', 'mean', 'std']
if zonal_stats_multiRasters(polygons_shp,
aspect_files,
stats=stats_list,
prefix='asp',
band=1,
all_touched=all_touched,
process_num=process_num) is False:
return False
else:
basic.outputlogMessage(
'warning, aspect file not exist, ignore adding aspect information')
# elevation difference
if dem_diffs is not None:
stats_list = ['min', 'max', 'mean', 'median', 'std', 'area']
# only count the pixel within this range when do statistics
dem_diff_range_str = parameters.get_string_list_parameters(
para_file, 'dem_difference_range')
range = [
None if item.upper() == 'NONE' else float(item)
for item in dem_diff_range_str
]
# expand the polygon when doing dem difference statistics
buffer_size_dem_diff = parameters.get_digit_parameters(
para_file, 'buffer_size_dem_diff', 'float')
if zonal_stats_multiRasters(polygons_shp,
dem_diffs,
stats=stats_list,
prefix='demD',
band=1,
all_touched=all_touched,
process_num=process_num,
range=range,
buffer=buffer_size_dem_diff) is False:
return False
else:
basic.outputlogMessage(
'warning, dem difference file not exist, ignore adding dem diff information'
)
# # hillshape
# copy the topography information
if b_use_buffer_area is True:
operation_obj.add_fields_shape(polygons_shp_backup, buffer_polygon_shp,
polygons_shp_backup)
return True
def segment_subsidence_grey_image(dem_diff_grey_8bit, dem_diff, save_dir,process_num, subsidence_thr_m=-0.5, min_area=40, max_area=100000000,
b_rm_files=False):
'''
segment subsidence areas based on 8bit dem difference
:param dem_diff_grey_8bit:
:param dem_diff:
:param save_dir:
:param process_num:
:param subsidence_thr_m: mean value less than this one consider as subsidence (in meter)
:param min_area: min size in m^2 (defualt is 40 m^2, 10 pixels on ArcticDEM)
:param max_area: min size in m^2 (default is 10km by 10 km)
:return:
'''
io_function.is_file_exist(dem_diff_grey_8bit)
out_pre = os.path.splitext(os.path.basename(dem_diff_grey_8bit))[0]
segment_shp_path = os.path.join(save_dir, out_pre + '.shp')
# check if the final result exist
final_shp_path = io_function.get_name_by_adding_tail(segment_shp_path, 'post')
if os.path.isfile(final_shp_path):
basic.outputlogMessage('Warning, Final results (%s) of subsidence shapefile exists, skip'%final_shp_path)
return True
# get initial polygons
# because the label from segmentation for superpixels are not unique, so we may need to get mean dem diff based on polygons, set org_raster=None
label_path = segment_a_grey_image(dem_diff_grey_8bit,save_dir,process_num, org_raster=None)
if os.path.isfile(segment_shp_path) and vector_gpd.is_field_name_in_shp(segment_shp_path,'demD_mean'):
basic.outputlogMessage('%s exists, skip'%segment_shp_path)
else:
# remove segment_shp_path if it exist, but don't have demD_mean
if os.path.isfile(segment_shp_path):
io_function.delete_shape_file(segment_shp_path)
# remove nodato (it was copy from the input image)
command_str = 'gdal_edit.py -unsetnodata ' + label_path
basic.os_system_exit_code(command_str)
# convert the label to shapefile # remove -8 (to use 4 connectedness.)
command_string = 'gdal_polygonize.py %s -b 1 -f "ESRI Shapefile" %s' % (label_path, segment_shp_path)
res = os.system(command_string)
if res != 0:
sys.exit(1)
# get dem elevation information for each polygon
raster_statistic.zonal_stats_multiRasters(segment_shp_path, dem_diff, tile_min_overlap=tile_min_overlap,
stats=['mean', 'std','count'], prefix='demD',process_num=process_num)
# get DEM diff information for each polygon.
dem_diff_shp = get_dem_subscidence_polygons(segment_shp_path, dem_diff, dem_diff_thread_m=subsidence_thr_m,
min_area=min_area, max_area=max_area, process_num=process_num,b_rm_files=b_rm_files)
if dem_diff_shp is None:
id_str = re.findall('grid\d+', os.path.basename(dem_diff))[0][4:]
if len(id_str) > 1:
grid_id = int(id_str)
save_id_grid_no_subsidence(grid_id)
else:
basic.outputlogMessage('obtain elevation reduction polygons: %s'%dem_diff_shp)
## remove files, only keep the final results.
if b_rm_files:
io_function.delete_file_or_dir(label_path)
IDrange_txt = os.path.splitext(label_path)[0] + '_IDrange.txt'
io_function.delete_file_or_dir(IDrange_txt)
io_function.delete_shape_file(segment_shp_path)
# other intermediate files
other_shp_names = ['merged','surrounding','rmreldemD','rmshapeinfo','rmslope']
for name in other_shp_names:
rm_shp = io_function.get_name_by_adding_tail(segment_shp_path, name)
io_function.delete_shape_file(rm_shp)
return True
def get_dem_subscidence_polygons(in_shp, dem_diff_tif, dem_diff_thread_m=-0.5, min_area=40, max_area=100000000, process_num=1,
b_rm_files=False):
save_shp = io_function.get_name_by_adding_tail(in_shp, 'post')
if os.path.isfile(save_shp):
basic.outputlogMessage('%s exists, skip'%save_shp)
return save_shp
demD_height, demD_width, demD_band_num, demD_date_type = raster_io.get_height_width_bandnum_dtype(dem_diff_tif)
# print(demD_date_type)
# # read mean elevation difference
# attributes_path = os.path.join(os.path.dirname(in_shp), shp_pre + '_attributes.txt')
#
# # for each seg lable [mean, std, pixel count], if dem_diff_tif is float 32, then in meters, if int16, then in centimeter
# poly_attributes = io_function.read_dict_from_txt_json(attributes_path)
# if int16, then it's in centimeter
if demD_date_type == 'int16':
dem_diff_thread_m = dem_diff_thread_m*100
# merge polygons touch each others
wkt = map_projection.get_raster_or_vector_srs_info_wkt(in_shp)
merged_shp = io_function.get_name_by_adding_tail(in_shp, 'merged')
if filter_merge_polygons(in_shp,merged_shp,wkt, min_area,max_area,dem_diff_tif,dem_diff_thread_m,process_num) is None:
return None
# in merge_polygons, it will remove some big polygons, convert MultiPolygon to Polygons, so neeed to update remain_polyons
remain_polyons = vector_gpd.read_polygons_gpd(merged_shp)
# check MultiPolygons again.
polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polyons)]
remain_polyons = []
for polys in polyons_noMulti:
polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon before buffer
remain_polyons.extend(polys)
print('convert MultiPolygon to polygons and remove small ones, remain %d' % (len(remain_polyons)))
# based on the merged polygons, surrounding polygons
buffer_surrounding = 20 # meters
surrounding_shp = io_function.get_name_by_adding_tail(in_shp, 'surrounding')
get_surrounding_polygons(remain_polyons, surrounding_shp, wkt, dem_diff_tif, buffer_surrounding, process_num)
rm_reldemD_shp = io_function.get_name_by_adding_tail(in_shp, 'rmreldemD')
if remove_polygons_based_relative_dem_diff(remain_polyons, merged_shp, surrounding_shp, wkt, rm_reldemD_shp, min_area,dem_diff_thread_m) is None:
return None
rm_shapeinfo_shp = io_function.get_name_by_adding_tail(in_shp, 'rmshapeinfo')
area_limit = 10000
circularit_limit = 0.1
holes_count = 20
if remove_polygons_based_shapeinfo(rm_reldemD_shp, rm_shapeinfo_shp, area_limit, circularit_limit, holes_count) is None:
return None
# remove based on slope
# use the slope derived from ArcitcDEM mosaic
slope_tif_list = io_function.get_file_list_by_ext('.tif',dem_common.arcticDEM_tile_slope_dir,bsub_folder=False)
basic.outputlogMessage('Find %d slope files in %s'%(len(slope_tif_list), dem_common.arcticDEM_tile_slope_dir))
rm_slope_shp = io_function.get_name_by_adding_tail(in_shp, 'rmslope')
max_slope = 20
if remove_based_slope(rm_shapeinfo_shp, rm_slope_shp,slope_tif_list, max_slope,process_num) is False:
return None
# copy
io_function.copy_shape_file(rm_slope_shp,save_shp)
# add date difference if they are available
date_diff_base = os.path.basename(dem_diff_tif).replace('DEM_diff','date_diff')
date_diff_tif = os.path.join(os.path.dirname(dem_diff_tif) , date_diff_base)
if os.path.isfile(date_diff_tif):
raster_statistic.zonal_stats_multiRasters(save_shp, date_diff_tif,tile_min_overlap=tile_min_overlap,
stats=['mean', 'std'], prefix='dateD',process_num=process_num)
return save_shp
def filter_merge_polygons(in_shp,merged_shp,wkt, min_area,max_area,dem_diff_tif,dem_diff_thread_m,process_num):
if os.path.isfile(merged_shp):
# also check the file is complete
polys, demD_values = vector_gpd.read_polygons_attributes_list(merged_shp,'demD_mean')
if len(polys) < 1 or demD_values is None or len(demD_values) < 1:
basic.outputlogMessage('%s already exists, but not complete, will be overwritten'%merged_shp)
else:
basic.outputlogMessage('%s exists, skip'%merged_shp)
return merged_shp
# read polygons and label from segment algorithm, note: some polygons may have the same label
# polygons, demD_mean_list = vector_gpd.read_polygons_attributes_list(in_shp,'demD_mean')
polygons, attributes = vector_gpd.read_polygons_attributes_list(in_shp,['demD_mean','DN'])
demD_mean_list = attributes[0]
DN_list = attributes[1]
print('Read %d polygons'%len(polygons))
if demD_mean_list is None:
raise ValueError('demD_mean not in %s, need to remove it and then re-create'%in_shp)
# replace None (if exists) as nan
demD_mean_list = np.array(demD_mean_list, dtype=float)
# replace nan values as 0
demD_mean_list = np.nan_to_num(demD_mean_list)
remain_polyons = []
rm_min_area_count = 0
rm_diff_thr_count = 0
for poly, demD_mean in zip(polygons, demD_mean_list):
if poly.area < min_area:
rm_min_area_count += 1
continue
# mean value: not subsidence
if demD_mean > dem_diff_thread_m: #
rm_diff_thr_count += 1
continue
remain_polyons.append(poly)
print('remove %d polygons based on min_area, %d polygons based on dem_diff_threshold, remain %d ones'%(rm_min_area_count, rm_diff_thr_count,len(remain_polyons)))
if len(remain_polyons) < 1:
return None
# we should only merge polygon with similar reduction, but we already remove polygons with mean reduction > threshhold
# merge touch polygons
# print(timeTools.get_now_time_str(), 'start building adjacent_matrix')
# # adjacent_matrix = vector_features.build_adjacent_map_of_polygons(remain_polyons)
# machine_name = os.uname()[1]
# if 'login' in machine_name or 'shas' in machine_name or 'sgpu' in machine_name:
# print('Warning, some problem of parallel running in build_adjacent_map_of_polygons on curc, '
# 'but ok in my laptop and uist, change process_num = 1')
# process_num = 1
############################################################
## build adjacent_matrix then merge for entire raster
# adjacent_matrix = vector_gpd.build_adjacent_map_of_polygons(remain_polyons, process_num=process_num)
# print(timeTools.get_now_time_str(), 'finish building adjacent_matrix')
#
# if adjacent_matrix is False:
# return None
# merged_polygons = vector_features.merge_touched_polygons(remain_polyons, adjacent_matrix)
############################################################
# ## build adjacent_matrix then merge, patch by patch (not too many improvements)
# label_id_range_txt = os.path.splitext(in_shp)[0] + '_label_IDrange.txt'
# merged_polygons = merge_polygons_patchBYpatch(label_id_range_txt, remain_polyons, DN_list, process_num=process_num)
############################################################
## merge polygons using rasterize
label_raster = os.path.splitext(in_shp)[0] + '_label.tif'
merged_polygons = merge_polygon_rasterize(label_raster, remain_polyons)
print(timeTools.get_now_time_str(), 'finish merging touched polygons, get %d ones' % (len(merged_polygons)))
# remove large ones
remain_polyons = []
rm_max_area_count = 0
for poly in merged_polygons:
if poly.area > max_area:
rm_max_area_count += 1
continue
remain_polyons.append(poly)
print('remove %d polygons based on max_area, remain %d' % (rm_max_area_count, len(remain_polyons)))
polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polyons)]
remain_polyons = []
for polys in polyons_noMulti:
polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon before buffer
remain_polyons.extend(polys)
print('convert MultiPolygon (filter_merge_polygons) to polygons and remove small ones, remain %d' % (len(remain_polyons)))
if len(remain_polyons) < 1:
return None
# calcualte attributes of remain ones: area, dem_diff: mean, std
merged_pd = pd.DataFrame({'Polygon': remain_polyons})
vector_gpd.save_polygons_to_files(merged_pd, 'Polygon', wkt, merged_shp)
# based on the merged polygons, calculate the mean dem diff
raster_statistic.zonal_stats_multiRasters(merged_shp, dem_diff_tif, tile_min_overlap=tile_min_overlap,
stats=['mean', 'std', 'count'], prefix='demD',process_num=process_num)
return merged_shp