def remove_based_slope(in_shp, output_shp,slope_files, max_slope,process_num): # not too many polygons, not sure if slope info has been calculate, so just let it calculate agian. # if os.path.isfile(output_shp): # basic.outputlogMessage('%s exists, skip'%output_shp) # return output_shp # calcuate slope info raster_statistic.zonal_stats_multiRasters(in_shp, slope_files, tile_min_overlap=tile_min_overlap, stats=['mean', 'std', 'count'], prefix='slope',process_num=process_num) # remove sloep greater than max_slope bsmaller = False if vector_gpd.remove_polygons(in_shp,'slope_mean',max_slope,bsmaller,output_shp) is False: return False return output_shp
def get_surrounding_polygons(remain_polyons,surrounding_shp,wkt, dem_diff_tif,buffer_surrounding,process_num): if os.path.isfile(surrounding_shp): # also check the file is complete surr_polys, surr_demD = vector_gpd.read_polygons_attributes_list(surrounding_shp,'demD_mean') if len(surr_polys) < len(remain_polyons) or surr_demD is None or len(surr_demD) < len(remain_polyons): basic.outputlogMessage('%s already exists, but not complete, will be overwritten'%surrounding_shp) else: basic.outputlogMessage('%s already exists, skip'%surrounding_shp) return surrounding_shp # based on the merged polygons, calculate the relative dem_diff surrounding_polygons = vector_gpd.get_surrounding_polygons(remain_polyons, buffer_surrounding) surr_pd = pd.DataFrame({'Polygon': surrounding_polygons}) vector_gpd.save_polygons_to_files(surr_pd, 'Polygon', wkt, surrounding_shp) raster_statistic.zonal_stats_multiRasters(surrounding_shp, dem_diff_tif, tile_min_overlap=tile_min_overlap, stats=['mean', 'std', 'count'],prefix='demD', process_num=process_num) return surrounding_shp
def polygonize_one_label(idx, label_path, org_raster, stats, prefix, b_remove_nodata, process_num=1): save_dir = os.path.dirname(label_path) out_pre = io_function.get_name_no_ext(label_path) label_shp_path = os.path.join(save_dir, out_pre + '.shp') if os.path.isfile(label_shp_path): print('%s exist, skip' % label_shp_path) return idx, label_shp_path if b_remove_nodata is True: # remove nodato (it was copy from the input image) command_str = 'gdal_edit.py -unsetnodata ' + label_path res = os.system(command_str) if res != 0: return None, None # convert the label to shapefile command_string = 'gdal_polygonize.py -8 %s -b 1 -f "ESRI Shapefile" %s' % ( label_path, label_shp_path) res = os.system(command_string) if res != 0: return None, None if org_raster is not None and stats is not None and prefix is not None: # get dem elevation information for each polygon, raster_statistic.zonal_stats_multiRasters(label_shp_path, org_raster, stats=stats, prefix=prefix, process_num=process_num) return idx, label_shp_path
def calculate_polygon_topography(polygons_shp, para_file, dem_files, slope_files, aspect_files=None, dem_diffs=None): """ calculate the topography information such elevation and slope of each polygon Args: polygons_shp: input shapfe file dem_files: DEM raster file or tiles, should have the same projection of shapefile slope_files: slope raster file or tiles (can be drived from dem file by using QGIS or ArcGIS) aspect_files: aspect raster file or tiles (can be drived from dem file by using QGIS or ArcGIS) Returns: True if successful, False Otherwise """ if io_function.is_file_exist(polygons_shp) is False: return False operation_obj = shape_opeation() ## calculate the topography information from the buffer area # the para file was set in parameters.set_saved_parafile_path(options.para_file) b_use_buffer_area = parameters.get_bool_parameters( para_file, 'b_topo_use_buffer_area') if b_use_buffer_area is True: b_buffer_size = 5 # meters (the same as the shape file) basic.outputlogMessage( "info: calculate the topography information from the buffer area") buffer_polygon_shp = io_function.get_name_by_adding_tail( polygons_shp, 'buffer') # if os.path.isfile(buffer_polygon_shp) is False: if vector_features.get_buffer_polygons( polygons_shp, buffer_polygon_shp, b_buffer_size) is False: basic.outputlogMessage( "error, failed in producing the buffer_polygon_shp") return False # else: # basic.outputlogMessage("warning, buffer_polygon_shp already exist, skip producing it") # replace the polygon shape file polygons_shp_backup = polygons_shp polygons_shp = buffer_polygon_shp else: basic.outputlogMessage( "info: calculate the topography information from the inside of each polygon" ) # all_touched: bool, optional # Whether to include every raster cell touched by a geometry, or only # those having a center point within the polygon. # defaults to `False` # Since the dem usually is coarser, so we set all_touched = True all_touched = True process_num = 4 # #DEM if dem_files is not None: stats_list = ['min', 'max', 'mean', 'median', 'std'] #['min', 'max', 'mean', 'count','median','std'] # if operation_obj.add_fields_from_raster(polygons_shp, dem_file, "dem", band=1,stats_list=stats_list,all_touched=all_touched) is False: # return False if zonal_stats_multiRasters(polygons_shp, dem_files, stats=stats_list, prefix='dem', band=1, all_touched=all_touched, process_num=process_num) is False: return False else: basic.outputlogMessage( "warning, DEM file not exist, skip the calculation of DEM information" ) # #slope if slope_files is not None: stats_list = ['min', 'max', 'mean', 'median', 'std'] if zonal_stats_multiRasters(polygons_shp, slope_files, stats=stats_list, prefix='slo', band=1, all_touched=all_touched, process_num=process_num) is False: return False else: basic.outputlogMessage( "warning, slope file not exist, skip the calculation of slope information" ) # #aspect if aspect_files is not None: stats_list = ['min', 'max', 'mean', 'std'] if zonal_stats_multiRasters(polygons_shp, aspect_files, stats=stats_list, prefix='asp', band=1, all_touched=all_touched, process_num=process_num) is False: return False else: basic.outputlogMessage( 'warning, aspect file not exist, ignore adding aspect information') # elevation difference if dem_diffs is not None: stats_list = ['min', 'max', 'mean', 'median', 'std', 'area'] # only count the pixel within this range when do statistics dem_diff_range_str = parameters.get_string_list_parameters( para_file, 'dem_difference_range') range = [ None if item.upper() == 'NONE' else float(item) for item in dem_diff_range_str ] # expand the polygon when doing dem difference statistics buffer_size_dem_diff = parameters.get_digit_parameters( para_file, 'buffer_size_dem_diff', 'float') if zonal_stats_multiRasters(polygons_shp, dem_diffs, stats=stats_list, prefix='demD', band=1, all_touched=all_touched, process_num=process_num, range=range, buffer=buffer_size_dem_diff) is False: return False else: basic.outputlogMessage( 'warning, dem difference file not exist, ignore adding dem diff information' ) # # hillshape # copy the topography information if b_use_buffer_area is True: operation_obj.add_fields_shape(polygons_shp_backup, buffer_polygon_shp, polygons_shp_backup) return True
def segment_subsidence_grey_image(dem_diff_grey_8bit, dem_diff, save_dir,process_num, subsidence_thr_m=-0.5, min_area=40, max_area=100000000, b_rm_files=False): ''' segment subsidence areas based on 8bit dem difference :param dem_diff_grey_8bit: :param dem_diff: :param save_dir: :param process_num: :param subsidence_thr_m: mean value less than this one consider as subsidence (in meter) :param min_area: min size in m^2 (defualt is 40 m^2, 10 pixels on ArcticDEM) :param max_area: min size in m^2 (default is 10km by 10 km) :return: ''' io_function.is_file_exist(dem_diff_grey_8bit) out_pre = os.path.splitext(os.path.basename(dem_diff_grey_8bit))[0] segment_shp_path = os.path.join(save_dir, out_pre + '.shp') # check if the final result exist final_shp_path = io_function.get_name_by_adding_tail(segment_shp_path, 'post') if os.path.isfile(final_shp_path): basic.outputlogMessage('Warning, Final results (%s) of subsidence shapefile exists, skip'%final_shp_path) return True # get initial polygons # because the label from segmentation for superpixels are not unique, so we may need to get mean dem diff based on polygons, set org_raster=None label_path = segment_a_grey_image(dem_diff_grey_8bit,save_dir,process_num, org_raster=None) if os.path.isfile(segment_shp_path) and vector_gpd.is_field_name_in_shp(segment_shp_path,'demD_mean'): basic.outputlogMessage('%s exists, skip'%segment_shp_path) else: # remove segment_shp_path if it exist, but don't have demD_mean if os.path.isfile(segment_shp_path): io_function.delete_shape_file(segment_shp_path) # remove nodato (it was copy from the input image) command_str = 'gdal_edit.py -unsetnodata ' + label_path basic.os_system_exit_code(command_str) # convert the label to shapefile # remove -8 (to use 4 connectedness.) command_string = 'gdal_polygonize.py %s -b 1 -f "ESRI Shapefile" %s' % (label_path, segment_shp_path) res = os.system(command_string) if res != 0: sys.exit(1) # get dem elevation information for each polygon raster_statistic.zonal_stats_multiRasters(segment_shp_path, dem_diff, tile_min_overlap=tile_min_overlap, stats=['mean', 'std','count'], prefix='demD',process_num=process_num) # get DEM diff information for each polygon. dem_diff_shp = get_dem_subscidence_polygons(segment_shp_path, dem_diff, dem_diff_thread_m=subsidence_thr_m, min_area=min_area, max_area=max_area, process_num=process_num,b_rm_files=b_rm_files) if dem_diff_shp is None: id_str = re.findall('grid\d+', os.path.basename(dem_diff))[0][4:] if len(id_str) > 1: grid_id = int(id_str) save_id_grid_no_subsidence(grid_id) else: basic.outputlogMessage('obtain elevation reduction polygons: %s'%dem_diff_shp) ## remove files, only keep the final results. if b_rm_files: io_function.delete_file_or_dir(label_path) IDrange_txt = os.path.splitext(label_path)[0] + '_IDrange.txt' io_function.delete_file_or_dir(IDrange_txt) io_function.delete_shape_file(segment_shp_path) # other intermediate files other_shp_names = ['merged','surrounding','rmreldemD','rmshapeinfo','rmslope'] for name in other_shp_names: rm_shp = io_function.get_name_by_adding_tail(segment_shp_path, name) io_function.delete_shape_file(rm_shp) return True
def get_dem_subscidence_polygons(in_shp, dem_diff_tif, dem_diff_thread_m=-0.5, min_area=40, max_area=100000000, process_num=1, b_rm_files=False): save_shp = io_function.get_name_by_adding_tail(in_shp, 'post') if os.path.isfile(save_shp): basic.outputlogMessage('%s exists, skip'%save_shp) return save_shp demD_height, demD_width, demD_band_num, demD_date_type = raster_io.get_height_width_bandnum_dtype(dem_diff_tif) # print(demD_date_type) # # read mean elevation difference # attributes_path = os.path.join(os.path.dirname(in_shp), shp_pre + '_attributes.txt') # # # for each seg lable [mean, std, pixel count], if dem_diff_tif is float 32, then in meters, if int16, then in centimeter # poly_attributes = io_function.read_dict_from_txt_json(attributes_path) # if int16, then it's in centimeter if demD_date_type == 'int16': dem_diff_thread_m = dem_diff_thread_m*100 # merge polygons touch each others wkt = map_projection.get_raster_or_vector_srs_info_wkt(in_shp) merged_shp = io_function.get_name_by_adding_tail(in_shp, 'merged') if filter_merge_polygons(in_shp,merged_shp,wkt, min_area,max_area,dem_diff_tif,dem_diff_thread_m,process_num) is None: return None # in merge_polygons, it will remove some big polygons, convert MultiPolygon to Polygons, so neeed to update remain_polyons remain_polyons = vector_gpd.read_polygons_gpd(merged_shp) # check MultiPolygons again. polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polyons)] remain_polyons = [] for polys in polyons_noMulti: polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon before buffer remain_polyons.extend(polys) print('convert MultiPolygon to polygons and remove small ones, remain %d' % (len(remain_polyons))) # based on the merged polygons, surrounding polygons buffer_surrounding = 20 # meters surrounding_shp = io_function.get_name_by_adding_tail(in_shp, 'surrounding') get_surrounding_polygons(remain_polyons, surrounding_shp, wkt, dem_diff_tif, buffer_surrounding, process_num) rm_reldemD_shp = io_function.get_name_by_adding_tail(in_shp, 'rmreldemD') if remove_polygons_based_relative_dem_diff(remain_polyons, merged_shp, surrounding_shp, wkt, rm_reldemD_shp, min_area,dem_diff_thread_m) is None: return None rm_shapeinfo_shp = io_function.get_name_by_adding_tail(in_shp, 'rmshapeinfo') area_limit = 10000 circularit_limit = 0.1 holes_count = 20 if remove_polygons_based_shapeinfo(rm_reldemD_shp, rm_shapeinfo_shp, area_limit, circularit_limit, holes_count) is None: return None # remove based on slope # use the slope derived from ArcitcDEM mosaic slope_tif_list = io_function.get_file_list_by_ext('.tif',dem_common.arcticDEM_tile_slope_dir,bsub_folder=False) basic.outputlogMessage('Find %d slope files in %s'%(len(slope_tif_list), dem_common.arcticDEM_tile_slope_dir)) rm_slope_shp = io_function.get_name_by_adding_tail(in_shp, 'rmslope') max_slope = 20 if remove_based_slope(rm_shapeinfo_shp, rm_slope_shp,slope_tif_list, max_slope,process_num) is False: return None # copy io_function.copy_shape_file(rm_slope_shp,save_shp) # add date difference if they are available date_diff_base = os.path.basename(dem_diff_tif).replace('DEM_diff','date_diff') date_diff_tif = os.path.join(os.path.dirname(dem_diff_tif) , date_diff_base) if os.path.isfile(date_diff_tif): raster_statistic.zonal_stats_multiRasters(save_shp, date_diff_tif,tile_min_overlap=tile_min_overlap, stats=['mean', 'std'], prefix='dateD',process_num=process_num) return save_shp
def filter_merge_polygons(in_shp,merged_shp,wkt, min_area,max_area,dem_diff_tif,dem_diff_thread_m,process_num): if os.path.isfile(merged_shp): # also check the file is complete polys, demD_values = vector_gpd.read_polygons_attributes_list(merged_shp,'demD_mean') if len(polys) < 1 or demD_values is None or len(demD_values) < 1: basic.outputlogMessage('%s already exists, but not complete, will be overwritten'%merged_shp) else: basic.outputlogMessage('%s exists, skip'%merged_shp) return merged_shp # read polygons and label from segment algorithm, note: some polygons may have the same label # polygons, demD_mean_list = vector_gpd.read_polygons_attributes_list(in_shp,'demD_mean') polygons, attributes = vector_gpd.read_polygons_attributes_list(in_shp,['demD_mean','DN']) demD_mean_list = attributes[0] DN_list = attributes[1] print('Read %d polygons'%len(polygons)) if demD_mean_list is None: raise ValueError('demD_mean not in %s, need to remove it and then re-create'%in_shp) # replace None (if exists) as nan demD_mean_list = np.array(demD_mean_list, dtype=float) # replace nan values as 0 demD_mean_list = np.nan_to_num(demD_mean_list) remain_polyons = [] rm_min_area_count = 0 rm_diff_thr_count = 0 for poly, demD_mean in zip(polygons, demD_mean_list): if poly.area < min_area: rm_min_area_count += 1 continue # mean value: not subsidence if demD_mean > dem_diff_thread_m: # rm_diff_thr_count += 1 continue remain_polyons.append(poly) print('remove %d polygons based on min_area, %d polygons based on dem_diff_threshold, remain %d ones'%(rm_min_area_count, rm_diff_thr_count,len(remain_polyons))) if len(remain_polyons) < 1: return None # we should only merge polygon with similar reduction, but we already remove polygons with mean reduction > threshhold # merge touch polygons # print(timeTools.get_now_time_str(), 'start building adjacent_matrix') # # adjacent_matrix = vector_features.build_adjacent_map_of_polygons(remain_polyons) # machine_name = os.uname()[1] # if 'login' in machine_name or 'shas' in machine_name or 'sgpu' in machine_name: # print('Warning, some problem of parallel running in build_adjacent_map_of_polygons on curc, ' # 'but ok in my laptop and uist, change process_num = 1') # process_num = 1 ############################################################ ## build adjacent_matrix then merge for entire raster # adjacent_matrix = vector_gpd.build_adjacent_map_of_polygons(remain_polyons, process_num=process_num) # print(timeTools.get_now_time_str(), 'finish building adjacent_matrix') # # if adjacent_matrix is False: # return None # merged_polygons = vector_features.merge_touched_polygons(remain_polyons, adjacent_matrix) ############################################################ # ## build adjacent_matrix then merge, patch by patch (not too many improvements) # label_id_range_txt = os.path.splitext(in_shp)[0] + '_label_IDrange.txt' # merged_polygons = merge_polygons_patchBYpatch(label_id_range_txt, remain_polyons, DN_list, process_num=process_num) ############################################################ ## merge polygons using rasterize label_raster = os.path.splitext(in_shp)[0] + '_label.tif' merged_polygons = merge_polygon_rasterize(label_raster, remain_polyons) print(timeTools.get_now_time_str(), 'finish merging touched polygons, get %d ones' % (len(merged_polygons))) # remove large ones remain_polyons = [] rm_max_area_count = 0 for poly in merged_polygons: if poly.area > max_area: rm_max_area_count += 1 continue remain_polyons.append(poly) print('remove %d polygons based on max_area, remain %d' % (rm_max_area_count, len(remain_polyons))) polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polyons)] remain_polyons = [] for polys in polyons_noMulti: polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon before buffer remain_polyons.extend(polys) print('convert MultiPolygon (filter_merge_polygons) to polygons and remove small ones, remain %d' % (len(remain_polyons))) if len(remain_polyons) < 1: return None # calcualte attributes of remain ones: area, dem_diff: mean, std merged_pd = pd.DataFrame({'Polygon': remain_polyons}) vector_gpd.save_polygons_to_files(merged_pd, 'Polygon', wkt, merged_shp) # based on the merged polygons, calculate the mean dem diff raster_statistic.zonal_stats_multiRasters(merged_shp, dem_diff_tif, tile_min_overlap=tile_min_overlap, stats=['mean', 'std', 'count'], prefix='demD',process_num=process_num) return merged_shp