def test_zonal_stats_multiRasters(): shp = os.path.expanduser( '~/Data/Arctic/canada_arctic/Willow_River/Willow_River_Thaw_Slumps.shp' ) # save_shp = os.path.basename(io_function.get_name_by_adding_tail(shp,'raster_stats')) # a single DEM # dem_file_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/DEM/WR_dem_ArcticDEM_mosaic') # dem_path = os.path.join(dem_file_dir,'WR_extent_2m_v3.0_ArcticTileDEM_sub_1_prj.tif') # dem patches dem_file_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/DEM/WR_dem_ArcticDEM_mosaic/dem_patches') dem_list = io_function.get_file_list_by_ext('.tif', dem_file_dir, bsub_folder=False) save_shp = os.path.basename( io_function.get_name_by_adding_tail(shp, 'multi_raster_stats')) io_function.copy_shape_file(shp, save_shp) zonal_stats_multiRasters(save_shp, dem_list, nodata=None, band=1, stats=None, prefix='dem', range=None, all_touched=True, process_num=4)
def copy_original_mapped_polygons(curr_dir_before_ray, work_dir): # when ray start a process, we need to add code_dir again and import user-defined modules import basic_src.io_function as io_function org_dir = os.path.join(curr_dir_before_ray, 'multi_inf_results') save_dir = os.path.join(work_dir, 'multi_inf_results') shp_list = io_function.get_file_list_by_pattern(org_dir, '*/*.shp') shp_list = [ item for item in shp_list if 'post' not in os.path.basename(item) ] # remove 'post' ones for shp in shp_list: area_dir = os.path.join(save_dir, os.path.basename(os.path.dirname(shp))) if os.path.isdir(area_dir) is False: io_function.mkdir(area_dir) dst_path = os.path.join(area_dir, os.path.basename(shp)) io_function.copy_shape_file(shp, dst_path)
def add_polygon_attributes(input, output, para_file, data_para_file): if io_function.is_file_exist(input) is False: return False # copy output if io_function.copy_shape_file(input, output) is False: raise IOError('copy shape file %s failed' % input) # remove narrow parts of mapped polygons polygon_narrow_part_thr = parameters.get_digit_parameters_None_if_absence( para_file, 'mapped_polygon_narrow_threshold', 'float') # if it is not None, then it will try to remove narrow parts of polygons if polygon_narrow_part_thr is not None and polygon_narrow_part_thr > 0: # use the buffer operation to remove narrow parts of polygons basic.outputlogMessage( "start removing narrow parts (thr %.2f) in polygons" % (polygon_narrow_part_thr * 2)) if vector_gpd.remove_narrow_parts_of_polygons_shp_NOmultiPolygon( input, output, polygon_narrow_part_thr): message = "Finished removing narrow parts (thr %.2f) in polygons and save to %s" % ( polygon_narrow_part_thr * 2, output) basic.outputlogMessage(message) else: pass else: basic.outputlogMessage( "warning, mapped_polygon_narrow_threshold is not in the parameter file, skip removing narrow parts" ) # calculate area, perimeter of polygons if cal_add_area_length_of_polygon(output) is False: return False # calculate the polygon information b_calculate_shape_info = parameters.get_bool_parameters_None_if_absence( para_file, 'b_calculate_shape_info') if b_calculate_shape_info: # remove "_shapeInfo.shp" to make it calculate shape information again os.system('rm *_shapeInfo.shp') if calculate_gully_information(output) is False: return False # add topography of each polygons dem_files, slope_files, aspect_files, dem_diff_files = get_topographic_files( data_para_file) if calculate_polygon_topography(output, para_file, dem_files, slope_files, aspect_files=aspect_files, dem_diffs=dem_diff_files) is False: basic.outputlogMessage( 'Warning: calculate information of topography failed') # return False # don't return return True
def add_boxes_attributes(input, output, para_file=None,data_para_file=None): if io_function.is_file_exist(input) is False: return False # copy output if io_function.copy_shape_file(input, output) is False: raise IOError('copy shape file %s failed'%input) # calculate area, perimeter of polygons if cal_add_area_length_of_polygon(output) is False: return False # calculate the width, height, and width_height ratio boxes = vector_gpd.read_polygons_gpd(input) bounding_boxes = [ vector_gpd.get_polygon_bounding_box(item) for item in boxes ] # bounding box (minx, miny, maxx, maxy) width_list = [ bound[2] - bound[0] for bound in bounding_boxes] height_list = [ bound[3] - bound[1] for bound in bounding_boxes] ratio_w_h_list = [] for w, h in zip(width_list, height_list): if w > h: ratio = h / w else: ratio = w / h ratio_w_h_list.append(ratio) add_attributes = {'WIDTH':width_list,'HEIGHT':height_list, 'ratio_w_h':ratio_w_h_list} vector_gpd.add_attributes_to_shp(output,add_attributes) # add topography of each box (consider the box as a polygon) if data_para_file is not None and para_file is not None: io_function.is_file_exist(data_para_file) io_function.is_file_exist(para_file) dem_files, slope_files, aspect_files, dem_diff_files = get_topographic_files(data_para_file) process_num = parameters.get_digit_parameters(para_file,'process_num','int') if calculate_polygon_topography(output,para_file,dem_files,slope_files,aspect_files=aspect_files,dem_diffs=dem_diff_files, process_num=process_num) is False: basic.outputlogMessage('Warning: calculate information of topography failed') # return False # don't return return output
def add_boxes_attributes(input, output): if io_function.is_file_exist(input) is False: return False # copy output if io_function.copy_shape_file(input, output) is False: raise IOError('copy shape file %s failed' % input) # calculate area, perimeter of polygons if cal_add_area_length_of_polygon(output) is False: return False # calculate the width, height, and width_height ratio boxes = vector_gpd.read_polygons_gpd(input) bounding_boxes = [ vector_gpd.get_polygon_bounding_box(item) for item in boxes ] # bounding box (minx, miny, maxx, maxy) width_list = [bound[2] - bound[0] for bound in bounding_boxes] height_list = [bound[3] - bound[1] for bound in bounding_boxes] ratio_w_h_list = [] for w, h in zip(width_list, height_list): if w > h: ratio = h / w else: ratio = w / h ratio_w_h_list.append(ratio) add_attributes = { 'WIDTH': width_list, 'HEIGHT': height_list, 'ratio_w_h': ratio_w_h_list } vector_gpd.add_attributes_to_shp(output, add_attributes) return output
def copy_shape_file(input, output): return io_function.copy_shape_file(input, output)
def postProcess(para_file, inf_post_note, b_skip_getshp=False, test_id=None): # test_id is the related to training if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd())) # the test string in 'exe.sh' test_note = inf_post_note WORK_DIR = os.getcwd() SECONDS = time.time() expr_name = parameters.get_string_parameters(para_file, 'expr_name') network_setting_ini = parameters.get_string_parameters( para_file, 'network_setting_ini') inf_dir = parameters.get_directory(para_file, 'inf_output_dir') if test_id is None: test_id = os.path.basename(WORK_DIR) + '_' + expr_name # get name of inference areas multi_inf_regions = parameters.get_string_list_parameters( para_file, 'inference_regions') # run post-processing parallel # max_parallel_postProc_task = 8 backup_dir = os.path.join(WORK_DIR, 'result_backup') io_function.mkdir(backup_dir) # loop each inference regions sub_tasks = [] same_area_time_inis = group_same_area_time_observations(multi_inf_regions) region_eva_reports = {} for key in same_area_time_inis.keys(): multi_observations = same_area_time_inis[key] area_name = parameters.get_string_parameters( multi_observations[0], 'area_name') # they have the same name and time area_time = parameters.get_string_parameters(multi_observations[0], 'area_time') merged_shp_list = [] map_raster_list_2d = [None] * len(multi_observations) for area_idx, area_ini in enumerate(multi_observations): area_remark = parameters.get_string_parameters( area_ini, 'area_remark') area_save_dir, shp_pre, _ = get_observation_save_dir_shp_pre( inf_dir, area_name, area_time, area_remark, test_id) # get image list inf_image_dir = parameters.get_directory(area_ini, 'inf_image_dir') # it is ok consider a file name as pattern and pass it the following functions to get file list inf_image_or_pattern = parameters.get_string_parameters( area_ini, 'inf_image_or_pattern') inf_img_list = io_function.get_file_list_by_pattern( inf_image_dir, inf_image_or_pattern) img_count = len(inf_img_list) if img_count < 1: raise ValueError( 'No image for inference, please check inf_image_dir and inf_image_or_pattern in %s' % area_ini) merged_shp = os.path.join(WORK_DIR, area_save_dir, shp_pre + '.shp') if b_skip_getshp: pass else: # post image one by one result_shp_list = [] map_raster_list = [] for img_idx, img_path in enumerate(inf_img_list): out_shp, out_raster = inf_results_to_shapefile( WORK_DIR, img_idx, area_save_dir, test_id) if out_shp is None or out_raster is None: continue result_shp_list.append(os.path.join(WORK_DIR, out_shp)) map_raster_list.append(out_raster) # merge shapefiles if merge_shape_files(result_shp_list, merged_shp) is False: continue map_raster_list_2d[area_idx] = map_raster_list merged_shp_list.append(merged_shp) if b_skip_getshp is False: # add occurrence to each polygons get_occurence_for_multi_observation(merged_shp_list) for area_idx, area_ini in enumerate(multi_observations): area_remark = parameters.get_string_parameters( area_ini, 'area_remark') area_save_dir, shp_pre, area_remark_time = get_observation_save_dir_shp_pre( inf_dir, area_name, area_time, area_remark, test_id) merged_shp = os.path.join(WORK_DIR, area_save_dir, shp_pre + '.shp') if os.path.isfile(merged_shp) is False: print('Warning, %s not exist, skip' % merged_shp) continue # add attributes to shapefile # add_attributes_script = os.path.join(code_dir,'datasets', 'get_polygon_attributes.py') shp_attributes = os.path.join(WORK_DIR, area_save_dir, shp_pre + '_post_NOrm.shp') # add_polygon_attributes(add_attributes_script,merged_shp, shp_attributes, para_file, area_ini ) add_polygon_attributes(merged_shp, shp_attributes, para_file, area_ini) # remove polygons # rm_polygon_script = os.path.join(code_dir,'datasets', 'remove_mappedPolygons.py') shp_post = os.path.join(WORK_DIR, area_save_dir, shp_pre + '_post.shp') # remove_polygons(rm_polygon_script,shp_attributes, shp_post, para_file) remove_polygons_main(shp_attributes, shp_post, para_file) # evaluate the mapping results # eval_shp_script = os.path.join(code_dir,'datasets', 'evaluation_result.py') out_report = os.path.join(WORK_DIR, area_save_dir, shp_pre + '_evaluation_report.txt') # evaluation_polygons(eval_shp_script, shp_post, para_file, area_ini,out_report) evaluation_polygons(shp_post, para_file, area_ini, out_report) ##### copy and backup files ###### # copy files to result_backup if len(test_note) > 0: backup_dir_area = os.path.join( backup_dir, area_name + '_' + area_remark_time + '_' + test_id + '_' + test_note) else: backup_dir_area = os.path.join( backup_dir, area_name + '_' + area_remark_time + '_' + test_id) io_function.mkdir(backup_dir_area) if len(test_note) > 0: bak_merged_shp = os.path.join( backup_dir_area, '_'.join([shp_pre, test_note]) + '.shp') bak_post_shp = os.path.join( backup_dir_area, '_'.join([shp_pre, 'post', test_note]) + '.shp') bak_eva_report = os.path.join( backup_dir_area, '_'.join([shp_pre, 'eva_report', test_note]) + '.txt') bak_area_ini = os.path.join( backup_dir_area, '_'.join([shp_pre, 'region', test_note]) + '.ini') else: bak_merged_shp = os.path.join(backup_dir_area, '_'.join([shp_pre]) + '.shp') bak_post_shp = os.path.join( backup_dir_area, '_'.join([shp_pre, 'post']) + '.shp') bak_eva_report = os.path.join( backup_dir_area, '_'.join([shp_pre, 'eva_report']) + '.txt') bak_area_ini = os.path.join( backup_dir_area, '_'.join([shp_pre, 'region']) + '.ini') io_function.copy_shape_file(merged_shp, bak_merged_shp) io_function.copy_shape_file(shp_post, bak_post_shp) if os.path.isfile(out_report): io_function.copy_file_to_dst(out_report, bak_eva_report, overwrite=True) io_function.copy_file_to_dst(area_ini, bak_area_ini, overwrite=True) # copy map raster b_backup_map_raster = parameters.get_bool_parameters_None_if_absence( area_ini, 'b_backup_map_raster') if b_backup_map_raster is True: if map_raster_list_2d[area_idx] is not None: for map_tif in map_raster_list_2d[area_idx]: bak_map_tif = os.path.join(backup_dir_area, os.path.basename(map_tif)) io_function.copy_file_to_dst(map_tif, bak_map_tif, overwrite=True) region_eva_reports[shp_pre] = bak_eva_report if len(test_note) > 0: bak_para_ini = os.path.join( backup_dir, '_'.join([test_id, 'para', test_note]) + '.ini') bak_network_ini = os.path.join( backup_dir, '_'.join([test_id, 'network', test_note]) + '.ini') bak_time_cost = os.path.join( backup_dir, '_'.join([test_id, 'time_cost', test_note]) + '.txt') else: bak_para_ini = os.path.join(backup_dir, '_'.join([test_id, 'para']) + '.ini') bak_network_ini = os.path.join(backup_dir, '_'.join([test_id, 'network']) + '.ini') bak_time_cost = os.path.join(backup_dir, '_'.join([test_id, 'time_cost']) + '.txt') io_function.copy_file_to_dst(para_file, bak_para_ini) io_function.copy_file_to_dst(network_setting_ini, bak_network_ini) if os.path.isfile('time_cost.txt'): io_function.copy_file_to_dst('time_cost.txt', bak_time_cost) # output the evaluation report to screen for key in region_eva_reports.keys(): report = region_eva_reports[key] if os.path.isfile(report) is False: continue print('evaluation report for %s:' % key) os.system('head -n 7 %s' % report) # output evaluation report to table if len(test_note) > 0: out_table = os.path.join( backup_dir, '_'.join([test_id, 'accuracy_table', test_note]) + '.xlsx') else: out_table = os.path.join( backup_dir, '_'.join([test_id, 'accuracy_table']) + '.xlsx') eva_reports = [ region_eva_reports[key] for key in region_eva_reports if os.path.isfile(region_eva_reports[key]) ] eva_report_to_tables.eva_reports_to_table(eva_reports, out_table) duration = time.time() - SECONDS os.system( 'echo "$(date): time cost of post-procesing: %.2f seconds">>time_cost.txt' % duration)
def extract_headwall_from_slope(idx, total, slope_tif, work_dir, save_dir,slope_threshold, min_area, max_area,max_axis_width,max_box_WH,process_num): ''' :param idx: tif index :param total: total slope file count :param slope_tif: slope file :param work_dir: :param save_dir: :param slope_threshold: :param min_area: :param max_area: :param max_axis_width: max width based on medial axis :param max_box_WH: max width or height based on minimum_rotated_rectangle :param process_num: :return: ''' headwall_shp = os.path.splitext(os.path.basename(io_function.get_name_by_adding_tail(slope_tif,'headwall')))[0] + '.shp' save_headwall_shp = os.path.join(save_dir,headwall_shp) if os.path.isfile(save_headwall_shp): print('%s exists, skip'%save_headwall_shp) return save_headwall_shp print('(%d/%d) extracting headwall from %s'%(idx,total,slope_tif)) wkt = map_projection.get_raster_or_vector_srs_info_wkt(slope_tif) # binary slope slope_bin_path = os.path.join(work_dir, os.path.basename(io_function.get_name_by_adding_tail(slope_tif, 'bin'))) slope_bin_shp = slope_tif_to_slope_shapefile(slope_tif,slope_bin_path,slope_threshold) # only keep small, but not too small rm_area_shp = io_function.get_name_by_adding_tail(slope_bin_shp, 'rmArea') if os.path.isfile(rm_area_shp): print('%s exists, skip removing based on area'%rm_area_shp) else: if remove_based_on_area(slope_bin_shp,min_area,max_area, wkt,rm_area_shp) is False: return False # add some shape info rm_shapeinfo_shp = io_function.get_name_by_adding_tail(slope_bin_shp, 'rmShape') if os.path.isfile(rm_shapeinfo_shp): print('%s exists, skip removing based on shape'%rm_shapeinfo_shp) else: if remove_based_on_shapeinfo(rm_area_shp, rm_shapeinfo_shp, max_box_WH) is False: return False rm_medialAxis_shp = io_function.get_name_by_adding_tail(slope_bin_shp, 'rmMedialAxis') if os.path.isfile(rm_medialAxis_shp): print('%s exists, skip removing based on Medial Axis') else: remove_based_medialAxis(rm_shapeinfo_shp, rm_medialAxis_shp,process_num,max_axis_width) # copy the results. io_function.copy_shape_file(rm_medialAxis_shp,save_headwall_shp) # add slope around surrounding? the sourrounding should be flat. NO. return save_headwall_shp
def main(options, args): input = args[0] output = args[1] if io_function.is_file_exist(input) is False: return False data_para_file = options.data_para if data_para_file is None: data_para_file = options.para_file ## remove non-gully polygons # output_rm_nonclass = io_function.get_name_by_adding_tail(input, 'rm_nonclass') # if remove_nonclass_polygon(input,output_rm_nonclass,field_name='svmclass') is False: # return False # merge the touched polygons # ouput_merged = io_function.get_name_by_adding_tail(input,'merged') # if merge_polygons_in_gully(input,ouput_merged) is False: # return False # ouput_merged = input # copy output if io_function.copy_shape_file(input, output) is False: raise IOError('copy shape file %s failed' % input) else: pass # remove narrow parts of mapped polygons polygon_narrow_part_thr = parameters.get_digit_parameters_None_if_absence( '', 'mapped_polygon_narrow_threshold', 'float') # if it is not None, then it will try to remove narrow parts of polygons if polygon_narrow_part_thr is not None and polygon_narrow_part_thr > 0: # use the buffer operation to remove narrow parts of polygons basic.outputlogMessage( "start removing narrow parts (thr %.2f) in polygons" % (polygon_narrow_part_thr * 2)) if vector_gpd.remove_narrow_parts_of_polygons_shp_NOmultiPolygon( input, output, polygon_narrow_part_thr): message = "Finished removing narrow parts (thr %.2f) in polygons and save to %s" % ( polygon_narrow_part_thr * 2, output) basic.outputlogMessage(message) else: pass else: basic.outputlogMessage( "warning, mapped_polygon_narrow_threshold is not in the parameter file, skip removing narrow parts" ) # calculate area, perimeter of polygons if cal_add_area_length_of_polygon(output) is False: return False # calculate the polygon information b_calculate_shape_info = parameters.get_bool_parameters_None_if_absence( '', 'b_calculate_shape_info') if b_calculate_shape_info: # remove "_shapeInfo.shp" to make it calculate shape information again os.system('rm *_shapeInfo.shp') if calculate_gully_information(output) is False: return False # # remove small and not narrow polygons # if options.min_area is None: # basic.outputlogMessage('minimum area is required for remove polygons') # return False # area_thr = options.min_area # # if options.min_ratio is None: # basic.outputlogMessage('minimum ratio of perimeter/area is required for remove polygons') # return False # ratio_thr = options.min_ratio # if remove_small_round_polygons(ouput_merged,output,area_thr,ratio_thr) is False: # return False # add topography of each polygons dem_file, slope_file, aspect_file, dem_diff_file = get_topographic_files( data_para_file) if calculate_polygon_topography(output, dem_file, slope_file, aspect_file=aspect_file, dem_diff=dem_diff_file) is False: basic.outputlogMessage( 'Warning: calculate information of topography failed') # return False # don't return # add hydrology information flow_accum = parameters.get_flow_accumulation() if os.path.isfile(flow_accum): if calculate_hydrology(output, flow_accum) is False: basic.outputlogMessage( 'Warning: calculate information of hydrology failed') # return False # don't return else: basic.outputlogMessage( "warning, flow accumulation file not exist, skip the calculation of flow accumulation" ) # # evaluation result # val_path = parameters.get_validation_shape() # if os.path.isfile(val_path): # evaluation_result(output,val_path) # else: # basic.outputlogMessage("warning, validation polygon not exist, skip evaluation") pass
def get_dem_subscidence_polygons(in_shp, dem_diff_tif, dem_diff_thread_m=-0.5, min_area=40, max_area=100000000, process_num=1, b_rm_files=False): save_shp = io_function.get_name_by_adding_tail(in_shp, 'post') if os.path.isfile(save_shp): basic.outputlogMessage('%s exists, skip'%save_shp) return save_shp demD_height, demD_width, demD_band_num, demD_date_type = raster_io.get_height_width_bandnum_dtype(dem_diff_tif) # print(demD_date_type) # # read mean elevation difference # attributes_path = os.path.join(os.path.dirname(in_shp), shp_pre + '_attributes.txt') # # # for each seg lable [mean, std, pixel count], if dem_diff_tif is float 32, then in meters, if int16, then in centimeter # poly_attributes = io_function.read_dict_from_txt_json(attributes_path) # if int16, then it's in centimeter if demD_date_type == 'int16': dem_diff_thread_m = dem_diff_thread_m*100 # merge polygons touch each others wkt = map_projection.get_raster_or_vector_srs_info_wkt(in_shp) merged_shp = io_function.get_name_by_adding_tail(in_shp, 'merged') if filter_merge_polygons(in_shp,merged_shp,wkt, min_area,max_area,dem_diff_tif,dem_diff_thread_m,process_num) is None: return None # in merge_polygons, it will remove some big polygons, convert MultiPolygon to Polygons, so neeed to update remain_polyons remain_polyons = vector_gpd.read_polygons_gpd(merged_shp) # check MultiPolygons again. polyons_noMulti = [vector_gpd.MultiPolygon_to_polygons(idx, poly) for idx, poly in enumerate(remain_polyons)] remain_polyons = [] for polys in polyons_noMulti: polys = [poly for poly in polys if poly.area > min_area] # remove tiny polygon before buffer remain_polyons.extend(polys) print('convert MultiPolygon to polygons and remove small ones, remain %d' % (len(remain_polyons))) # based on the merged polygons, surrounding polygons buffer_surrounding = 20 # meters surrounding_shp = io_function.get_name_by_adding_tail(in_shp, 'surrounding') get_surrounding_polygons(remain_polyons, surrounding_shp, wkt, dem_diff_tif, buffer_surrounding, process_num) rm_reldemD_shp = io_function.get_name_by_adding_tail(in_shp, 'rmreldemD') if remove_polygons_based_relative_dem_diff(remain_polyons, merged_shp, surrounding_shp, wkt, rm_reldemD_shp, min_area,dem_diff_thread_m) is None: return None rm_shapeinfo_shp = io_function.get_name_by_adding_tail(in_shp, 'rmshapeinfo') area_limit = 10000 circularit_limit = 0.1 holes_count = 20 if remove_polygons_based_shapeinfo(rm_reldemD_shp, rm_shapeinfo_shp, area_limit, circularit_limit, holes_count) is None: return None # remove based on slope # use the slope derived from ArcitcDEM mosaic slope_tif_list = io_function.get_file_list_by_ext('.tif',dem_common.arcticDEM_tile_slope_dir,bsub_folder=False) basic.outputlogMessage('Find %d slope files in %s'%(len(slope_tif_list), dem_common.arcticDEM_tile_slope_dir)) rm_slope_shp = io_function.get_name_by_adding_tail(in_shp, 'rmslope') max_slope = 20 if remove_based_slope(rm_shapeinfo_shp, rm_slope_shp,slope_tif_list, max_slope,process_num) is False: return None # copy io_function.copy_shape_file(rm_slope_shp,save_shp) # add date difference if they are available date_diff_base = os.path.basename(dem_diff_tif).replace('DEM_diff','date_diff') date_diff_tif = os.path.join(os.path.dirname(dem_diff_tif) , date_diff_base) if os.path.isfile(date_diff_tif): raster_statistic.zonal_stats_multiRasters(save_shp, date_diff_tif,tile_min_overlap=tile_min_overlap, stats=['mean', 'std'], prefix='dateD',process_num=process_num) return save_shp