def dem_list_to_slope_list(dem_list, save_dir, extent_id, process_num=1): slope_list = [] slope_tif_dir = os.path.join(save_dir, 'slope_sub_%d' % extent_id) if os.path.isdir(slope_tif_dir) is False: io_function.mkdir(slope_tif_dir) if process_num == 1: for idx, tif in enumerate(dem_list): slope_tif = one_dem_to_slope(tif, slope_tif_dir) if slope_tif is not False: slope_list.append(slope_tif) elif process_num > 1: # change backk to multi process of gdalwarp, when get mosaic, gdalwarp multi-thread cannot fully utlized CPUs theadPool = Pool(process_num) # multi processes parameters_list = [(tif, slope_tif_dir) for idx, tif in enumerate(dem_list)] results = theadPool.starmap(one_dem_to_slope, parameters_list) # need python3 slope_list = [out for out in results if out is not False] theadPool.close() else: raise ValueError('Wrong process number: %s' % str(process_num)) return slope_list
def main(): basic.setlogfile('log_to_relative_dem_8bit.txt') if os.path.isdir(relative_dem_dir) is False: io_function.mkdir(relative_dem_dir) # 500 pixel by 500 pixel, that is 1 km by 1 km patch_width = 500 patch_height = 500 process_num = 1 failed_tifs = [] dem_reg_list = io_function.get_file_list_by_pattern( arcticDEM_reg_tif_dir, '*dem_reg.tif') count = len(dem_reg_list) for idx, tif in enumerate(dem_reg_list): print('%d/%d convert %s to relative DEM (8bit)' % (idx + 1, count, tif)) rel_dem_8bit = io_function.get_name_by_adding_tail(tif, 'relDEM8bit') rel_dem_8bit = os.path.join(relative_dem_dir, os.path.basename(rel_dem_8bit)) try: dem_to_relative_dem(tif, rel_dem_8bit, patch_width, patch_height, process_num) except: failed_tifs.append(tif) with open('to_relative_dem_failed_cases.txt', 'w') as f_obj: for item in failed_tifs: f_obj.writelines(item + '\n') pass
def crop_to_same_exent_for_diff(dem_tif_list, save_dir, extent_id, extent_poly, process_num): # crop to the same extent crop_tif_dir = os.path.join(save_dir, 'dem_crop_for_diff_sub_%d' % extent_id) if os.path.isdir(crop_tif_dir) is False: io_function.mkdir(crop_tif_dir) crop_tif_list = [] for tif in dem_tif_list: save_crop_path = os.path.join( crop_tif_dir, os.path.basename( io_function.get_name_by_adding_tail(tif, 'sub_poly_%d' % extent_id))) if os.path.isfile(save_crop_path): basic.outputlogMessage('%s exists, skip cropping' % save_crop_path) crop_tif_list.append(save_crop_path) else: crop_tif = subset_image_by_polygon_box(tif, save_crop_path, extent_poly, resample_m='near', same_extent=True, thread_num=process_num) if crop_tif is False: # raise ValueError('warning, crop %s failed' % tif) continue crop_tif_list.append(crop_tif) dem_tif_list = crop_tif_list return dem_tif_list
def split_an_image(para_file, image_path, save_dir, patch_w, patch_h, overlay_x, overlay_y): split_format = parameters.get_string_parameters(para_file, 'split_image_format') out_format = 'PNG' # default is PNG if split_format == '.tif': out_format = 'GTIFF' if split_format == '.jpg': out_format = 'JPEG' if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) split_image.split_image(image_path, save_dir, patch_w, patch_h, overlay_x, overlay_y, out_format, pre_name=None, process_num=8) # get list patch_list = io_function.get_file_list_by_ext(split_format, save_dir, bsub_folder=False) if len(patch_list) < 1: print('Wanring, no images in %s' % save_dir) return None list_txt_path = save_dir + '_list.txt' io_function.save_list_to_txt(list_txt_path, patch_list) return list_txt_path
def mosaic_dem_same_stripID(demTif_groups,save_tif_dir, resample_method, process_num=1, save_source=False, o_format='GTiff'): if os.path.isdir(save_tif_dir): io_function.mkdir(save_tif_dir) # when run in parallel, it has "Finalize object, dead" after a while, cannot figure out why?, so temporal set process_num = 1 # could related to the output logfile to disk. # on tesia, it's fine, but on uist, the issue occurs just in a few minutes. # could be useful: Why your multiprocessing Pool is stuck: https://pythonspeed.com/articles/python-multiprocessing/ # update on 15 March, 2021. I changed the python from 3.8 on uist to 3.7 (same as tesia), then problem solved. # but sometime, the program crash without specific reason (get killed) # process_num = 1 mosaic_list = [] if process_num == 1: for key in demTif_groups.keys(): save_mosaic = mosaic_dem_list(key, demTif_groups[key], save_tif_dir,resample_method,save_source, o_format,thread_num=process_num) mosaic_list.append(save_mosaic) elif process_num > 1: # change backk to multi process of gdalwarp, when get mosaic, gdalwarp multi-thread cannot fully utlized CPUs theadPool = Pool(process_num) # multi processes parameters_list = [(key, demTif_groups[key], save_tif_dir, resample_method, save_source, o_format,1) for key in demTif_groups.keys()] results = theadPool.starmap(mosaic_dem_list, parameters_list) # need python3 mosaic_list = [ out for out in results if out is not False] theadPool.close() else: raise ValueError('Wrong process_num: %d'%process_num) return mosaic_list
def segment_subsidence_on_dem_diff(dem_diff_tif, save_dir): out_pre = os.path.splitext(os.path.basename(dem_diff_tif))[0] # read images one_band_img, nodata = raster_io.read_raster_one_band_np(dem_diff_tif) # segmentation by threshold (may have too many noise) # mean = np.nanmean(one_band_img) # print("mean value is: %.4f"%mean) # one_band_img = one_band_img - mean # cannot use mean which may affect by some Outliers out_labels = np.zeros_like(one_band_img,dtype=np.uint8) out_labels[ one_band_img < -2 ] = 1 # end in a lot of noise, change to -2, -1 results in a lot of polygons # apply median filter out_labels = cv2.medianBlur(out_labels, 3) # with kernal=3 # save the label if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) label_path = os.path.join(save_dir, out_pre + '_label.tif') raster_io.save_numpy_array_to_rasterfile(out_labels, label_path, dem_diff_tif, nodata=0) # convert the label to shapefile out_shp = os.path.join(save_dir, out_pre + '.shp') command_string = 'gdal_polygonize.py -8 %s -b 1 -f "ESRI Shapefile" %s' % (label_path, out_shp) res = os.system(command_string) if res != 0: sys.exit(1) # post-processing post_processing_subsidence(out_shp)
def run_extract_headwall_jobs(max_job_count, n_tif_per_jobs): from dem_common import dem_headwall_shp_dir, dem_slope_dir if os.path.isdir(dem_headwall_shp_dir) is False: io_function.mkdir(dem_headwall_shp_dir) # get slope file list slope_tifs = io_function.get_file_list_by_ext('.tif', dem_slope_dir, bsub_folder=False) print('Found %d tif in %s' % (len(slope_tifs), dem_slope_dir)) # divide grid_ids to many groups slope_tif_count = len(slope_tifs) slope_tif_groups = [ slope_tifs[i:i + n_tif_per_jobs] for i in range(0, slope_tif_count, n_tif_per_jobs) ] for idx, slope_tifs_group in enumerate(slope_tif_groups): print( datetime.now(), 'processing %d group for extracting headwall, total %d ones' % (idx, len(slope_tif_groups))) submit_extract_headwall_job(slope_tifs_group, idx, max_job_count)
def main(options, args): input = args[0] if input.endswith('.txt'): slope_tifs = io_function.read_list_from_txt(input) elif os.path.isdir(input): slope_tifs = io_function.get_file_list_by_ext('.tif',input, bsub_folder=True) else: slope_tifs = [ input] process_num = options.process_num working_dir = './' save_dir = dem_headwall_shp_dir if os.path.isdir(working_dir) is False: io_function.mkdir(working_dir) if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) failed_tifs = [] min_slope = options.min_slope min_size = options.min_area max_size = options.max_area max_axis_width = options.max_axis_width max_box_WH = options.max_box_WH for idx, slope in enumerate(slope_tifs): if extract_headwall_from_slope(idx, len(slope_tifs), slope,working_dir,save_dir, min_slope,min_size,max_size,max_axis_width,max_box_WH,process_num) is False: failed_tifs.append(slope) io_function.save_list_to_txt('extract_headwall_failed_tifs.txt',failed_tifs)
def main(): # run in ~/Data/Arctic/canada_arctic/autoMapping/multiArea_sub_images on tesia ini_list = io_function.get_file_list_by_pattern('./','area*.ini') txt_list = io_function.get_file_list_by_pattern('./','area*.txt') for txt in txt_list: ini_s = io_function.read_list_from_txt(txt) ini_list.extend(ini_s) ini_list = [os.path.abspath(item) for item in ini_list] file_names = [ io_function.get_name_no_ext(item) for item in ini_list ] cur_dir = os.getcwd() # show [print(item) for item in ini_list] time.sleep(3) for name, area_ini in zip(file_names,ini_list): word_dir = os.path.join(cur_dir,name) io_function.mkdir(word_dir) os.chdir(word_dir) # copy and modify main_para.ini io_function.copyfiletodir(os.path.join(cur_dir,'main_para.ini'),'./',overwrite=True) io_function.copyfiletodir(os.path.join(cur_dir,'exe.sh'),'./',overwrite=True) parameters.write_Parameters_file('main_para.ini','training_regions',area_ini) # run exe.sh res = os.system('./exe.sh') if res !=0: print(res) sys.exit(1) os.chdir(cur_dir)
def main(options, args): t_polygons_shp = args[0] image_folder = args[1] # folder for store image tile (many split block of a big image) b_label_image = options.no_label_image process_num = options.process_num # check training polygons assert io_function.is_file_exist(t_polygons_shp) t_polygons_shp_all = options.all_training_polygons if t_polygons_shp_all is None: basic.outputlogMessage('Warning, the full set of training polygons is not assigned, ' 'it will consider the one in input argument is the full set of training polygons') t_polygons_shp_all = t_polygons_shp else: if get_projection_proj4(t_polygons_shp) != get_projection_proj4(t_polygons_shp_all): raise ValueError('error, projection insistence between %s and %s'%(t_polygons_shp, t_polygons_shp_all)) assert io_function.is_file_exist(t_polygons_shp_all) # get image tile list # image_tile_list = io_function.get_file_list_by_ext(options.image_ext, image_folder, bsub_folder=False) image_tile_list = io_function.get_file_list_by_pattern(image_folder,options.image_ext) if len(image_tile_list) < 1: raise IOError('error, failed to get image tiles in folder %s'%image_folder) check_projection_rasters(image_tile_list) # it will raise errors if found problems # comment out on June 18, 2021, # check_1or3band_8bit(image_tile_list) # it will raise errors if found problems #need to check: the shape file and raster should have the same projection. if get_projection_proj4(t_polygons_shp) != get_projection_proj4(image_tile_list[0]): raise ValueError('error, the input raster (e.g., %s) and vector (%s) files don\'t have the same projection'%(image_tile_list[0],t_polygons_shp)) # check these are EPSG:4326 projection if get_projection_proj4(t_polygons_shp).strip() == '+proj=longlat +datum=WGS84 +no_defs': bufferSize = meters_to_degress_onEarth(options.bufferSize) else: bufferSize = options.bufferSize saved_dir = options.out_dir # if os.system('mkdir -p ' + os.path.join(saved_dir,'subImages')) != 0: # sys.exit(1) # if os.system('mkdir -p ' + os.path.join(saved_dir,'subLabels')) !=0: # sys.exit(1) io_function.mkdir(os.path.join(saved_dir,'subImages')) if b_label_image: io_function.mkdir(os.path.join(saved_dir,'subLabels')) dstnodata = options.dstnodata if 'qtb_sentinel2' in image_tile_list[0]: # for qtb_sentinel-2 mosaic pre_name = '_'.join(os.path.splitext(os.path.basename(image_tile_list[0]))[0].split('_')[:4]) else: pre_name = os.path.splitext(os.path.basename(image_tile_list[0]))[0] get_sub_images_and_labels(t_polygons_shp, t_polygons_shp_all, bufferSize, image_tile_list, saved_dir, pre_name, dstnodata, brectangle=options.rectangle, b_label=b_label_image, proc_num=process_num)
def predict_one_image_mmseg(para_file, image_path, img_save_dir, inf_list_file, gpuid, trained_model): """ run prediction of one image """ expr_name = parameters.get_string_parameters(para_file, 'expr_name') network_ini = parameters.get_string_parameters(para_file, 'network_setting_ini') base_config_file = parameters.get_string_parameters( network_ini, 'base_config') config_file = osp.basename( io_function.get_name_by_adding_tail(base_config_file, expr_name)) inf_batch_size = parameters.get_digit_parameters(network_ini, 'inf_batch_size', 'int') patch_width = parameters.get_digit_parameters(para_file, 'inf_patch_width', 'int') patch_height = parameters.get_digit_parameters(para_file, 'inf_patch_height', 'int') adj_overlay_x = parameters.get_digit_parameters(para_file, 'inf_pixel_overlay_x', 'int') adj_overlay_y = parameters.get_digit_parameters(para_file, 'inf_pixel_overlay_y', 'int') done_indicator = '%s_done' % inf_list_file if os.path.isfile(done_indicator): basic.outputlogMessage('warning, %s exist, skip prediction' % done_indicator) return if os.path.isdir(img_save_dir) is False: io_function.mkdir(img_save_dir) # use a specific GPU for prediction, only inference one image time0 = time.time() if gpuid is None: gpuid = 0 predict_rsImage_mmseg(config_file, trained_model, image_path, img_save_dir, batch_size=inf_batch_size, gpuid=gpuid, tile_width=patch_width, tile_height=patch_height, overlay_x=adj_overlay_x, overlay_y=adj_overlay_y) duration = time.time() - time0 os.system( 'echo "$(date): time cost of inference for image in %s: %.2f seconds">>"time_cost.txt"' % (inf_list_file, duration)) # write a file to indicate that the prediction has done. os.system('echo %s > %s_done' % (inf_list_file, inf_list_file)) return
def one_dem_diff_to_8bit(demDiff_tif): if os.path.isdir(grid_dem_diffs_8bit_dir) is False: io_function.mkdir(grid_dem_diffs_8bit_dir) tif_8bit = io_function.get_name_by_adding_tail(demDiff_tif, '8bit') output = os.path.join(grid_dem_diffs_8bit_dir, os.path.basename(tif_8bit)) if dem_tif_to_8bit(demDiff_tif, output) is False: basic.outputlogMessage('failed to generate 8bit grey image from DEM differnce') return False return True
def move_files(save_dir, out_fig, out_hist_info): if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) trim_fig = io_function.get_name_by_adding_tail(out_fig, 'trim') os.system('convert -trim %s %s' % (out_fig, trim_fig)) io_function.movefiletodir(trim_fig, save_dir, overwrite=True) io_function.delete_file_or_dir(out_fig) # io_function.movefiletodir(out_fig,save_dir,overwrite=True) io_function.movefiletodir(out_hist_info, save_dir, overwrite=True)
def check_dem_valid_per(dem_tif_list, work_dir, process_num =1, move_dem_threshold = None, area_pixel_num=None): ''' get the valid pixel percentage for each DEM :param dem_tif_list: :param work_dir: :param move_dem_threshold: move a DEM to a sub-folder if its valid percentage small then the threshold :return: ''' keep_dem_list = [] print('start getting valid pixel percent for %d files'%len(dem_tif_list)) dem_tif_valid_per = {} # when run in parallel, it has "Finalize object, dead" after a while, cannot figure out why?, so temporal set process_num = 1 # process_num = 1 #update on 15 March, 2021. I changed the python from 3.8 on uist to 3.7 (same as tesia), then problem solved. if process_num == 1: for idx,tif in enumerate(dem_tif_list): # RSImage.get_valid_pixel_count(tif) # per = RSImage.get_valid_pixel_percentage(tif,total_pixel_num=area_pixel_num) print('(%d/%d) get valid pixel percent for %s'%(idx+1, len(dem_tif_list),tif)) per = raster_io.get_valid_pixel_percentage(tif, total_pixel_num=area_pixel_num) if per is False: return False dem_tif_valid_per[tif] = per keep_dem_list.append(tif) elif process_num > 1: theadPool = Pool(process_num) # multi processes parameters_list = [(tif, area_pixel_num, '%d/%d'%(idx+1, len(dem_tif_list)) ) for idx,tif in enumerate(dem_tif_list)] results = theadPool.starmap(raster_io.get_valid_pixel_percentage, parameters_list) # need python3 for res, tif in zip(results, dem_tif_list): if res is False: return False dem_tif_valid_per[tif] = res keep_dem_list.append(tif) else: raise ValueError("Wrong process_num: %d"%process_num) # sort dem_tif_valid_per_d = dict(sorted(dem_tif_valid_per.items(), key=operator.itemgetter(1), reverse=True)) percent_txt = os.path.join(work_dir,'dem_valid_percent.txt') with open(percent_txt,'w') as f_obj: for key in dem_tif_valid_per_d: f_obj.writelines('%s %.4f\n'%(os.path.basename(key),dem_tif_valid_per_d[key])) basic.outputlogMessage('save dem valid pixel percentage to %s'%percent_txt) # only keep dem with valid pixel greater than a threshold if move_dem_threshold is not None: # int or float keep_dem_list = [] # reset the list mosaic_dir_rm = os.path.join(work_dir,'dem_valid_lt_%.2f'%move_dem_threshold) io_function.mkdir(mosaic_dir_rm) for tif in dem_tif_valid_per.keys(): if dem_tif_valid_per[tif] < move_dem_threshold: io_function.movefiletodir(tif,mosaic_dir_rm) else: keep_dem_list.append(tif) return keep_dem_list
def main(options, args): extent_shp_or_ids_txt = args[0] process_num = options.process_num o_res = options.out_res if os.path.isdir(grid_matchtag_sum_dir) is False: io_function.mkdir(grid_matchtag_sum_dir) basic.setlogfile('produce_matchtag_sum_ArcticDEM_log_%s.txt' % timeTools.get_now_time_str()) # read grids and ids time0 = time.time() all_grid_polys, all_ids = vector_gpd.read_polygons_attributes_list( grid_20_shp, 'id') print('time cost of read polygons and attributes', time.time() - time0) # get grid ids based on input extent grid_base_name = os.path.splitext( os.path.basename(extent_shp_or_ids_txt))[0] grid_polys, grid_ids = get_grid_20(extent_shp_or_ids_txt, all_grid_polys, all_ids) # check dem difference existence grid_dem_tifs, grid_ids_no_sum = get_existing_matchtag_sum( grid_matchtag_sum_dir, grid_base_name, grid_ids) if len(grid_ids_no_sum) > 0: # refine grid_polys if len(grid_ids) > len(grid_ids_no_sum): id_index = [grid_ids.index(id) for id in grid_ids_no_sum] grid_polys = [grid_polys[idx] for idx in id_index] # # download ArcticDEM and applying registration # tarballs, reg_tifs = download_dem_tarball(dem_strip_shp, grid_polys, arcticDEM_tarball_dir, grid_base_name, # reg_tif_dir=arcticDEM_reg_tif_dir, poly_ids=grid_ids_no_demDiff) # # # unpack and applying registration # if len(tarballs) > 0: # basic.outputlogMessage('Processs %d dem tarballs'%len(tarballs)) # out_reg_tifs = process_dem_tarball(tarballs,'./',arcticDEM_reg_tif_dir,remove_inter_data=True, apply_registration=True) # basic.outputlogMessage('Get %d new registration dem tifs' % len(out_reg_tifs)) # reg_tifs.extend(out_reg_tifs) reg_tifs = io_function.get_file_list_by_ext('.tif', arcticDEM_reg_tif_dir, bsub_folder=False) matchtag_tifs = [tif for tif in reg_tifs if 'matchtag' in tif] # only keep matchtag # crop, sum out_dem_diffs = produce_matchtag_sum_grids(grid_polys, grid_ids_no_sum, grid_base_name, matchtag_tifs, o_res, process_num=process_num)
def get_sub_images_pixel_json_files(polygons_shp, image_folder_or_path, image_pattern, class_names, bufferSize, dstnodata, saved_dir, b_rectangle, process_num): # check training polygons assert io_function.is_file_exist(polygons_shp) # get image tile list # image_tile_list = io_function.get_file_list_by_ext(options.image_ext, image_folder, bsub_folder=False) if os.path.isdir(image_folder_or_path): image_tile_list = io_function.get_file_list_by_pattern( image_folder_or_path, image_pattern) else: assert io_function.is_file_exist(image_folder_or_path) image_tile_list = [image_folder_or_path] if len(image_tile_list) < 1: raise IOError('error, failed to get image tiles in folder %s' % image_folder_or_path) get_subImages.check_projection_rasters( image_tile_list) # it will raise errors if found problems get_subImages.check_1or3band_8bit( image_tile_list) # it will raise errors if found problems # need to check: the shape file and raster should have the same projection. if get_subImages.get_projection_proj4( polygons_shp) != get_subImages.get_projection_proj4( image_tile_list[0]): raise ValueError( 'error, the input raster (e.g., %s) and vector (%s) files don\'t have the same projection' % (image_tile_list[0], polygons_shp)) # check these are EPSG:4326 projection if get_subImages.get_projection_proj4( polygons_shp).strip() == '+proj=longlat +datum=WGS84 +no_defs': bufferSize = get_subImages.meters_to_degress_onEarth(bufferSize) pre_name = os.path.splitext(os.path.basename(image_tile_list[0]))[0] saved_dir = os.path.join(saved_dir, pre_name + '_subImages') if os.path.isdir(saved_dir) is False: io_function.mkdir(saved_dir) get_sub_images_and_json_files(polygons_shp, class_names, bufferSize, image_tile_list, saved_dir, pre_name, dstnodata, brectangle=b_rectangle, proc_num=process_num)
def copy_subImages_labels_directly(subImage_dir, subLabel_dir, area_ini): input_image_dir = parameters.get_directory_None_if_absence( area_ini, 'input_image_dir') # it is ok consider a file name as pattern and pass it the following functions to get file list input_image_or_pattern = parameters.get_string_parameters( area_ini, 'input_image_or_pattern') # label raster folder label_raster_dir = parameters.get_directory_None_if_absence( area_ini, 'label_raster_dir') sub_images_list = [] label_path_list = [] if os.path.isdir(subImage_dir) is False: io_function.mkdir(subImage_dir) if os.path.isdir(subLabel_dir) is False: io_function.mkdir(subLabel_dir) sub_images = io_function.get_file_list_by_pattern(input_image_dir, input_image_or_pattern) for sub_img in sub_images: # find the corresponding label raster label_name = io_function.get_name_by_adding_tail( os.path.basename(sub_img), 'label') label_path = os.path.join(label_raster_dir, label_name) if os.path.isfile(label_path): sub_images_list.append(sub_img) label_path_list.append(label_path) else: print('Warning, cannot find label for %s in %s' % (sub_img, label_raster_dir)) # copy sub-images, adding to txt files with open('sub_images_labels_list.txt', 'a') as f_obj: for tif_path, label_file in zip(sub_images_list, label_path_list): if label_file is None: continue dst_subImg = os.path.join(subImage_dir, os.path.basename(tif_path)) # copy sub-images io_function.copy_file_to_dst(tif_path, dst_subImg, overwrite=True) dst_label_file = os.path.join(subLabel_dir, os.path.basename(label_file)) io_function.copy_file_to_dst(label_file, dst_label_file, overwrite=True) sub_image_label_str = dst_subImg + ":" + dst_label_file + '\n' f_obj.writelines(sub_image_label_str)
def main(options, args): extent_shp_or_ids_txt = args[0] process_num = options.process_num keep_dem_percent = options.keep_dem_percent o_res = options.out_res basic.setlogfile('produce_headwall_shp_ArcticDEM_log_%s.txt' % timeTools.get_now_time_str()) if os.path.isdir(grid_dem_headwall_shp_dir) is False: io_function.mkdir(grid_dem_headwall_shp_dir) # read grids and ids time0 = time.time() all_grid_polys, all_ids = vector_gpd.read_polygons_attributes_list( grid_20_shp, 'id') print('time cost of read polygons and attributes', time.time() - time0) # get grid ids based on input extent grid_base_name = os.path.splitext( os.path.basename(extent_shp_or_ids_txt))[0] grid_polys, grid_ids = get_grid_20(extent_shp_or_ids_txt, all_grid_polys, all_ids) # check dem difference existence grid_headwall_shps, grid_id_no_headwall_shp = get_existing_grid_headwall_shp( grid_dem_headwall_shp_dir, grid_base_name, grid_ids) if len(grid_id_no_headwall_shp) > 0: # refine grid_polys if len(grid_ids) > len(grid_id_no_headwall_shp): id_index = [grid_ids.index(id) for id in grid_id_no_headwall_shp] grid_polys = [grid_polys[idx] for idx in id_index] reg_tifs = io_function.get_file_list_by_ext('.tif', arcticDEM_reg_tif_dir, bsub_folder=False) reg_tifs = [tif for tif in reg_tifs if 'matchtag' not in tif] # remove matchtag # headwall_shp_folders = extract_headwall_grids(grid_polys, grid_id_no_headwall_shp, grid_base_name, reg_tifs, b_mosaic_id, b_mosaic_date, keep_dem_percent, o_res, process_num=process_num)
def save_id_grid_no_subsidence(grid_id): # grid_no_headwall_txt if os.path.isdir(process_log_dir) is False: io_function.mkdir(process_log_dir) id_list = [] if os.path.isfile(grid_no_subscidence_poly_txt): id_list = io_function.read_list_from_txt(grid_no_subscidence_poly_txt) # no need covert to int id_str = str(grid_id) if id_str in id_list: return True else: # save by adding one line with open(grid_no_subscidence_poly_txt,'a') as f_obj: f_obj.writelines(str(grid_id) + '\n') return True
def add_id_grid_to_txt(grid_id, txt_path): # grid_no_valid_dem_ids.txt if os.path.isdir(process_log_dir) is False: io_function.mkdir(process_log_dir) # update txt file id_list = [] if os.path.isfile(txt_path): id_list = io_function.read_list_from_txt(txt_path) # no need covert to int id_str = str(grid_id) if id_str in id_list: return True else: # save by adding one line with open(txt_path,'a') as f_obj: f_obj.writelines(str(grid_id) + '\n') return True
def submit_hillshade_newest_headwall_line_grid_job(ids_list, idx, grid_base_name, max_job_count): wait_if_reach_max_jobs(max_job_count, 'dLi') # draw Line on hillshade job_name = 'dLi%d' % idx check_length_jobname(job_name) work_dir = working_dir_string(idx, 'hillshade_newest_headwall_line_', root=root_dir) if os.path.isdir(work_dir) is False: io_function.mkdir(work_dir) os.chdir(work_dir) ids_list = [str(item) for item in ids_list] io_function.save_list_to_txt(grid_base_name + '.txt', ids_list) # prepare job sh_list = [ 'hillshade_headwall_line_grid.sh', 'job_hillshade_headwall_line_grid.sh' ] copy_curc_job_files(jobsh_dir, work_dir, sh_list) slurm_utility.modify_slurm_job_sh( 'job_hillshade_headwall_line_grid.sh', 'job-name', job_name) else: os.chdir(work_dir) submit_job_names = slurm_utility.get_submited_job_names(curc_username) if job_name in submit_job_names: print( 'The folder: %s already exist and the job has been submitted, skip submitting a new job' % work_dir) return # job is completed if os.path.isfile('done.txt'): print('The job in the folder: %s is Done' % work_dir) return # submit the job # sometime, when submit a job, end with: singularity: command not found,and exist, wired, then try run submit a job in scomplie note submit_job_curc_or_run_script_local('job_hillshade_headwall_line_grid.sh', 'hillshade_headwall_line_grid.sh') os.chdir(curr_dir_before_start)
def copy_original_mapped_polygons(curr_dir_before_ray, work_dir): # when ray start a process, we need to add code_dir again and import user-defined modules import basic_src.io_function as io_function org_dir = os.path.join(curr_dir_before_ray, 'multi_inf_results') save_dir = os.path.join(work_dir, 'multi_inf_results') shp_list = io_function.get_file_list_by_pattern(org_dir, '*/*.shp') shp_list = [ item for item in shp_list if 'post' not in os.path.basename(item) ] # remove 'post' ones for shp in shp_list: area_dir = os.path.join(save_dir, os.path.basename(os.path.dirname(shp))) if os.path.isdir(area_dir) is False: io_function.mkdir(area_dir) dst_path = os.path.join(area_dir, os.path.basename(shp)) io_function.copy_shape_file(shp, dst_path)
def polygons2geojson(input_shp, save_folder): ''' convert polygons in shapefiles to many geojson (each for one polygon) :param input_shp: :param save_folder: :return: ''' io_function.is_file_exist(input_shp) if os.path.isdir(save_folder) is False: io_function.mkdir(save_folder) polygons, ids = vector_gpd.read_polygons_attributes_list(input_shp, 'id') prj_info = map_projection.get_raster_or_vector_srs_info_epsg( input_shp) # geojson need EPSG, such as "EPSG:3413" # print(prj_info) for poly, id in zip(polygons, ids): save_one_polygon_2geojson(poly, id, prj_info, save_folder)
def make_note_all_task_done(extent_shp, reomte_node): if os.path.isdir(grid_ids_txt_dir) is False: io_function.mkdir(grid_ids_txt_dir) shp_grid_id_txt, log_grid_ids_txt, log_grid_ids_txt_done = get_extent_grid_id_txt_done_files( extent_shp) # shp_grid_id_txt should be in the current folder if os.path.isfile(log_grid_ids_txt) is False: io_function.copy_file_to_dst(shp_grid_id_txt, log_grid_ids_txt) if os.path.isfile(log_grid_ids_txt_done) is False: io_function.save_list_to_txt(log_grid_ids_txt_done, ['Done']) # copy the curc r_grid_ids_txt_dir = '/scratch/summit/lihu9680/ArcticDEM_tmp_dir/grid_ids_txt' scp_communicate.copy_file_folder_to_remote_machine( reomte_node, r_grid_ids_txt_dir, log_grid_ids_txt_done)
def main(): basic.setlogfile('log_convert_dem_diff_to8bit.txt') if os.path.isdir(grid_dem_diffs_8bit_dir) is False: io_function.mkdir(grid_dem_diffs_8bit_dir) dem_diff_list = io_function.get_file_list_by_pattern(grid_dem_diffs_dir,'*DEM_diff_grid*.tif') count = len(dem_diff_list) failed_tifs = [] for idx, tif in enumerate(dem_diff_list): print('%d/%d convert %s to 8 bit'%(idx+1, count, tif)) tif_8bit = io_function.get_name_by_adding_tail(tif, '8bit') output = os.path.join(grid_dem_diffs_8bit_dir, os.path.basename(tif_8bit)) if dem_tif_to_8bit(tif,output) is False: failed_tifs.append(tif) if len(failed_tifs)>0: io_function.save_list_to_txt('failed_dem_diff_to8bit.txt',failed_tifs)
def test_merge_polygon_for_demDiff_headwall_grids(): dem_subsidence_shp = os.path.expanduser( '~/Data/tmp_data/segment_result_grid9999/ala_north_slo_extent_latlon_grid_ids_DEM_diff_grid9999_8bit_post_final.shp' ) headwall_shp_list = io_function.get_file_list_by_pattern( os.path.expanduser('~/Data/tmp_data/headwall_shps_grid9999'), '*.shp') # merge the results of these two output_dir = os.path.join(grid_dem_subsidence_select, 'subsidence_grid%d' % 9999) if os.path.isdir(output_dir) is False: io_function.mkdir(output_dir) merge_polygon_for_demDiff_headwall_grids(dem_subsidence_shp, headwall_shp_list, output_dir, buffer_size=50)
def segment_subsidence_grey_image_v2(dem_diff_grey_8bit, dem_diff, save_dir,process_num, subsidence_thr_m=-0.5, min_area=40, max_area=100000000): ''' segment subsidence areas based on 8bit dem difference :param dem_diff_grey_8bit: :param dem_diff: :param save_dir: :param process_num: :param subsidence_thr_m: mean value less than this one consider as subsidence (in meter) :param min_area: min size in m^2 (defualt is 40 m^2, 10 pixels on ArcticDEM) :param max_area: min size in m^2 (default is 10km by 10 km) :return: ''' io_function.is_file_exist(dem_diff_grey_8bit) out_pre = os.path.splitext(os.path.basename(dem_diff_grey_8bit))[0] segment_shp_path = os.path.join(save_dir, out_pre + '.shp') # get initial polygons # because the label from segmentation for superpixels are not unique, so we may need to get mean dem diff based on polygons, set org_raster=None label_path_list = segment_a_grey_image(dem_diff_grey_8bit,save_dir,process_num, org_raster=None,b_save_patch_label=True) patch_shp_list = polygonize_label_images(label_path_list, org_raster=dem_diff, stats=['mean', 'std', 'count'], prefix='demD', process_num=process_num, b_remove_nodata=True) # post-processing for each patch shp post_patch_shp_list = [] for idx, shp in enumerate(patch_shp_list): # get DEM diff information for each polygon. post_shp = get_dem_subscidence_polygons(shp, dem_diff, dem_diff_thread_m=subsidence_thr_m, min_area=min_area, max_area=max_area, process_num=1) if post_shp is not None: post_patch_shp_list.append(post_shp) # merge shapefile if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) vector_gpd.merge_shape_files(post_patch_shp_list,segment_shp_path) # post-processing again dem_diff_shp = get_dem_subscidence_polygons(segment_shp_path, dem_diff, dem_diff_thread_m=subsidence_thr_m, min_area=min_area, max_area=max_area, process_num=1) basic.outputlogMessage('obtain elevation reduction polygons: %s'%dem_diff_shp) return True
class TestdeeplabTrainclass(): if os.path.isdir('split_images'): io_function.delete_file_or_dir('split_images') if os.path.isdir('split_labels'): io_function.delete_file_or_dir('split_labels') io_function.mkdir('split_images') io_function.mkdir('split_labels') def test_split_a_pair_sub_image_label(self): ### split the training image to many small patch (480*480) patch_w= 160 # parameters.get_string_parameters(para_file,'train_patch_width') patch_h= 160 #parameters.get_string_parameters(para_file,'train_patch_height') # notes # set overlay as 80, then width or height of patches range from 240 to 320. # so it will generate more patches than 160 ones # overlay_x= 80 # parameters.get_string_parameters(para_file,'train_pixel_overlay_x') # overlay_y= 80 #parameters.get_string_parameters(para_file,'train_pixel_overlay_y') overlay_x= 160 # parameters.get_string_parameters(para_file,'train_pixel_overlay_x') overlay_y= 160 #parameters.get_string_parameters(para_file,'train_pixel_overlay_y') split_image_format= '.png' # parameters.get_string_parameters(para_file,'split_image_format') trainImg_dir= 'subImages' # parameters.get_string_parameters(para_file,'input_train_dir') labelImg_dir= 'subLabels' # parameters.get_string_parameters(para_file,'input_label_dir') if os.path.isdir(trainImg_dir) is False: raise IOError('%s not in the current folder, please get subImages first'%trainImg_dir) if os.path.isdir(labelImg_dir) is False: raise IOError('%s not in the current folder, please get subImages first'%labelImg_dir) # sub_img_label_txt = 'sub_images_labels_list_test.txt' sub_img_label_txt = 'sub_images_labels_list_1.txt' if os.path.isfile(sub_img_label_txt) is False: raise IOError('%s not in the current folder, please get subImages first' % sub_img_label_txt) with open(sub_img_label_txt) as txt_obj: line_list = [name.strip() for name in txt_obj.readlines()] for line in line_list: split_sub_images.split_a_pair_sub_image_label(line, patch_w, patch_h, overlay_x, overlay_y, split_image_format)
def test_darknet_batch_detection_rs_images(): print('\n') print('Run test_darknet_batch_detection_rs_images') config_file = 'yolov4_obj_oneband.cfg' yolo_data = os.path.join('data', 'obj.data') weights = os.path.join('exp1', 'yolov4_obj_oneband_best.weights') batch_size = 1 # these three have the same size # image_names = ['20200818_mosaic_8bit_rgb_p_1001.png'] #, '20200818_mosaic_8bit_rgb_p_1038.png', '20200818_mosaic_8bit_rgb_p_1131.png'] image_names = ['Alaska_north_slope_hillshade_20170426_poly_28_p_0.png'] #, '20200818_mosaic_8bit_rgb_p_1038.png', '20200818_mosaic_8bit_rgb_p_1131.png'] image_names = [os.path.join('debug_img', item) for item in image_names] image_path = image_names[0] save_dir = './' height, width, band_num, date_type = raster_io.get_height_width_bandnum_dtype(image_path) # print('input image: height, width, band_num, date_type',height, width, band_num, date_type) # divide the image the many small patches, then calcuate one by one, solving memory issues. patch_w = 480 patch_h = 480 overlay_x = 160 overlay_y = 160 image_patches = split_image.sliding_window(width,height,patch_w,patch_h,adj_overlay_x=overlay_x,adj_overlay_y=overlay_y) patch_count = len(image_patches) if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) # group patches based on size, each patch is (xoff,yoff ,xsize, ysize) patch_groups = {} for patch in image_patches: wh_str = 'w%d'%patch[2] + '_' + 'h%d'%patch[3] if wh_str in patch_groups.keys(): patch_groups[wh_str].append(patch) else: patch_groups[wh_str] = [patch] # load network network, class_names, class_colors = load_darknet_network(config_file, yolo_data, weights, batch_size=batch_size) darknet_batch_detection_rs_images(network, image_path, save_dir, patch_groups, patch_count, class_names, batch_size, thresh=0.25, hier_thresh=.5, nms=.45)
def test_dataloader(): # test, run in ~/Data/tmp_data/test_mmsegmentation/test_landuse_dl para_file = 'main_para.ini' set_pythonpath(para_file) expr_name = parameters.get_string_parameters(para_file, 'expr_name') trained_model = '%s/latest.pth' % expr_name # test rgb, using rgb in Willow River # img_idx = 0 # image_path = os.path.expanduser('~/Data/Arctic/canada_arctic/Willow_River/Planet2020/20200818_mosaic_8bit_rgb.tif') # img_save_dir = os.path.join('predict_output','I%d' % img_idx) # io_function.mkdir(img_save_dir) # inf_list_file = os.path.join('predict_output','%d.txt'%img_idx) # gpuid = None # predict_one_image_mmseg(para_file, image_path, img_save_dir, inf_list_file, gpuid, trained_model) # # # curr_dir,img_idx, area_save_dir, test_id # curr_dir = os.getcwd() # inf_results_to_shapefile(curr_dir,img_idx,'predict_output','1') ############ test nirGB, using rgb in Willow River # img_idx = 1 # image_path = os.path.expanduser('~/Data/Arctic/canada_arctic/Willow_River/Planet2020/20200818_mosaic_8bit_nirGB.tif') # img_save_dir = os.path.join('predict_output','I%d' % img_idx) # io_function.mkdir(img_save_dir) # inf_list_file = os.path.join('predict_output','%d.txt'%img_idx) # gpuid = None # # predict_one_image_mmseg(para_file, image_path, img_save_dir, inf_list_file, gpuid, trained_model) # # # curr_dir,img_idx, area_save_dir, test_id # curr_dir = os.getcwd() # inf_results_to_shapefile(curr_dir,img_idx,'predict_output','1') ######### try a tiny images img_idx = 2 image_path = os.path.expanduser( '~/Data/test_mmsegmentation/test_landuse_dl/WR_nirGB_sub_images/20200818_mosaic_8bit_nirGB_0_class_1.tif' ) img_save_dir = os.path.join('predict_output', 'I%d' % img_idx) io_function.mkdir(img_save_dir) inf_list_file = os.path.join('predict_output', '%d.txt' % img_idx) gpuid = None predict_one_image_mmseg(para_file, image_path, img_save_dir, inf_list_file, gpuid, trained_model)