def main(): hillshade_dir = os.path.join(work_dir,'hillshade_sub_images') dem_slope_8bit_dir = os.path.join(work_dir,'dem_slope_8bit_sub_images') dem_relative_8bit_dir = os.path.join(work_dir,'dem_relative_8bit_sub_images') other_dirs = [dem_slope_8bit_dir,dem_relative_8bit_dir] other_dirs_tifs = [ io_function.get_file_list_by_ext('.tif', o_dir, bsub_folder=True) for o_dir in other_dirs] json_list = io_function.get_file_list_by_ext('.json', hillshade_dir, bsub_folder=True) json_base_list = [os.path.basename(item) for item in json_list] for json_path, base_name in zip(json_list, json_base_list): date_str, poly_num = get_date_str_poly_num(base_name) for tif_list in other_dirs_tifs: for tif in tif_list: name_noext = io_function.get_name_no_ext(tif) if date_str in name_noext and poly_num in name_noext: # modify and save the json file dst_path = os.path.join(os.path.dirname(tif), name_noext+'.json') # io_function.copy_file_to_dst(json_path,dst_path) data_dict = io_function.read_dict_from_txt_json(json_path) data_dict['imagePath'] = os.path.basename(tif) data_dict['imageData'] = None io_function.save_dict_to_txt_json(dst_path, data_dict) print('saving %s'%dst_path) break pass
def read_precipitation_series(data_folder, station_no): txt_list = io_function.get_file_list_by_ext('.TXT', data_folder, bsub_folder=False) txt_list_2 = io_function.get_file_list_by_ext('.txt', data_folder, bsub_folder=False) txt_list.extend(txt_list_2) # read daily mean, max, and min precipitation date_list, mean_pre, max_pre, min_pre = read_data_series( station_no, txt_list) # convert to mm mean_pre = [tem / 10.0 for tem in mean_pre] # max_pre = [tem / 10.0 for tem in max_pre] min_pre = [tem / 10.0 for tem in min_pre] date_list_new = [] mean_pre_new = [] max_pre_new = [] min_pre_new = [] # remove outliers for date, pre1, pre2, pre3 in zip(date_list, mean_pre, max_pre, min_pre): if pre1 > 1000 or pre2 > 1000 or pre3 > 1000: continue date_list_new.append(date) mean_pre_new.append(pre1) max_pre_new.append(pre2) min_pre_new.append(pre3) return date_list_new, mean_pre_new, max_pre_new, min_pre_new
def main(): ntf_list = io_function.get_file_list_by_ext('.ntf', os.path.join(dir, 'DATA'), bsub_folder=True) io_function.save_list_to_txt('ntf_list.txt', ntf_list) dem_list = io_function.get_file_list_by_ext('.tif', os.path.join(dir, 'PRODUCTS'), bsub_folder=True) dem_list = [ item for item in dem_list if item.endswith('_dem.tif') and 'strips' in item ] io_function.save_list_to_txt('dem_list.txt', dem_list) for idx, ntf in enumerate(ntf_list): print(' (%d/%d) working on ' % (idx + 1, len(ntf_list)), ntf) name = os.path.basename(ntf) scene_id = name.split('_')[2] print('scene_id:', scene_id) dem_path = None for dem_tif in dem_list: if scene_id in os.path.basename(dem_tif): dem_path = dem_tif break if dem_path is None: raise ValueError('Cannot find the corresponding DEM') output = os.path.splitext(name)[0] + '_ortho_sub.tif' ortho_rectified_gdalwarp(ntf, output, dem_path) # break pass
def split_an_image(para_file, image_path, save_dir, patch_w, patch_h, overlay_x, overlay_y): split_format = parameters.get_string_parameters(para_file, 'split_image_format') out_format = 'PNG' # default is PNG if split_format == '.tif': out_format = 'GTIFF' if split_format == '.jpg': out_format = 'JPEG' if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) split_image.split_image(image_path, save_dir, patch_w, patch_h, overlay_x, overlay_y, out_format, pre_name=None, process_num=8) # get list patch_list = io_function.get_file_list_by_ext(split_format, save_dir, bsub_folder=False) if len(patch_list) < 1: print('Wanring, no images in %s' % save_dir) return None list_txt_path = save_dir + '_list.txt' io_function.save_list_to_txt(list_txt_path, patch_list) return list_txt_path
def main(options, args): save_dir = options.save_dir dem_dir_or_txt = args[0] ref_dem = options.ref_dem dem_valid_per_txt = options.dem_valid_per_txt process_num = options.process_num if os.path.isfile(dem_dir_or_txt): dem_list = io_function.read_list_from_txt(dem_dir_or_txt) else: dem_list = io_function.get_file_list_by_ext('.tif', dem_dir_or_txt, bsub_folder=False) if dem_valid_per_txt is None: dem_valid_per_txt = os.path.join(dem_dir_or_txt, 'dem_valid_percent.txt') dem_count = len(dem_list) if dem_count < 1: raise ValueError('No input dem files in %s' % dem_dir_or_txt) if ref_dem is None: ref_dem = choose_reference_dem(dem_list, dem_valid_per_txt) if ref_dem is None: raise ValueError('Cannot find a reference DEM') if ref_dem in dem_list: dem_list.remove(ref_dem) # co_registration_parallel(ref_dem,dem_list,save_dir,process_num) co_registration_multi_process(ref_dem, dem_list, save_dir, process_num)
def read_down_load_geometry(folder): ''' read geojson files in a folder. geojson file stores the geometry of a file, and save to global varialbes :param folder: the save folder :return: ''' global downloaded_scene_geometry json_list = io_function.get_file_list_by_ext('.geojson', folder, bsub_folder=False) for json_file in json_list: # ignore the scenes in the excluded list item_id = os.path.splitext(os.path.basename(json_file))[0] if item_id in manually_excluded_scenes: continue scene_folder = os.path.splitext(json_file)[0] asset_files = io_function.get_file_list_by_pattern(scene_folder, '*') if len(asset_files) < 3: basic.outputlogMessage( 'downloading of scene %s is not compelte, ignore it' % item_id) continue with open(json_file) as json_file: data = json.load(json_file) # p(data) # test downloaded_scene_geometry.append(data)
def main(options, args): # folder containing images (download from Google Earth Engine) img_folder = args[0] img_file_list = io_function.get_file_list_by_ext('.tif', img_folder, bsub_folder=False) # img_file_list = img_file_list[:2] # for test # ndvi # batch_cal_msi(img_file_list, 'landsat8_ndvi.tif', cal_ndvi_landsat8) # ndwi # batch_cal_msi(img_file_list, 'landsat8_ndwi.tif', cal_ndwi_landsat8) # ndmi # batch_cal_msi(img_file_list, 'landsat8_ndmi.tif', cal_ndmi_landsat8) # brightness # batch_cal_msi(img_file_list, 'landsat8_brightness.tif', cal_brightness_landsat8) # greenness batch_cal_msi(img_file_list, 'landsat8_greenness.tif', cal_greenness_landsat8) # wetness batch_cal_msi(img_file_list, 'landsat8_wetness.tif', cal_wetness_landsat8) pass
def main(options, args): in_folder = args[0] template_ini = args[1] image_paths = io_function.get_file_list_by_ext('.tif', in_folder, bsub_folder=True) if len(image_paths) < 1: raise IOError('no tif files in %s' % in_folder) # get unique dir list img_dir_list = [os.path.dirname(item) for item in image_paths] img_dir_list = set(img_dir_list) region_ini_files_list = [] for img_dir in img_dir_list: # copy template file out_ini = create_new_region_defined_parafile(template_ini, img_dir, options.area_remark) region_ini_files_list.append(out_ini) with open('region_ini_files.txt', 'a') as f_obj: for ini in region_ini_files_list: f_obj.writelines(ini + '\n') pass
def main(options, args): save_dir = options.save_dir b_rm_inter = options.remove_inter_data b_rm_tarball = options.remove_tarball tar_dir = args[0] if os.path.isfile(tar_dir): tar_list = [tar_dir] else: tar_list = io_function.get_file_list_by_ext('.gz', tar_dir, bsub_folder=False) tar_count = len(tar_list) if tar_count < 1: raise ValueError('No input tar.gz files in %s' % tar_dir) if is_ArcticDEM_tiles(tar_list): apply_registration = False else: apply_registration = True work_dir = './' b_rm_tarball = True # remove tarball after unpacking process_dem_tarball(tar_list, work_dir, save_dir, remove_inter_data=b_rm_inter, rm_tarball=b_rm_tarball, apply_registration=apply_registration)
def main(options, args): input = args[0] if input.endswith('.txt'): slope_tifs = io_function.read_list_from_txt(input) elif os.path.isdir(input): slope_tifs = io_function.get_file_list_by_ext('.tif',input, bsub_folder=True) else: slope_tifs = [ input] process_num = options.process_num working_dir = './' save_dir = dem_headwall_shp_dir if os.path.isdir(working_dir) is False: io_function.mkdir(working_dir) if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) failed_tifs = [] min_slope = options.min_slope min_size = options.min_area max_size = options.max_area max_axis_width = options.max_axis_width max_box_WH = options.max_box_WH for idx, slope in enumerate(slope_tifs): if extract_headwall_from_slope(idx, len(slope_tifs), slope,working_dir,save_dir, min_slope,min_size,max_size,max_axis_width,max_box_WH,process_num) is False: failed_tifs.append(slope) io_function.save_list_to_txt('extract_headwall_failed_tifs.txt',failed_tifs)
def run_extract_headwall_jobs(max_job_count, n_tif_per_jobs): from dem_common import dem_headwall_shp_dir, dem_slope_dir if os.path.isdir(dem_headwall_shp_dir) is False: io_function.mkdir(dem_headwall_shp_dir) # get slope file list slope_tifs = io_function.get_file_list_by_ext('.tif', dem_slope_dir, bsub_folder=False) print('Found %d tif in %s' % (len(slope_tifs), dem_slope_dir)) # divide grid_ids to many groups slope_tif_count = len(slope_tifs) slope_tif_groups = [ slope_tifs[i:i + n_tif_per_jobs] for i in range(0, slope_tif_count, n_tif_per_jobs) ] for idx, slope_tifs_group in enumerate(slope_tif_groups): print( datetime.now(), 'processing %d group for extracting headwall, total %d ones' % (idx, len(slope_tif_groups))) submit_extract_headwall_job(slope_tifs_group, idx, max_job_count)
def main(options, args): # folder containing images (download from Google Earth Engine) # img_folder = args[0] img_folder = '/Users/huanglingcao/Data/Qinghai-Tibet/beiluhe/beiluhe_landsat/LT05_2010to2011' img_file_list = io_function.get_file_list_by_ext('.tif',img_folder,bsub_folder=False) # img_file_list = img_file_list[:2] # for test satellite = 'landsat5' # #ndvi batch_cal_msi(img_file_list, satellite+'_ndvi.tif', cal_ndvi_landsat5) #ndwi batch_cal_msi(img_file_list, satellite+'_ndwi.tif', cal_ndwi_landsat5) #ndmi batch_cal_msi(img_file_list, satellite+'_ndmi.tif', cal_ndmi_landsat5) #brightness batch_cal_msi(img_file_list, satellite+'_brightness.tif', cal_brightness_landsat5) # greenness batch_cal_msi(img_file_list, satellite+'_greenness.tif', cal_greenness_landsat5) # wetness batch_cal_msi(img_file_list, satellite+'_wetness.tif', cal_wetness_landsat5) pass
def test_zonal_stats_multiRasters(): shp = os.path.expanduser( '~/Data/Arctic/canada_arctic/Willow_River/Willow_River_Thaw_Slumps.shp' ) # save_shp = os.path.basename(io_function.get_name_by_adding_tail(shp,'raster_stats')) # a single DEM # dem_file_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/DEM/WR_dem_ArcticDEM_mosaic') # dem_path = os.path.join(dem_file_dir,'WR_extent_2m_v3.0_ArcticTileDEM_sub_1_prj.tif') # dem patches dem_file_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/DEM/WR_dem_ArcticDEM_mosaic/dem_patches') dem_list = io_function.get_file_list_by_ext('.tif', dem_file_dir, bsub_folder=False) save_shp = os.path.basename( io_function.get_name_by_adding_tail(shp, 'multi_raster_stats')) io_function.copy_shape_file(shp, save_shp) zonal_stats_multiRasters(save_shp, dem_list, nodata=None, band=1, stats=None, prefix='dem', range=None, all_touched=True, process_num=4)
def main(options, args): # data_ini_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/WR_multiDate_inis') # training_root_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/ray_results/tune_dataAug_para_tesia') # template_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/eval_new_data') data_ini_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/autoMapping/area_multiDate_inis') training_root_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/autoMapping/multiArea_deeplabV3+_8') template_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/autoMapping/multiArea_deeplabV3+_8') if options.data_ini_dir_or_list is not None: data_ini_dir = options.data_ini_dir_or_list if options.training_root_dir is not None: training_root_dir = options.training_root_dir if options.template_dir is not None: template_dir = options.template_dir # get data list if os.path.isdir(data_ini_dir): area_ini_list = io_function.get_file_list_by_ext('.ini', data_ini_dir, bsub_folder=False) else: area_ini_list = io_function.read_list_from_txt(data_ini_dir) # change to abslute path, since later, directory will be changed area_ini_list = [os.path.abspath(item) for item in area_ini_list] for idx, area_ini in enumerate(area_ini_list): basic.outputlogMessage('%d/%d evaluation on %d areas' % (idx, len(area_ini_list), idx)) run_evaluation_one_dataset(idx, area_ini, training_root_dir, template_dir)
def main(options, args): input_path = args[0] if os.path.isdir(input_path) is True: if options.ext is None: basic.outputlogMessage( 'image file extenstion is need, type help for more information' ) return False file_list = io_function.get_file_list_by_ext(options.ext, input_path, bsub_folder=True) f_obj = open('images_list.txt', 'w') f_obj.writelines(["%s\n" % item for item in file_list]) f_obj.close() elif os.path.isfile(input_path): f_obj = open(input_path, 'r') file_list = f_obj.readlines() f_obj.close() else: basic.outputlogMessage('input error: %s' % input_path) return False for i in range(0, len(file_list)): file_list[i] = file_list[i].strip() return calculate_mean_of_images(file_list)
def main(options, args): org_dir = '/scratch/summit/lihu9680' bak_dir = '/home/lihu9680/scripts_para_bak' if options.org_dir is not None: org_dir = options.org_dir if options.bak_dir is not None: bak_dir = options.bak_dir print('org_dir:', org_dir) print('bak_dir:', bak_dir) # get bash file (*.sh) list print('find *.sh and *.ini files') sh_list = io_function.get_file_list_by_ext('.sh', org_dir, bsub_folder=True) ini_list = io_function.get_file_list_by_ext('.ini', org_dir, bsub_folder=True) sh_list.extend(ini_list) changed_list = [] for sh in sh_list: new_path = bak_dir + sh.replace(org_dir, '') # print(bak_path) mo_time = os.path.getmtime(sh) if os.path.isfile(new_path): bak_mo_time = os.path.getmtime(new_path) if mo_time == bak_mo_time: continue new_dir = os.path.dirname(new_path) if os.path.isdir(new_dir) is False: os.system('mkdir -p %s' % new_dir) res = os.system('cp -p %s %s' % (sh, new_path)) if res != 0: sys.exit(1) changed_list.append(sh) if len(changed_list) < 1: print("no new or modified files") else: print("backup files:") for sh in changed_list: print(sh)
def read_training_pixels_from_multi_images(input, subImg_folder, subLabel_folder): """ read pixels from subset images, which are extracted from Planet images based on trainig polygons :param subImg_folder: the folder containing images :param subLabel_folder: the folder containing labels :return: X, y arrays or None """ img_list = io_function.get_file_list_by_ext('.tif', subImg_folder, bsub_folder=False) label_list = io_function.get_file_list_by_ext('.tif', subLabel_folder, bsub_folder=False) img_list.sort() label_list.sort() if len(img_list) < 1 or len(label_list) < 1: raise IOError('No tif images or labels in folder %s or %s' % (subImg_folder, subLabel_folder)) if len(img_list) != len(label_list): raise ValueError( 'the number of images is not equal to the one of labels') # read them one by one Xs, ys = [], [] for img, label in zip(img_list, label_list): # # test by hlc # polygon_index_img = os.path.basename(img).split('_')[-3] # # print(polygon_index_img) # if polygon_index_img not in [str(83), str(86)] : # continue X_aImg, y_a = read_training_pixels(img, label) Xs.append(X_aImg) ys.append(y_a) X_pixels = np.concatenate(Xs, axis=1) y_pixels = np.concatenate(ys, axis=0) X_pixels = np.transpose(X_pixels, (1, 0)) basic.outputlogMessage(str(X_pixels.shape)) basic.outputlogMessage(str(y_pixels.shape)) return X_pixels, y_pixels
def main(): # get data list area_ini_list = io_function.get_file_list_by_ext('.ini', data_ini_dir, bsub_folder=False) for idx, area_ini in enumerate(area_ini_list): basic.outputlogMessage('%d/%d evaluation on %d areas' % (idx, len(area_ini_list), idx)) run_evaluation_one_dataset(idx, area_ini)
def main(options, args): extent_shp_or_ids_txt = args[0] process_num = options.process_num o_res = options.out_res if os.path.isdir(grid_matchtag_sum_dir) is False: io_function.mkdir(grid_matchtag_sum_dir) basic.setlogfile('produce_matchtag_sum_ArcticDEM_log_%s.txt' % timeTools.get_now_time_str()) # read grids and ids time0 = time.time() all_grid_polys, all_ids = vector_gpd.read_polygons_attributes_list( grid_20_shp, 'id') print('time cost of read polygons and attributes', time.time() - time0) # get grid ids based on input extent grid_base_name = os.path.splitext( os.path.basename(extent_shp_or_ids_txt))[0] grid_polys, grid_ids = get_grid_20(extent_shp_or_ids_txt, all_grid_polys, all_ids) # check dem difference existence grid_dem_tifs, grid_ids_no_sum = get_existing_matchtag_sum( grid_matchtag_sum_dir, grid_base_name, grid_ids) if len(grid_ids_no_sum) > 0: # refine grid_polys if len(grid_ids) > len(grid_ids_no_sum): id_index = [grid_ids.index(id) for id in grid_ids_no_sum] grid_polys = [grid_polys[idx] for idx in id_index] # # download ArcticDEM and applying registration # tarballs, reg_tifs = download_dem_tarball(dem_strip_shp, grid_polys, arcticDEM_tarball_dir, grid_base_name, # reg_tif_dir=arcticDEM_reg_tif_dir, poly_ids=grid_ids_no_demDiff) # # # unpack and applying registration # if len(tarballs) > 0: # basic.outputlogMessage('Processs %d dem tarballs'%len(tarballs)) # out_reg_tifs = process_dem_tarball(tarballs,'./',arcticDEM_reg_tif_dir,remove_inter_data=True, apply_registration=True) # basic.outputlogMessage('Get %d new registration dem tifs' % len(out_reg_tifs)) # reg_tifs.extend(out_reg_tifs) reg_tifs = io_function.get_file_list_by_ext('.tif', arcticDEM_reg_tif_dir, bsub_folder=False) matchtag_tifs = [tif for tif in reg_tifs if 'matchtag' in tif] # only keep matchtag # crop, sum out_dem_diffs = produce_matchtag_sum_grids(grid_polys, grid_ids_no_sum, grid_base_name, matchtag_tifs, o_res, process_num=process_num)
def create_new_region_defined_parafile(template_para_file, img_dir, area_remark=None): ''' create a new region defined para file. Only defined the new images (did not change others) :param template_para_file: :param img_dir: :param area_remark: :return: ''' io_function.is_file_exist(template_para_file) dir_base = os.path.basename(img_dir) date_strs = re.findall('\d{8}', dir_base) if len(date_strs) == 1: date = date_strs[0] else: date = 'unknown' new_para_file = io_function.get_name_by_adding_tail( template_para_file, date + '_' + area_remark) new_para_file = os.path.basename(new_para_file) # save to current folder if os.path.isfile(new_para_file): raise IOError('%s already exists, please check or remove first' % new_para_file) # copy the file io_function.copy_file_to_dst(template_para_file, new_para_file) if area_remark is not None: modify_parameter(new_para_file, 'area_remark', area_remark) if date != 'unknown': modify_parameter(new_para_file, 'area_time', date) modify_parameter(new_para_file, 'input_image_dir', img_dir) modify_parameter(new_para_file, 'inf_image_dir', img_dir) tif_list = io_function.get_file_list_by_ext('.tif', img_dir, bsub_folder=False) if len(tif_list) < 1: raise ValueError('No tif in %s' % img_dir) if len(tif_list) == 1: modify_parameter(new_para_file, 'input_image_or_pattern', os.path.basename(tif_list[0])) modify_parameter(new_para_file, 'inf_image_or_pattern', os.path.basename(tif_list[0])) else: modify_parameter(new_para_file, 'input_image_or_pattern', '*.tif') modify_parameter(new_para_file, 'inf_image_or_pattern', '*.tif') print("modified and saved new parameter file: %s " % new_para_file) return new_para_file
def read_grid_ids_from_other_extent(): grid_txt_list = io_function.get_file_list_by_ext('.txt', grid_ids_txt_dir, bsub_folder=False) other_grid_ids = [] for txt in grid_txt_list: id_list = io_function.read_list_from_txt(txt) other_grid_ids.extend(id_list) other_grid_ids = [int(item) for item in other_grid_ids] return other_grid_ids
def read_temperature_series(data_folder, station_no): txt_list = io_function.get_file_list_by_ext('.TXT', data_folder, bsub_folder=False) txt_list_2 = io_function.get_file_list_by_ext('.txt', data_folder, bsub_folder=False) txt_list.extend(txt_list_2) # read daily mean, max, and min temperature date_list, mean_tem, max_tem, min_tem = read_data_series( station_no, txt_list) # convert to degree mean_tem = [tem / 10.0 for tem in mean_tem] # ignore if tem < 1000 and tem > -1000 max_tem = [tem / 10.0 for tem in max_tem] min_tem = [tem / 10.0 for tem in min_tem] return date_list, mean_tem, max_tem, min_tem
def test_remove_based_slope(): dir = os.path.expanduser('~/Data/dem_processing/segment_parallel_9274') in_shp = os.path.join(dir,'WR_extent_grid_ids_DEM_diff_grid9274_8bit.shp') rm_shapeinfo_shp = os.path.join(dir,'WR_extent_grid_ids_DEM_diff_grid9274_8bit_rmshapeinfo.shp') process_num = 1 # remove based on slope # use the slope derived from ArcitcDEM mosaic slope_tif_list = io_function.get_file_list_by_ext('.tif',dem_common.arcticDEM_tile_slope_dir,bsub_folder=False) basic.outputlogMessage('Find %d slope files in %s'%(len(slope_tif_list), dem_common.arcticDEM_tile_slope_dir)) rm_slope_shp = io_function.get_name_by_adding_tail(in_shp, 'rmslope') max_slope = 20 remove_based_slope(rm_shapeinfo_shp, rm_slope_shp,slope_tif_list, max_slope,process_num)
def test_get_dem_tif_ext_polygons(): work_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/DEM/WR_dem_diff/dem_tifs') os.chdir(work_dir) tifs = io_function.get_file_list_by_ext('.tif', work_dir, bsub_folder=False) polygons = dem_mosaic_crop.get_dem_tif_ext_polygons(tifs) data = {'poly': polygons} pddata = pd.DataFrame(data) wkt_str = map_projection.get_raster_or_vector_srs_info_wkt(tifs[0]) save_path = 'tif_extent.shp' vector_gpd.save_polygons_to_files(pddata, 'poly', wkt_str, save_path)
def main(options, args): extent_shp_or_ids_txt = args[0] process_num = options.process_num keep_dem_percent = options.keep_dem_percent o_res = options.out_res basic.setlogfile('produce_headwall_shp_ArcticDEM_log_%s.txt' % timeTools.get_now_time_str()) if os.path.isdir(grid_dem_headwall_shp_dir) is False: io_function.mkdir(grid_dem_headwall_shp_dir) # read grids and ids time0 = time.time() all_grid_polys, all_ids = vector_gpd.read_polygons_attributes_list( grid_20_shp, 'id') print('time cost of read polygons and attributes', time.time() - time0) # get grid ids based on input extent grid_base_name = os.path.splitext( os.path.basename(extent_shp_or_ids_txt))[0] grid_polys, grid_ids = get_grid_20(extent_shp_or_ids_txt, all_grid_polys, all_ids) # check dem difference existence grid_headwall_shps, grid_id_no_headwall_shp = get_existing_grid_headwall_shp( grid_dem_headwall_shp_dir, grid_base_name, grid_ids) if len(grid_id_no_headwall_shp) > 0: # refine grid_polys if len(grid_ids) > len(grid_id_no_headwall_shp): id_index = [grid_ids.index(id) for id in grid_id_no_headwall_shp] grid_polys = [grid_polys[idx] for idx in id_index] reg_tifs = io_function.get_file_list_by_ext('.tif', arcticDEM_reg_tif_dir, bsub_folder=False) reg_tifs = [tif for tif in reg_tifs if 'matchtag' not in tif] # remove matchtag # headwall_shp_folders = extract_headwall_grids(grid_polys, grid_id_no_headwall_shp, grid_base_name, reg_tifs, b_mosaic_id, b_mosaic_date, keep_dem_percent, o_res, process_num=process_num)
def main(options, args): # process_num = multiprocessing.cpu_count() process_num = options.process_num if len(args) > 0: data_folder = args[0] else: data_folder = grid_dem_headwall_shp_dir basic.outputlogMessage('check shapefiles in %s, with %d processes'%(data_folder,process_num)) vector_files = io_function.get_file_list_by_ext('.shp',data_folder ,bsub_folder=True) save_invalid_txt_path = os.path.basename(data_folder) + '_incomplete_list.txt' save_good_txt_path = os.path.basename(data_folder) + '_good_list.txt' file_count = len(vector_files) basic.outputlogMessage('The number of vector files: %d'%file_count) good_files = [] if os.path.isfile(save_good_txt_path): good_files.extend(io_function.read_list_from_txt(save_good_txt_path)) incomplete_files = [] # remove good one for the list if len(good_files)>0: vector_files = [item for item in vector_files if os.path.basename(item) not in good_files] if process_num == 1: # tifs = io_function.get_file_list_by_ext('.tif',arcticDEM_reg_tif_dir, bsub_folder=False) for idx, tif in enumerate(vector_files): if check_one_vector_file(idx, file_count, tif, good_files): good_files.append(os.path.basename(tif)) else: incomplete_files.append(os.path.basename(tif)) else: theadPool = Pool(process_num) # multi processes parameters_list = [(idx, file_count, tif, good_files) for idx, tif in enumerate(vector_files)] results = theadPool.starmap(check_one_vector_file, parameters_list) # need python3 for tif, res in zip(vector_files, results): if res: good_files.append(os.path.basename(tif)) else: incomplete_files.append(os.path.basename(tif)) theadPool.close() io_function.save_list_to_txt(save_invalid_txt_path, incomplete_files) io_function.save_list_to_txt(save_good_txt_path, good_files)
def plot_valid_entropy(in_folder, save_file_pre=None): if save_file_pre is None: save_file_pre = os.path.basename(in_folder) logfile = basic.logfile basic.setlogfile(save_file_pre + 'hist_info.txt') image_paths = io_function.get_file_list_by_ext('.tif', in_folder, bsub_folder=True) if len(image_paths) < 1: raise IOError('no tif files in %s' % in_folder) valid_per_list = [] entropy_list = [] img_count = len(image_paths) for idx, img_path in enumerate(image_paths): print('%d/%d' % (idx + 1, img_count)) valid_per, entropy = raster_io.get_valid_percent_shannon_entropy( img_path, log_base=10) valid_per_list.append(valid_per) entropy_list.append(entropy) per_entropy_txt = save_file_pre + '_' + 'valid_per_entropy.txt' save_hist_path = save_file_pre + '_' + 'hist.jpg' with open(per_entropy_txt, 'w') as f_obj: for path, per, entropy in zip(image_paths, valid_per_list, entropy_list): f_obj.writelines( os.path.basename(path) + ' %.4f %.6f \n' % (per, entropy)) # plot the histogram fig = plt.figure(figsize=(6, 4)) # ax1 = fig.add_subplot(111) n, bins, patches = plt.hist(x=entropy_list, bins=50, color='b', rwidth=0.85) # print(n, bins, patches) plt.savefig(save_hist_path, dpi=200) # 300 histogram2logfile(entropy_list, bins, hist_tag=save_hist_path) basic.setlogfile(logfile) # change log file name back return save_hist_path
def get_failed_grid_ids(task): if task == 'dem_diff': fail_log_dir = os.path.join(process_log_dir, 'get_dem_diff') elif task == 'dem_headwall_grid': fail_log_dir = os.path.join(process_log_dir, 'extract_headwall_from_slope_grid') else: raise ValueError('Unknow task: %s' % str(task)) fail_id_txt_list = io_function.get_file_list_by_ext('.txt', fail_log_dir, bsub_folder=False) old_id_txts = [] # check they are old enough (24 hours) for txt in fail_id_txt_list: if check_file_or_dir_is_old(txt, 24, print_time=True): old_id_txts.append(txt) return old_id_txts
def predict_remoteSensing_image(para_file, image_path, save_dir,model, config_file, yolo_data, batch_size=1, b_python_api=True): ''' run prediction of a remote sensing using yolov4 :param image_path: :param model: :param config_file: :param yolo_data: :param batch_size: :param b_python_api: if true, use the python API of yolo :return: ''' patch_w = parameters.get_digit_parameters(para_file, "inf_patch_width", 'int') patch_h = parameters.get_digit_parameters(para_file, "inf_patch_height", 'int') overlay_x = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_x", 'int') overlay_y = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_y", 'int') if b_python_api: # using the python API predict_rs_image_yolo_poythonAPI(image_path, save_dir, model, config_file, yolo_data, patch_w, patch_h, overlay_x, overlay_y, batch_size=batch_size) # for each patch has a json file, may end up with a lot of json files, affect I/O # try to merge them to one json file. res_json_files = io_function.get_file_list_by_ext('.json', save_dir, bsub_folder=False) merge_josn_path = os.path.join(save_dir,'all_patches.json') merge_patch_json_files_to_one(res_json_files,merge_josn_path) for f_json in res_json_files: io_function.delete_file_or_dir(f_json) else: # divide image the many patches, then run prediction. patch_list_txt = split_an_image(para_file,image_path,save_dir,patch_w,patch_h,overlay_x,overlay_y) if patch_list_txt is None: return False result_json = save_dir + '_result.json' commond_str = 'darknet detector test ' + yolo_data + ' ' + config_file + ' ' + model + ' -dont_show ' commond_str += ' -ext_output -out ' + result_json + ' < ' + patch_list_txt print(commond_str) res = os.system(commond_str) if res !=0: sys.exit(1)
def read_down_load_geometry(folder): ''' read geojson files in a folder. geojson file stores the geometry of a file, and save to global varialbes :param folder: the save folder :return: ''' global downloaed_scene_geometry json_list = io_function.get_file_list_by_ext('.geojson', folder, bsub_folder=False) for json_file in json_list: # ignore the scenes in the excluded list item_id = os.path.splitext(os.path.basename(json_file))[0] if item_id in manually_excluded_scenes: continue with open(json_file) as json_file: data = json.load(json_file) # p(data) # test downloaed_scene_geometry.append(data)