def get_complete_ignore_grid_ids(): id_list = [] if os.path.isfile(grid_complete_list_txt): completed_id_list = [ int(item) for item in io_function.read_list_from_txt(grid_complete_list_txt) ] id_list.extend(completed_id_list) # get manual excluded ids if os.path.isfile(grid_excluded_list_txt): exclude_id_list = [ int(item) for item in io_function.read_list_from_txt(grid_excluded_list_txt) ] id_list.extend(exclude_id_list) # get ids that don't have DEM if os.path.isfile(grid_no_dem_txt): nodem_id_list = [ int(item) for item in io_function.read_list_from_txt(grid_no_dem_txt) ] id_list.extend(nodem_id_list) # get ids that don't have DEM if os.path.isfile(grid_no_valid_dem_txt): no_valid_dem_id_list = [ int(item) for item in io_function.read_list_from_txt(grid_no_valid_dem_txt) ] id_list.extend(no_valid_dem_id_list) return id_list
def b_exist_grid_dem_subsidence(id): # if an grid don't have dem subsidence, then we think it's complete if os.path.isfile(grid_no_subscidence_poly_txt): grid_ids_no_subsidence = [ int(item) for item in io_function.read_list_from_txt( grid_no_subscidence_poly_txt) ] if id in grid_ids_no_subsidence: return True # if an id don't have enough dem for dem diff, then we think it's complete if os.path.isfile(grid_dem_diff_less2dem_txt): grid_ids_less_2dem = [ int(item) for item in io_function.read_list_from_txt( grid_dem_diff_less2dem_txt) ] if id in grid_ids_less_2dem: return True dem_subsidence_shps = io_function.get_file_list_by_pattern( grid_dem_diffs_segment_dir, '*_grid%d/*_grid%d_8bit_post.shp' % (id, id)) if len(dem_subsidence_shps) == 1: return True elif len(dem_subsidence_shps) > 1: basic.outputlogMessage( 'warning, There are multiple DEM subsidence for grid: %d' % id) for item in dem_subsidence_shps: basic.outputlogMessage(item) return True else: return False
def main(options, args): # data_ini_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/WR_multiDate_inis') # training_root_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/ray_results/tune_dataAug_para_tesia') # template_dir = os.path.expanduser('~/Data/Arctic/canada_arctic/autoMapping/eval_new_data') data_ini_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/autoMapping/area_multiDate_inis') training_root_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/autoMapping/multiArea_deeplabV3+_8') template_dir = os.path.expanduser( '~/Data/Arctic/canada_arctic/autoMapping/multiArea_deeplabV3+_8') if options.data_ini_dir_or_list is not None: data_ini_dir = options.data_ini_dir_or_list if options.training_root_dir is not None: training_root_dir = options.training_root_dir if options.template_dir is not None: template_dir = options.template_dir # get data list if os.path.isdir(data_ini_dir): area_ini_list = io_function.get_file_list_by_ext('.ini', data_ini_dir, bsub_folder=False) else: area_ini_list = io_function.read_list_from_txt(data_ini_dir) # change to abslute path, since later, directory will be changed area_ini_list = [os.path.abspath(item) for item in area_ini_list] for idx, area_ini in enumerate(area_ini_list): basic.outputlogMessage('%d/%d evaluation on %d areas' % (idx, len(area_ini_list), idx)) run_evaluation_one_dataset(idx, area_ini, training_root_dir, template_dir)
def get_train_val_sample_count(work_dir, para_file): train_sample_txt = parameters.get_string_parameters( para_file, 'training_sample_list_txt') val_sample_txt = parameters.get_string_parameters( para_file, 'validation_sample_list_txt') train_list_txt = os.path.join(work_dir, 'list', train_sample_txt) val_list_txt = os.path.join(work_dir, 'list', val_sample_txt) train_lines = io_function.read_list_from_txt(train_list_txt) val_lines = io_function.read_list_from_txt(val_list_txt) basic.outputlogMessage( 'The count of training and validation samples are %d and %d' % (len(train_lines), len(val_lines))) return len(train_lines), len(val_lines)
def merge_polygons_patchBYpatch(patch_DN_range_txt, in_polygons, polygon_DNs, process_num=1): patch_DN_range_list = [ int(item) for item in io_function.read_list_from_txt(patch_DN_range_txt)] # print(patch_DN_range_list) # divide polygons patch_polygons = {} range_count = len(patch_DN_range_list) - 1 for idx in range(range_count): patch_polygons[idx] = [] for poly, dn in zip(in_polygons, polygon_DNs): for idx in range(range_count): if dn > patch_DN_range_list[idx] and dn <= patch_DN_range_list[idx+1]: patch_polygons[idx].append(poly) # merge polygon patch by patch polygons_patch_merge = [] for idx in range(range_count): print(timeTools.get_now_time_str(), 'will merge %d polygons for %d th patch'%(len(patch_polygons[idx]),idx)) if len(patch_polygons[idx]) < 2: polygons_patch_merge.extend(patch_polygons[idx]) continue adjacent_matrix = vector_gpd.build_adjacent_map_of_polygons(patch_polygons[idx], process_num=process_num) if adjacent_matrix is False: polygons_patch_merge.extend(patch_polygons[idx]) continue merged_polygons = vector_features.merge_touched_polygons(patch_polygons[idx], adjacent_matrix) polygons_patch_merge.extend(merged_polygons) # merge polygon all print(timeTools.get_now_time_str(), 'will merge %d polygons for the entire raster' % (len(polygons_patch_merge))) adjacent_matrix = vector_gpd.build_adjacent_map_of_polygons(polygons_patch_merge, process_num=process_num) if adjacent_matrix is False: return polygons_patch_merge last_merged_polygons = vector_features.merge_touched_polygons(polygons_patch_merge, adjacent_matrix) return last_merged_polygons
def get_dem_diff_list_to_seg(): dem_diff_list = io_function.get_file_list_by_pattern( dem_common.grid_dem_diffs_dir, '*DEM_diff_grid*.tif') dem_diff_list_copy = dem_diff_list.copy() if len(dem_diff_list) < 1: print('No *DEM_diff*.tif in %s' % dem_common.grid_dem_diffs_dir) return [] dem_diff_ids = [ int(re.findall('grid\d+', os.path.basename(item))[0][4:]) for item in dem_diff_list ] dem_subsidence_shps = io_function.get_file_list_by_pattern( dem_common.grid_dem_diffs_segment_dir, 'segment_result*/*_post.shp') subsidence_ids = [ int(re.findall('grid\d+', os.path.basename(item))[0][4:]) for item in dem_subsidence_shps ] no_subsidence_ids = [] if os.path.isfile(dem_common.grid_no_subscidence_poly_txt): no_subsidence_ids = [ int(item) for item in io_function.read_list_from_txt( dem_common.grid_no_subscidence_poly_txt) ] subsidence_ids.extend(no_subsidence_ids) # remove dem diff already been segmented or no subsidence for id, dem_diff in zip(dem_diff_ids, dem_diff_list_copy): if id in subsidence_ids: dem_diff_list.remove(dem_diff) return dem_diff_list
def update_complete_grid_list(grid_ids, task_list): # based on some criteria, to check if results exist, then update grid_complete_list_txt completed_id_list = [] if os.path.isfile(grid_complete_list_txt): completed_id_list = [ int(item) for item in io_function.read_list_from_txt(grid_complete_list_txt) ] n_task = len(task_list) if n_task < 1: raise ValueError('No task in %s' % str(task_list)) for g_id in grid_ids: if g_id in completed_id_list: continue # check if it has been completed based on multiple criteria complete_count = 0 if 'dem_diff' in task_list and b_exist_gid_dem_diff(g_id): complete_count += 1 if 'hillshade_headwall_line' in task_list and b_exist_dem_hillshade_newest_HWLine_grid( g_id): complete_count += 1 if 'dem_headwall_grid' in task_list and b_exist_grid_headwall_shp( g_id): complete_count += 1 if 'segment' in task_list and b_exist_grid_dem_subsidence(g_id): complete_count += 1 # we may check more task results: segment, dem_headwall if complete_count == n_task: completed_id_list.append(g_id) # save the txt completed_id_list = [str(item) for item in completed_id_list] io_function.save_list_to_txt(grid_complete_list_txt, completed_id_list)
def main(): # run in ~/Data/Arctic/canada_arctic/autoMapping/multiArea_sub_images on tesia ini_list = io_function.get_file_list_by_pattern('./','area*.ini') txt_list = io_function.get_file_list_by_pattern('./','area*.txt') for txt in txt_list: ini_s = io_function.read_list_from_txt(txt) ini_list.extend(ini_s) ini_list = [os.path.abspath(item) for item in ini_list] file_names = [ io_function.get_name_no_ext(item) for item in ini_list ] cur_dir = os.getcwd() # show [print(item) for item in ini_list] time.sleep(3) for name, area_ini in zip(file_names,ini_list): word_dir = os.path.join(cur_dir,name) io_function.mkdir(word_dir) os.chdir(word_dir) # copy and modify main_para.ini io_function.copyfiletodir(os.path.join(cur_dir,'main_para.ini'),'./',overwrite=True) io_function.copyfiletodir(os.path.join(cur_dir,'exe.sh'),'./',overwrite=True) parameters.write_Parameters_file('main_para.ini','training_regions',area_ini) # run exe.sh res = os.system('./exe.sh') if res !=0: print(res) sys.exit(1) os.chdir(cur_dir)
def main(options, args): save_dir = options.save_dir dem_dir_or_txt = args[0] ref_dem = options.ref_dem dem_valid_per_txt = options.dem_valid_per_txt process_num = options.process_num if os.path.isfile(dem_dir_or_txt): dem_list = io_function.read_list_from_txt(dem_dir_or_txt) else: dem_list = io_function.get_file_list_by_ext('.tif', dem_dir_or_txt, bsub_folder=False) if dem_valid_per_txt is None: dem_valid_per_txt = os.path.join(dem_dir_or_txt, 'dem_valid_percent.txt') dem_count = len(dem_list) if dem_count < 1: raise ValueError('No input dem files in %s' % dem_dir_or_txt) if ref_dem is None: ref_dem = choose_reference_dem(dem_list, dem_valid_per_txt) if ref_dem is None: raise ValueError('Cannot find a reference DEM') if ref_dem in dem_list: dem_list.remove(ref_dem) # co_registration_parallel(ref_dem,dem_list,save_dir,process_num) co_registration_multi_process(ref_dem, dem_list, save_dir, process_num)
def main(options, args): input = args[0] if input.endswith('.txt'): slope_tifs = io_function.read_list_from_txt(input) elif os.path.isdir(input): slope_tifs = io_function.get_file_list_by_ext('.tif',input, bsub_folder=True) else: slope_tifs = [ input] process_num = options.process_num working_dir = './' save_dir = dem_headwall_shp_dir if os.path.isdir(working_dir) is False: io_function.mkdir(working_dir) if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) failed_tifs = [] min_slope = options.min_slope min_size = options.min_area max_size = options.max_area max_axis_width = options.max_axis_width max_box_WH = options.max_box_WH for idx, slope in enumerate(slope_tifs): if extract_headwall_from_slope(idx, len(slope_tifs), slope,working_dir,save_dir, min_slope,min_size,max_size,max_axis_width,max_box_WH,process_num) is False: failed_tifs.append(slope) io_function.save_list_to_txt('extract_headwall_failed_tifs.txt',failed_tifs)
def merge_grid_ids_txt(task, fail_id_txt_list): id_list = [] for txt in fail_id_txt_list: id_list.extend(io_function.read_list_from_txt(txt)) id_list = list(set(id_list)) # remove redudant ones save_path = '%s_fail_grid_ids.txt' % task io_function.save_list_to_txt(save_path, id_list) return save_path
def produce_dem_diff_grids(grid_polys, grid_ids, pre_name, reg_tifs,b_apply_matchtag, b_mosaic_id,b_mosaic_date,keep_dem_percent,o_res,process_num=4): dem_ext_polys = get_dem_tif_ext_polygons(reg_tifs) dem_diff_tifs = [] # mosaic and crop if os.path.isfile(grid_dem_diff_less2dem_txt): grid_id_less2dem_list = [ int(item) for item in io_function.read_list_from_txt(grid_dem_diff_less2dem_txt) ] # no need covert to int else: grid_id_less2dem_list = [] for grid_id, grid_poly in zip(grid_ids, grid_polys): if grid_id in grid_id_less2dem_list: basic.outputlogMessage('skip grid %d, previous processing shows that, the count of DEM is smaller than 2'%grid_id) continue save_dir = 'grid_%d_tmp_files'%grid_id # check free disk space work_dir = './' free_GB = io_function.get_free_disk_space_GB(work_dir) total_wait_time = 0 while free_GB < 50 and total_wait_time < 60*60*12: basic.outputlogMessage(' The free disk space (%.4f) is less than 50 GB, wait 60 seconds'%free_GB) time.sleep(60) total_wait_time += 60 free_GB = io_function.get_free_disk_space_GB(work_dir) # get subset of tifs dem_poly_index = vector_gpd.get_poly_index_within_extent(dem_ext_polys, grid_poly) if len(dem_poly_index) < 1: basic.outputlogMessage('warning, no dem tifs within %d grid, skip' % grid_id) save_id_grid_no_valid_dem(grid_id) continue dem_list_sub = [reg_tifs[index] for index in dem_poly_index] # filter by month # cancel, because it removes many good data. # dem_list_sub = filter_dem_by_month(dem_list_sub) mosaic_tif_list = mosaic_crop_dem(dem_list_sub, save_dir, grid_id, grid_poly, b_mosaic_id, b_mosaic_date, process_num, keep_dem_percent, o_res, pre_name, resample_method='average',b_mask_matchtag=b_apply_matchtag, b_mask_stripDEM_outlier=b_mask_stripDEM_outlier,b_mask_surface_water=b_mask_surface_water, b_mosaic_year=b_mosaic_year) # dem co-registration (cancel, the result in not good with the default setting) # dem differencing save_dem_diff = os.path.join(grid_dem_diffs_dir, pre_name + '_DEM_diff_grid%d.tif'%grid_id) save_date_diff = os.path.join(grid_dem_diffs_dir, pre_name + '_date_diff_grid%d.tif'%grid_id) if dem_diff_newest_oldest(mosaic_tif_list, save_dem_diff, save_date_diff, process_num, b_max_subsidence=b_max_subsidence,b_save_cm=True): dem_diff_tifs.append(save_dem_diff) else: save_id_grid_dem_less_2(grid_id) grid_id_less2dem_list.append(grid_id) return dem_diff_tifs
def read_dem_diff_assigned_to_other_machine(job_list_pre): dem_list_txts = io_function.get_file_list_by_pattern( dem_common.process_log_dir, job_list_pre + '*.txt') assign_dem_diff = [] for txt in dem_list_txts: assign_dem_diff.extend(io_function.read_list_from_txt(txt)) return assign_dem_diff
def get_ext_shps(): # make sure in different machine, the extent file are in the same order, we create a list: ArcticDEM_subsets_list.txt shps_list_txt = os.path.join(ext_shp_dir, 'ArcticDEM_subsets_list.txt') shps_list = io_function.read_list_from_txt(shps_list_txt) ext_shps = [os.path.join(ext_shp_dir, item) for item in shps_list] # check existence for ext in ext_shps: io_function.is_file_exist(ext) return ext_shps
def read_grid_ids_from_other_extent(): grid_txt_list = io_function.get_file_list_by_ext('.txt', grid_ids_txt_dir, bsub_folder=False) other_grid_ids = [] for txt in grid_txt_list: id_list = io_function.read_list_from_txt(txt) other_grid_ids.extend(id_list) other_grid_ids = [int(item) for item in other_grid_ids] return other_grid_ids
def save_list_no_need_dem_files(file_name, file_list): if len(file_list) < 1: return True # update the file list save_list = [] if os.path.isfile(file_name): save_list = io_function.read_list_from_txt(file_name) for item in file_list: if item in save_list: continue save_list.append(item) return io_function.save_list_to_txt(file_name, save_list)
def sync_log_files(process_node, r_log_dir, process_log_dir): # copy complete id list, dem info, grid_no_dem_ids.txt to remote machine files_to_processNode = [ 'strip_dem_cover_grids.txt', 'tile_dem_cover_grids.txt', 'grid_complete_ids.txt', 'grid_no_dem_ids.txt' ] for file in files_to_processNode: scp_communicate.copy_file_folder_to_remote_machine( process_node, os.path.join(r_log_dir, file), os.path.join(process_log_dir, file)) files_from_processNode = [ 'grid_dem_diff_less2dem_ids.txt', 'grid_no_valid_dem_ids.txt', 'grid_no_headwall_ids.txt', 'grid_no_subscidence_poly_ids.txt', 'grid_no_watermask_ids.txt' ] remote_name = process_node[1:].replace('_host', '') # change $curc_host to curc for file in files_from_processNode: # copy the file, do not overwrite the local file remote_file = os.path.join( process_log_dir, io_function.get_name_by_adding_tail(file, remote_name)) scp_communicate.copy_file_folder_from_remote_machine( process_node, os.path.join(r_log_dir, file), remote_file) # if they are new ids, then merged to "file" local_file = os.path.join(process_log_dir, file) remote_ids = io_function.read_list_from_txt( remote_file) if os.path.isfile(remote_file) else [ ] # no need, to int local_ids = io_function.read_list_from_txt( local_file) if os.path.isfile(local_file) else [] new_ids = [id for id in remote_ids if id not in local_ids] if len(new_ids) < 1: continue else: local_ids.extend(new_ids) io_function.save_list_to_txt(local_file, local_ids)
def main(options, args): # process_num = multiprocessing.cpu_count() process_num = options.process_num data_folder = arcticDEM_reg_tif_dir if len(args) > 0: data_folder = args[0] tifs = io_function.get_file_list_by_pattern( data_folder, '*.tif') # _dem_reg check all tifs save_invalid_txt_path = os.path.basename(data_folder) + '_invalid_list.txt' save_good_txt_path = os.path.basename(data_folder) + '_good_list.txt' tif_count = len(tifs) basic.outputlogMessage('get %d tif files in %s' % (tif_count, data_folder)) good_tifs = [] if os.path.isfile(save_good_txt_path): good_tifs.extend(io_function.read_list_from_txt(save_good_txt_path)) invalid_tif = [] # remove good one for the list if len(good_tifs) > 0: tifs = [ item for item in tifs if os.path.basename(item) not in good_tifs ] if process_num == 1: for idx, tif in enumerate(tifs): if check_one_tif(idx, tif_count, tif, good_tifs): good_tifs.append(os.path.basename(tif)) else: invalid_tif.append(os.path.basename(tif)) else: theadPool = Pool(process_num) # multi processes parameters_list = [(idx, tif_count, tif, good_tifs) for idx, tif in enumerate(tifs)] results = theadPool.starmap(check_one_tif, parameters_list) # need python3 for tif, res in zip(tifs, results): if res: good_tifs.append(os.path.basename(tif)) else: invalid_tif.append(os.path.basename(tif)) theadPool.close() io_function.save_list_to_txt(save_invalid_txt_path, invalid_tif) io_function.save_list_to_txt(save_good_txt_path, good_tifs)
def save_id_grid_no_subsidence(grid_id): # grid_no_headwall_txt if os.path.isdir(process_log_dir) is False: io_function.mkdir(process_log_dir) id_list = [] if os.path.isfile(grid_no_subscidence_poly_txt): id_list = io_function.read_list_from_txt(grid_no_subscidence_poly_txt) # no need covert to int id_str = str(grid_id) if id_str in id_list: return True else: # save by adding one line with open(grid_no_subscidence_poly_txt,'a') as f_obj: f_obj.writelines(str(grid_id) + '\n') return True
def add_id_grid_to_txt(grid_id, txt_path): # grid_no_valid_dem_ids.txt if os.path.isdir(process_log_dir) is False: io_function.mkdir(process_log_dir) # update txt file id_list = [] if os.path.isfile(txt_path): id_list = io_function.read_list_from_txt(txt_path) # no need covert to int id_str = str(grid_id) if id_str in id_list: return True else: # save by adding one line with open(txt_path,'a') as f_obj: f_obj.writelines(str(grid_id) + '\n') return True
def check_one_extent(extent_shp): print('start to check %s' % extent_shp) # local_grid_id_txt is in the current dir # log_grid_ids_txt, log_grid_ids_txt_done is in grid_ids_txt_dir local_grid_id_txt, log_grid_ids_txt, log_grid_ids_txt_done = get_extent_grid_id_txt_done_files( extent_shp) if os.path.isfile(local_grid_id_txt) is False and os.path.isfile( log_grid_ids_txt): io_function.copy_file_to_dst(log_grid_ids_txt, local_grid_id_txt) if os.path.isfile(local_grid_id_txt) is False: print('the _grid_ids.txt for %s does not exist, maybe it has started' % extent_shp) return False # check if it has been complete if os.path.isfile(log_grid_ids_txt_done): basic.outputlogMessage('Tasks for extent %s have been completed' % extent_shp) return True grid_ids_to_process_txt = io_function.get_name_no_ext( extent_shp) + '_' + 'grid_ids_to_process.txt' # read from txt file directly grid_ids = [ int(item) for item in io_function.read_list_from_txt(local_grid_id_txt) ] update_complete_grid_list(grid_ids, task_list) # check complete files, to see if it's done # remove grids that has been complete or ignored ignore_ids = get_complete_ignore_grid_ids() num_grid_ids = save_grid_ids_need_to_process( grid_ids, ignore_ids=ignore_ids, save_path=grid_ids_to_process_txt) if num_grid_ids < 1: print(datetime.now(), ' %s is marked as completed' % extent_shp) make_note_all_task_done(extent_shp, curc_node) else: print( datetime.now(), ' %s has not completed, %d grids to process, total: %d' % (extent_shp, num_grid_ids, len(grid_ids))) return True
def main(options, args): # process_num = multiprocessing.cpu_count() process_num = options.process_num if len(args) > 0: data_folder = args[0] else: data_folder = grid_dem_headwall_shp_dir basic.outputlogMessage('check shapefiles in %s, with %d processes'%(data_folder,process_num)) vector_files = io_function.get_file_list_by_ext('.shp',data_folder ,bsub_folder=True) save_invalid_txt_path = os.path.basename(data_folder) + '_incomplete_list.txt' save_good_txt_path = os.path.basename(data_folder) + '_good_list.txt' file_count = len(vector_files) basic.outputlogMessage('The number of vector files: %d'%file_count) good_files = [] if os.path.isfile(save_good_txt_path): good_files.extend(io_function.read_list_from_txt(save_good_txt_path)) incomplete_files = [] # remove good one for the list if len(good_files)>0: vector_files = [item for item in vector_files if os.path.basename(item) not in good_files] if process_num == 1: # tifs = io_function.get_file_list_by_ext('.tif',arcticDEM_reg_tif_dir, bsub_folder=False) for idx, tif in enumerate(vector_files): if check_one_vector_file(idx, file_count, tif, good_files): good_files.append(os.path.basename(tif)) else: incomplete_files.append(os.path.basename(tif)) else: theadPool = Pool(process_num) # multi processes parameters_list = [(idx, file_count, tif, good_files) for idx, tif in enumerate(vector_files)] results = theadPool.starmap(check_one_vector_file, parameters_list) # need python3 for tif, res in zip(vector_files, results): if res: good_files.append(os.path.basename(tif)) else: incomplete_files.append(os.path.basename(tif)) theadPool.close() io_function.save_list_to_txt(save_invalid_txt_path, incomplete_files) io_function.save_list_to_txt(save_good_txt_path, good_files)
def monitor_process_failed_grids(): tasks = ['dem_diff', 'dem_headwall_grid'] while True: all_fail_list = [] for task in tasks: all_fail_list.extend(get_failed_grid_ids(task)) if len(all_fail_list) < 1: print( datetime.now(), 'there is no failed jobs for %s to process, wait 10 minutes' % machine_name) time.sleep(600) continue for task in tasks: fail_id_txt_list = get_failed_grid_ids(task) if len(fail_id_txt_list) < 1: continue print('\n', task, 'fail_id_txt_list:', fail_id_txt_list, '\n') fail_ids_txt = merge_grid_ids_txt(task, fail_id_txt_list) # start processing, res = os.system('./run.sh %s %s' % (fail_ids_txt, task)) if res != 0: sys.exit(1) # wait all job done? Yes, in "parallel_processing_curc.py" # check 'done.txt' in each folder fail_ids = [ int(item) for item in io_function.read_list_from_txt(fail_ids_txt) ] for idx, grid_id in enumerate(fail_ids): job_folder = '%s_%s' % (task_2_job_folder[task], str(idx).zfill(5)) print('job_folder for grid_id: %d, is' % grid_id, job_folder) if os.path.isfile(job_folder + '/done.txt'): os.system('rm -r %s' % job_folder) else: sys.exit(1) # remove fail txt anyway if all "done.txt" exists for txt in fail_id_txt_list: io_function.delete_file_or_dir(txt)
def save_id_grid_dem_less_2(grid_id): # grid_dem_diff_less2dem_txt if os.path.isdir(process_log_dir) is False: io_function.mkdir(process_log_dir) # update grid_dem_diff_less2dem_txt file id_list = [] if os.path.isfile(grid_dem_diff_less2dem_txt): id_list = io_function.read_list_from_txt(grid_dem_diff_less2dem_txt) # no need covert to int id_str = str(grid_id) if id_str in id_list: return True else: # save by adding one line with open(grid_dem_diff_less2dem_txt,'a') as f_obj: f_obj.writelines(str(grid_id) + '\n') # # save # id_list.append(str(grid_id)) # io_function.save_list_to_txt(grid_dem_diff_less2dem_txt,id_list) return True
def save_id_grid_no_dem(grid_id): # grid_dem_diff_less2dem_txt if os.path.isdir(process_log_dir) is False: io_function.mkdir(process_log_dir) # update grid_dem_diff_less2dem_txt file id_list = [] if os.path.isfile(grid_no_dem_txt): id_list = io_function.read_list_from_txt( grid_no_dem_txt) # no need covert to int id_str = str(grid_id) if id_str in id_list: return True else: # save id_list.append(str(grid_id)) io_function.save_list_to_txt(grid_no_dem_txt, id_list) basic.outputlogMessage('Save gird id (%d) to %s' % (grid_id, grid_no_dem_txt)) return True
def get_not_completed_grids(grid_polys, grid_ids): completed_id_list = [] if os.path.isfile(grid_complete_list_txt): completed_id_list = [ int(item) for item in io_function.read_list_from_txt(grid_complete_list_txt) ] if len(completed_id_list) < 1: return grid_polys, grid_ids no_complete_polys = [] no_complete_ids = [] for poly, id in zip(grid_polys, grid_ids): if id in completed_id_list: pass else: no_complete_polys.append(poly) no_complete_ids.append(id) return no_complete_polys, no_complete_ids
def test_get_sub_images_pixel_json_files(): print('\n') print('running test_get_sub_images_pixel_json_files') dir = os.path.expanduser( '~/Data/Arctic/alaska/autoMapping/alaskaNS_yolov4_3/multi_inf_results') polygons_shp = os.path.join( dir, 'I0/I0_alaska_north_slope_planet_rgb_2020_alaskaNS_yolov4_3_exp3.shp') image_folder_or_path = io_function.read_list_from_txt( os.path.join(dir, '0.txt'))[0] process_num = 1 bufferSize = 300 dstnodata = 0 saved_dir = './' b_rectangle = True image_pattern = '.tif' class_names = ['rts'] get_sub_images_pixel_json_files(polygons_shp, image_folder_or_path, image_pattern, class_names, bufferSize, dstnodata, saved_dir, b_rectangle, process_num)
def b_exist_grid_headwall_shp(id): # if a grid don't have headwall (due to really flat or other reasons), then we think it's complete if os.path.isfile(grid_no_headwall_txt): grid_ids_no_headwall = [ int(item) for item in io_function.read_list_from_txt(grid_no_headwall_txt) ] if id in grid_ids_no_headwall: return True headwall_shps_dir = io_function.get_file_list_by_pattern( grid_dem_headwall_shp_dir, '*_grid%d' % id) if len(headwall_shps_dir) == 1: return True elif len(headwall_shps_dir) > 1: basic.outputlogMessage( 'warning, There are multiple headwall shps for grid: %d' % id) for item in headwall_shps_dir: basic.outputlogMessage(item) return True else: return False
def b_exist_gid_dem_diff(id): # if an id don't have enough dem for dem diff, then we think it's complete if os.path.isfile(grid_dem_diff_less2dem_txt): grid_ids_less_2dem = [ int(item) for item in io_function.read_list_from_txt( grid_dem_diff_less2dem_txt) ] if id in grid_ids_less_2dem: return True dem_diff_files = io_function.get_file_list_by_pattern( grid_dem_diffs_dir, '*_DEM_diff_grid%d.tif' % id) if len(dem_diff_files) == 1: return True elif len(dem_diff_files) > 1: basic.outputlogMessage( 'warning, There are multiple DEM difference for grid: %d' % id) for item in dem_diff_files: basic.outputlogMessage(item) return True else: return False
def b_exist_dem_hillshade_newest_HWLine_grid(id): # if a grid don't have headwall (due to really flat or other reasons), then we think it's complete if os.path.isfile(grid_no_headwall_txt): grid_ids_no_headwall = [ int(item) for item in io_function.read_list_from_txt(grid_no_headwall_txt) ] if id in grid_ids_no_headwall: return True hillshade_newest_HDLine_tifs = io_function.get_file_list_by_pattern( grid_hillshade_newest_HDLine_dir, '*_grid%d.tif' % id) if len(hillshade_newest_HDLine_tifs) == 1: return True elif len(hillshade_newest_HDLine_tifs) > 1: basic.outputlogMessage( 'warning, There are multiple hillshade (newest) HDLine tif for grid: %d' % id) for item in hillshade_newest_HDLine_tifs: basic.outputlogMessage(item) return True else: return False