def main(): # run in ~/Data/Arctic/canada_arctic/autoMapping/multiArea_sub_images on tesia ini_list = io_function.get_file_list_by_pattern('./','area*.ini') txt_list = io_function.get_file_list_by_pattern('./','area*.txt') for txt in txt_list: ini_s = io_function.read_list_from_txt(txt) ini_list.extend(ini_s) ini_list = [os.path.abspath(item) for item in ini_list] file_names = [ io_function.get_name_no_ext(item) for item in ini_list ] cur_dir = os.getcwd() # show [print(item) for item in ini_list] time.sleep(3) for name, area_ini in zip(file_names,ini_list): word_dir = os.path.join(cur_dir,name) io_function.mkdir(word_dir) os.chdir(word_dir) # copy and modify main_para.ini io_function.copyfiletodir(os.path.join(cur_dir,'main_para.ini'),'./',overwrite=True) io_function.copyfiletodir(os.path.join(cur_dir,'exe.sh'),'./',overwrite=True) parameters.write_Parameters_file('main_para.ini','training_regions',area_ini) # run exe.sh res = os.system('./exe.sh') if res !=0: print(res) sys.exit(1) os.chdir(cur_dir)
def get_dem_diff_list_to_seg(): dem_diff_list = io_function.get_file_list_by_pattern( dem_common.grid_dem_diffs_dir, '*DEM_diff_grid*.tif') dem_diff_list_copy = dem_diff_list.copy() if len(dem_diff_list) < 1: print('No *DEM_diff*.tif in %s' % dem_common.grid_dem_diffs_dir) return [] dem_diff_ids = [ int(re.findall('grid\d+', os.path.basename(item))[0][4:]) for item in dem_diff_list ] dem_subsidence_shps = io_function.get_file_list_by_pattern( dem_common.grid_dem_diffs_segment_dir, 'segment_result*/*_post.shp') subsidence_ids = [ int(re.findall('grid\d+', os.path.basename(item))[0][4:]) for item in dem_subsidence_shps ] no_subsidence_ids = [] if os.path.isfile(dem_common.grid_no_subscidence_poly_txt): no_subsidence_ids = [ int(item) for item in io_function.read_list_from_txt( dem_common.grid_no_subscidence_poly_txt) ] subsidence_ids.extend(no_subsidence_ids) # remove dem diff already been segmented or no subsidence for id, dem_diff in zip(dem_diff_ids, dem_diff_list_copy): if id in subsidence_ids: dem_diff_list.remove(dem_diff) return dem_diff_list
def main(options, args): root_dir = args[0] if os.path.isdir(root_dir) is False: raise ValueError('%s not exists'%root_dir) folder_pattern = options.folder_pattern folder_list = io_function.get_file_list_by_pattern(root_dir,folder_pattern) folder_list = [item for item in folder_list if os.path.isdir(item) ] folder_list.sort() # ray may create a new folder if the previous one already exists dupli_foldes = io_function.get_file_list_by_pattern(root_dir,folder_pattern + '_????') dupli_foldes = [item for item in dupli_foldes if os.path.isdir(item) ] dupli_foldes.sort() folder_list.extend(dupli_foldes) para_file = options.para_file output_file = options.output if output_file is None: output_file = os.path.basename(root_dir) + '.xlsx' train_output = {} train_output['folder'] = [] for para in para_ini_list: train_output[para] = [] train_output['train_class_0'] = [] train_output['train_class_1'] = [] train_output['val_class_0'] = [] train_output['val_class_1'] = [] train_output['class_1'] = [] train_output['overall'] = [] train_output['step'] = [] train_output['early_stopping'] = [] train_output['model_train_iter'] = [] for folder in folder_list: print('read parameter and results for %s'%folder) train_output['folder'].append(os.path.basename(folder)) read_para_values(folder,para_file,train_output) get_training_image_patch_count(folder,train_output) get_miou_of_overall_and_class_1_step(folder, para_file, train_output) get_early_stopping_trained_iteration(folder, para_file, train_output) # save to excel file train_out_table_pd = pd.DataFrame(train_output) with pd.ExcelWriter(output_file) as writer: train_out_table_pd.to_excel(writer, sheet_name='training parameter and results') # set format workbook = writer.book format = workbook.add_format({'num_format': '#0.000'}) train_out_table_sheet = writer.sheets['training parameter and results'] train_out_table_sheet.set_column('O:P',None,format) pass
def read_a_meta_of_scene(scene_folder_or_geojson,scene_id_list): # get scene id if os.path.isfile(scene_folder_or_geojson): # geojson file scene_id = os.path.splitext(os.path.basename(scene_folder_or_geojson))[0] geojson_path = scene_folder_or_geojson scene_folder = os.path.splitext(scene_folder_or_geojson)[0] else: # scene_folder scene_id = os.path.basename(scene_folder_or_geojson) geojson_path = scene_folder_or_geojson + '.geojson' scene_folder = scene_folder_or_geojson # if already exists if scene_id in scene_id_list: return None,None,None,None,None,None,None,None,None print(scene_id) # get metadata path cloud_cover = 101 acquisitionDate = datetime(1970,1,1) metadata_paths = io_function.get_file_list_by_pattern(scene_folder,'*metadata.xml') if len(metadata_paths) < 1: basic.outputlogMessage('warning, there is no metadata file in %s'%scene_folder) elif len(metadata_paths) > 1: basic.outputlogMessage('warning, there are more than one metadata files in %s' % scene_folder) else: # read metadata metadata_path = metadata_paths[0] cloud_cover = read_cloud_cover(metadata_path) acquisitionDate = read_acquired_datetime(metadata_path) assets = io_function.get_file_list_by_pattern(scene_folder,'*') asset_count = len(assets) asset_files = sorted([ os.path.basename(item) for item in assets]) asset_files =','.join(asset_files) image_type = 'analytic' # 'analytic_sr' (surface reflectance) or 'analytic' sr_tif = io_function.get_file_list_by_pattern(scene_folder,'*_SR.tif') if len(sr_tif) == 1: image_type = 'analytic_sr' # consider as the downloading time if os.path.isfile(geojson_path): modified_time = io_function.get_file_modified_time(geojson_path) else: geojson_path = '' modified_time = io_function.get_file_modified_time(scene_folder) return scene_id,cloud_cover,acquisitionDate,geojson_path,scene_folder,asset_count,image_type,asset_files,modified_time
def get_Planet_SR_image_list_overlap_a_polygon(polygon,geojson_list, cloud_cover_thr, save_list_path=None): ''' get planet surface reference (SR) list overlap a polygon (within or overlap part of the polygon) :param polygon: polygon in the shapely format :param geojson_list: :param save_list_path: save the list to a txt file. :return: ''' image_path_list = [] cloud_cover_list = [] for geojson_file in geojson_list: # print(geojson_file) with open(geojson_file) as json_obj: geom = json.load(json_obj) img_ext = shape(geom) inter = polygon.intersection(img_ext) if inter.is_empty is False: img_dir = os.path.splitext(geojson_file)[0] sr_img_paths = io_function.get_file_list_by_pattern(img_dir,'*_SR.tif') if len(sr_img_paths) < 1: basic.outputlogMessage('warning, no SR image in %s, try to find Analytic images'%img_dir) sr_img_paths = io_function.get_file_list_by_pattern(img_dir, '*AnalyticMS.tif') meta_data_paths = io_function.get_file_list_by_pattern(img_dir,'*_metadata.xml') if len(sr_img_paths) != len(meta_data_paths): # raise ValueError('the count of metadata files and images is different') basic.outputlogMessage('warning: the count of metadata files and images is different for %s'%img_dir) continue if len(sr_img_paths) < 1: basic.outputlogMessage('warning, no Planet SR image in the %s'%img_dir) elif len(sr_img_paths) > 1: basic.outputlogMessage('warning, more than one Planet SR image in the %s'%img_dir) else: # check cloud cover cloud_cover = read_cloud_cover(meta_data_paths[0]) if cloud_cover > cloud_cover_thr: continue # add image image_path_list.append(sr_img_paths[0]) cloud_cover_list.append(cloud_cover) if save_list_path is not None: with open(save_list_path,'w') as f_obj: for image_path in image_path_list: f_obj.writelines(image_path + '\n') return image_path_list, cloud_cover_list
def read_down_load_geometry(folder): ''' read geojson files in a folder. geojson file stores the geometry of a file, and save to global varialbes :param folder: the save folder :return: ''' global downloaded_scene_geometry json_list = io_function.get_file_list_by_ext('.geojson', folder, bsub_folder=False) for json_file in json_list: # ignore the scenes in the excluded list item_id = os.path.splitext(os.path.basename(json_file))[0] if item_id in manually_excluded_scenes: continue scene_folder = os.path.splitext(json_file)[0] asset_files = io_function.get_file_list_by_pattern(scene_folder, '*') if len(asset_files) < 3: basic.outputlogMessage( 'downloading of scene %s is not compelte, ignore it' % item_id) continue with open(json_file) as json_file: data = json.load(json_file) # p(data) # test downloaded_scene_geometry.append(data)
def find_results_for_one_grid_id(root_dir, folder, pattern, grid_id, grid_poly, description='dem_subsidence'): shp_dir = os.path.join(root_dir, folder) shps = io_function.get_file_list_by_pattern(shp_dir, pattern) if len(shps) < 1: print('warning, no results in folder: %s, with pattern: %s' % (folder, pattern)) return None if description == 'dem_subsidence': if len(shps) == 1: return shps[0] # if it's subsidence from dem diff, should only have one shp if len(shps) > 1: raise ValueError( 'warning, more than one result in folder: %s, with pattern: %s' % (folder, pattern)) if description == 'headwall': return shps
def save_planet_images_to_excel(image_dir,save_xlsx): if os.path.isfile(image_dir): basic.outputlogMessage('Warning, Input %s is a file, expected a folder, skip it'%image_dir) return False # read save_xlsx if it exist # may be not good to exclude these the scene id if we want to update some records. # if os.path.isfile(save_xlsx): # df = pd.read_excel(save_xlsx) # scene_id_list.extend(df['scene_id'].to_list()) # get scene folders (idealy, the number of scene folder are the same to the one of geojson files) scene_geojson_folders = io_function.get_file_list_by_pattern(image_dir,'????????_??????_*') # acquired date_time if len(scene_geojson_folders) < 1: raise ValueError('There is no scene folder or geojson in %s'%image_dir) scene_table, scene_without_asset = get_meta_dict(scene_geojson_folders) df = pd.DataFrame(scene_table) with pd.ExcelWriter(save_xlsx) as writer: df.to_excel(writer) basic.outputlogMessage('write records of downloaded scenes to %s'%save_xlsx) scene_folder_no_assets_txt = os.path.splitext(save_xlsx)[0] + '_scenes_noAsset.txt' with open(scene_folder_no_assets_txt, 'w') as f_obj: for scene_dir in scene_without_asset: f_obj.writelines(scene_dir + '\n') return True
def get_local_folder(local_dir, folder_pattern): # get local dir folder_list = io_function.get_file_list_by_pattern(local_dir, folder_pattern) folder_list = [item for item in folder_list if os.path.isdir(item)] folder_list.sort() return folder_list
def b_exist_grid_dem_subsidence(id): # if an grid don't have dem subsidence, then we think it's complete if os.path.isfile(grid_no_subscidence_poly_txt): grid_ids_no_subsidence = [ int(item) for item in io_function.read_list_from_txt( grid_no_subscidence_poly_txt) ] if id in grid_ids_no_subsidence: return True # if an id don't have enough dem for dem diff, then we think it's complete if os.path.isfile(grid_dem_diff_less2dem_txt): grid_ids_less_2dem = [ int(item) for item in io_function.read_list_from_txt( grid_dem_diff_less2dem_txt) ] if id in grid_ids_less_2dem: return True dem_subsidence_shps = io_function.get_file_list_by_pattern( grid_dem_diffs_segment_dir, '*_grid%d/*_grid%d_8bit_post.shp' % (id, id)) if len(dem_subsidence_shps) == 1: return True elif len(dem_subsidence_shps) > 1: basic.outputlogMessage( 'warning, There are multiple DEM subsidence for grid: %d' % id) for item in dem_subsidence_shps: basic.outputlogMessage(item) return True else: return False
def main(): basic.setlogfile('log_to_relative_dem_8bit.txt') if os.path.isdir(relative_dem_dir) is False: io_function.mkdir(relative_dem_dir) # 500 pixel by 500 pixel, that is 1 km by 1 km patch_width = 500 patch_height = 500 process_num = 1 failed_tifs = [] dem_reg_list = io_function.get_file_list_by_pattern( arcticDEM_reg_tif_dir, '*dem_reg.tif') count = len(dem_reg_list) for idx, tif in enumerate(dem_reg_list): print('%d/%d convert %s to relative DEM (8bit)' % (idx + 1, count, tif)) rel_dem_8bit = io_function.get_name_by_adding_tail(tif, 'relDEM8bit') rel_dem_8bit = os.path.join(relative_dem_dir, os.path.basename(rel_dem_8bit)) try: dem_to_relative_dem(tif, rel_dem_8bit, patch_width, patch_height, process_num) except: failed_tifs.append(tif) with open('to_relative_dem_failed_cases.txt', 'w') as f_obj: for item in failed_tifs: f_obj.writelines(item + '\n') pass
def get_existing_dem_diff(dem_diff_dir, grid_base_name, grid_ids): existing_tif = [] grid_id_no_dem_tiff = [] for id in grid_ids: # dem_diff = os.path.join(dem_diff_dir, grid_base_name + '_DEM_diff_grid%d.tif'%id) # if os.path.isfile(dem_diff): # existing_tif.append(dem_diff) # continue dem_diff_files = io_function.get_file_list_by_pattern(dem_diff_dir, '*_DEM_diff_grid%d.tif'%id) if len(dem_diff_files) == 1: existing_tif.append(dem_diff_files[0]) continue elif len(dem_diff_files) > 1: existing_tif.append(dem_diff_files[0]) basic.outputlogMessage('warning, There are multiple DEM_diff tif for grid: %d'%id) for item in dem_diff_files: basic.outputlogMessage(item) continue else: pass grid_id_no_dem_tiff.append(id) if len(existing_tif) > 0: basic.outputlogMessage('%d existing grid dem diff files for the input grid_ids or extent'%len(existing_tif)) else: basic.outputlogMessage('no existing grid dem diff files') return existing_tif, grid_id_no_dem_tiff
def get_existing_grid_headwall_shp(headwall_shp_dir, grid_base_name, grid_ids): existing_grid_headwall_shp = [] grid_id_no_headwall_shp = [] for id in grid_ids: headwall_shps_dir = io_function.get_file_list_by_pattern( headwall_shp_dir, '*_grid%d' % id) if len(headwall_shps_dir) == 1: existing_grid_headwall_shp.append(headwall_shps_dir[0]) continue elif len(headwall_shps_dir) > 1: existing_grid_headwall_shp.append(headwall_shps_dir[0]) basic.outputlogMessage( 'warning, There are multiple headwall shps for grid: %d' % id) for item in headwall_shps_dir: basic.outputlogMessage(item) continue else: pass grid_id_no_headwall_shp.append(id) if len(existing_grid_headwall_shp) > 0: basic.outputlogMessage( '%d existing grid headwall shps for the input grid_ids or extent' % len(existing_grid_headwall_shp)) else: basic.outputlogMessage('no existing grid headwall shps') return existing_grid_headwall_shp, grid_id_no_headwall_shp
def main(options, args): t_polygons_shp = args[0] image_folder = args[1] # folder for store image tile (many split block of a big image) b_label_image = options.no_label_image process_num = options.process_num # check training polygons assert io_function.is_file_exist(t_polygons_shp) t_polygons_shp_all = options.all_training_polygons if t_polygons_shp_all is None: basic.outputlogMessage('Warning, the full set of training polygons is not assigned, ' 'it will consider the one in input argument is the full set of training polygons') t_polygons_shp_all = t_polygons_shp else: if get_projection_proj4(t_polygons_shp) != get_projection_proj4(t_polygons_shp_all): raise ValueError('error, projection insistence between %s and %s'%(t_polygons_shp, t_polygons_shp_all)) assert io_function.is_file_exist(t_polygons_shp_all) # get image tile list # image_tile_list = io_function.get_file_list_by_ext(options.image_ext, image_folder, bsub_folder=False) image_tile_list = io_function.get_file_list_by_pattern(image_folder,options.image_ext) if len(image_tile_list) < 1: raise IOError('error, failed to get image tiles in folder %s'%image_folder) check_projection_rasters(image_tile_list) # it will raise errors if found problems # comment out on June 18, 2021, # check_1or3band_8bit(image_tile_list) # it will raise errors if found problems #need to check: the shape file and raster should have the same projection. if get_projection_proj4(t_polygons_shp) != get_projection_proj4(image_tile_list[0]): raise ValueError('error, the input raster (e.g., %s) and vector (%s) files don\'t have the same projection'%(image_tile_list[0],t_polygons_shp)) # check these are EPSG:4326 projection if get_projection_proj4(t_polygons_shp).strip() == '+proj=longlat +datum=WGS84 +no_defs': bufferSize = meters_to_degress_onEarth(options.bufferSize) else: bufferSize = options.bufferSize saved_dir = options.out_dir # if os.system('mkdir -p ' + os.path.join(saved_dir,'subImages')) != 0: # sys.exit(1) # if os.system('mkdir -p ' + os.path.join(saved_dir,'subLabels')) !=0: # sys.exit(1) io_function.mkdir(os.path.join(saved_dir,'subImages')) if b_label_image: io_function.mkdir(os.path.join(saved_dir,'subLabels')) dstnodata = options.dstnodata if 'qtb_sentinel2' in image_tile_list[0]: # for qtb_sentinel-2 mosaic pre_name = '_'.join(os.path.splitext(os.path.basename(image_tile_list[0]))[0].split('_')[:4]) else: pre_name = os.path.splitext(os.path.basename(image_tile_list[0]))[0] get_sub_images_and_labels(t_polygons_shp, t_polygons_shp_all, bufferSize, image_tile_list, saved_dir, pre_name, dstnodata, brectangle=options.rectangle, b_label=b_label_image, proc_num=process_num)
def test_dem_tif_to_8bit(): dem_diff_list = io_function.get_file_list_by_pattern('./','*.tif') count = len(dem_diff_list) for idx, tif in enumerate(dem_diff_list): print('%d/%d convert %s to 8 bit'%(idx+1, count, tif)) tif_8bit = io_function.get_name_by_adding_tail(tif, '8bit') output = os.path.join(grid_dem_diffs_8bit_dir, os.path.basename(tif_8bit)) dem_tif_to_8bit(tif,output)
def read_dem_diff_assigned_to_other_machine(job_list_pre): dem_list_txts = io_function.get_file_list_by_pattern( dem_common.process_log_dir, job_list_pre + '*.txt') assign_dem_diff = [] for txt in dem_list_txts: assign_dem_diff.extend(io_function.read_list_from_txt(txt)) return assign_dem_diff
def main(options, args): root_dir = args[0] if os.path.isdir(root_dir) is False: raise ValueError('%s not exists' % root_dir) folder_pattern = options.folder_pattern checked_folders = [] while True: folder_list = io_function.get_file_list_by_pattern( root_dir, folder_pattern) folder_list = [item for item in folder_list if os.path.isdir(item)] folder_list.sort() # ray may create a new folder if the previous one already exists dupli_foldes = io_function.get_file_list_by_pattern( root_dir, folder_pattern + '_????') dupli_foldes = [item for item in dupli_foldes if os.path.isdir(item)] dupli_foldes.sort() folder_list.extend(dupli_foldes) print(str(datetime.now()), 'start moving or removing files or folders\n') check_folder_count = 0 for folder in folder_list: if folder in checked_folders: continue print('checking folder: %s' % folder) if io_function.check_file_or_dir_is_old(folder, time_hour_thr): print( '%s is older than %f hours, will remove some files inside it' % (folder, time_hour_thr)) remove_files(folder) checked_folders.append(folder) check_folder_count += 1 print(str(datetime.now()), 'removing files in %d folders' % check_folder_count) time.sleep(60) # wait pass
def get_loss_learning_rate_list(log_dir): # add the tensorboard in the tf1x version tf1x_dir = os.path.join(os.path.dirname(os.path.dirname(tf1x_python)), 'lib', 'python3.7', 'site-packages') sys.path.insert(0, tf1x_dir) from tensorboard.backend.event_processing.event_accumulator import EventAccumulator tf_size_guidance = { 'compressedHistograms': 10, 'images': 0, 'scalars': 0, # set a 0, to load all scalars 'histograms': 1 } events_files = io_function.get_file_list_by_pattern(log_dir, 'events*') if len(events_files) < 1: print('warning, No events file in %s' % log_dir) return None event_acc = EventAccumulator(log_dir, tf_size_guidance) event_acc.Reload() #Show all tags in the log file # tag_dict = event_acc.Tags() # io_function.save_dict_to_txt_json('event_acc.txt',tag_dict) # "scalars": [ # "clone_0/Losses/clone_0//clone_loss", # "total_loss_1", # "learning_rate", # "losses/clone_0/semantic_merged_logits/mul_1", # "clone_0/Losses/regularization_loss", # "global_step/sec" : how much time it takes for each step # ], loss_learnrate_dic = {} total_loss_1_event = event_acc.Scalars('total_loss_1') total_loss_list = [ item[2] for item in total_loss_1_event ] # item[0] is wall_time, item[1] is step, item [2] is the value loss_learnrate_dic['total_loss'] = total_loss_list step_list = [item[1] for item in total_loss_1_event] wall_time_list = [ item[0] for item in total_loss_1_event ] # we can use datetime.fromtimestamp() to convert datetime learning_rate_event = event_acc.Scalars('learning_rate') learning_rate_list = [item[2] for item in learning_rate_event] loss_learnrate_dic['learning_rate'] = learning_rate_list loss_learnrate_dic['step'] = step_list loss_learnrate_dic['wall_time'] = wall_time_list io_function.save_dict_to_txt_json( os.path.join(log_dir, 'loss_learning_rate.txt'), loss_learnrate_dic)
def check_align_folder(dem_tif): # by default, dem_align.py save the results to where dem_tif is res_dir = os.path.dirname(dem_tif) align_folder = os.path.splitext( os.path.basename(dem_tif))[0] + '_dem_align' align_dir = os.path.join(res_dir, align_folder) # after dem_align.py usually have 9 files align_outputs = io_function.get_file_list_by_pattern(align_dir, '*') # print(align_outputs) return align_outputs
def get_sub_images_pixel_json_files(polygons_shp, image_folder_or_path, image_pattern, class_names, bufferSize, dstnodata, saved_dir, b_rectangle, process_num): # check training polygons assert io_function.is_file_exist(polygons_shp) # get image tile list # image_tile_list = io_function.get_file_list_by_ext(options.image_ext, image_folder, bsub_folder=False) if os.path.isdir(image_folder_or_path): image_tile_list = io_function.get_file_list_by_pattern( image_folder_or_path, image_pattern) else: assert io_function.is_file_exist(image_folder_or_path) image_tile_list = [image_folder_or_path] if len(image_tile_list) < 1: raise IOError('error, failed to get image tiles in folder %s' % image_folder_or_path) get_subImages.check_projection_rasters( image_tile_list) # it will raise errors if found problems get_subImages.check_1or3band_8bit( image_tile_list) # it will raise errors if found problems # need to check: the shape file and raster should have the same projection. if get_subImages.get_projection_proj4( polygons_shp) != get_subImages.get_projection_proj4( image_tile_list[0]): raise ValueError( 'error, the input raster (e.g., %s) and vector (%s) files don\'t have the same projection' % (image_tile_list[0], polygons_shp)) # check these are EPSG:4326 projection if get_subImages.get_projection_proj4( polygons_shp).strip() == '+proj=longlat +datum=WGS84 +no_defs': bufferSize = get_subImages.meters_to_degress_onEarth(bufferSize) pre_name = os.path.splitext(os.path.basename(image_tile_list[0]))[0] saved_dir = os.path.join(saved_dir, pre_name + '_subImages') if os.path.isdir(saved_dir) is False: io_function.mkdir(saved_dir) get_sub_images_and_json_files(polygons_shp, class_names, bufferSize, image_tile_list, saved_dir, pre_name, dstnodata, brectangle=b_rectangle, proc_num=process_num)
def tifs_to_png(image_dir): tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif') for idx, tif in enumerate(tif_list): print('tif to png: %d/%d tif' % (idx + 1, len(tif_list))) basename = io_function.get_name_no_ext(tif) save_path = os.path.join(image_dir, basename + '.png') if os.path.isfile(save_path): print('%s exists, skip' % save_path) continue command_str = "gdal_translate -of PNG %s %s" % (tif, save_path) basic.os_system_exit_code(command_str)
def main(): file_list = io_function.get_file_list_by_pattern(arcticDEM_reg_tif_dir, '*_dem_reg.tif') print('Get %d dem_reg.tif from %s' % (len(file_list), arcticDEM_reg_tif_dir)) year_dates = [ timeTools.get_yeardate_yyyymmdd(os.path.basename(item), pattern='[0-9]{8}_') for item in file_list ] month_list = [item.month for item in year_dates] value_list = month_list # save unique date to txt file dates_unique = set(year_dates) dates_unique = sorted(dates_unique) dates_unique_str = [ timeTools.date2str(item, '%Y-%m-%d') for item in dates_unique ] io_function.save_list_to_txt('dates_unique.txt', dates_unique_str) # plot a histogram # bin_count = 12 bins = np.arange(0, 12, 1) fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8, 8)) n, bins, patches = ax.hist(value_list, bins=bins, alpha=0.75, ec="black", linewidth='1.5', color='grey', hatch='', rwidth=1) # density = True, # label=labels, # ax.legend(prop={'size': 12}) plt.xticks(bins) ax.tick_params(axis='both', which='both', direction='out', length=7, labelsize=20) # ,width=50 #, # if xlabelrotation is not None: # ax.tick_params(axis='x', labelrotation=90) # if ylim is not None: # ax.set_ylim(ylim) plt.gcf().subplots_adjust(bottom=0.15) # plt.grid(True) plt.savefig('ArcticDEM_strip_date_hist.jpg') #
def copy_subImages_labels_directly(subImage_dir, subLabel_dir, area_ini): input_image_dir = parameters.get_directory_None_if_absence( area_ini, 'input_image_dir') # it is ok consider a file name as pattern and pass it the following functions to get file list input_image_or_pattern = parameters.get_string_parameters( area_ini, 'input_image_or_pattern') # label raster folder label_raster_dir = parameters.get_directory_None_if_absence( area_ini, 'label_raster_dir') sub_images_list = [] label_path_list = [] if os.path.isdir(subImage_dir) is False: io_function.mkdir(subImage_dir) if os.path.isdir(subLabel_dir) is False: io_function.mkdir(subLabel_dir) sub_images = io_function.get_file_list_by_pattern(input_image_dir, input_image_or_pattern) for sub_img in sub_images: # find the corresponding label raster label_name = io_function.get_name_by_adding_tail( os.path.basename(sub_img), 'label') label_path = os.path.join(label_raster_dir, label_name) if os.path.isfile(label_path): sub_images_list.append(sub_img) label_path_list.append(label_path) else: print('Warning, cannot find label for %s in %s' % (sub_img, label_raster_dir)) # copy sub-images, adding to txt files with open('sub_images_labels_list.txt', 'a') as f_obj: for tif_path, label_file in zip(sub_images_list, label_path_list): if label_file is None: continue dst_subImg = os.path.join(subImage_dir, os.path.basename(tif_path)) # copy sub-images io_function.copy_file_to_dst(tif_path, dst_subImg, overwrite=True) dst_label_file = os.path.join(subLabel_dir, os.path.basename(label_file)) io_function.copy_file_to_dst(label_file, dst_label_file, overwrite=True) sub_image_label_str = dst_subImg + ":" + dst_label_file + '\n' f_obj.writelines(sub_image_label_str)
def main(): reg_tif_dir = 'arcticdem_registration_tifs' while True: print(str(datetime.now()), 'start moving or removing files or folders\n\n') reg_files = io_function.get_file_list_by_pattern(reg_tif_dir, '*') print('registration file count: %d in %s' % (len(reg_files), reg_tif_dir)) for file in reg_files: if check_file_or_dir_is_old(file, time_hour_thr): print( '%s is older than %f hours, will be moved to archieved dir' % (file, time_hour_thr)) io_function.movefiletodir(file, arcticDEM_reg_tif_dir, overwrite=True) SETSM_dir = io_function.get_file_list_by_pattern( './', 'SETSM_*2m_v3.0') print('SETSM folder count: %d in %s' % (len(SETSM_dir), './')) for folder in SETSM_dir: if check_file_or_dir_is_old(folder, time_hour_thr): print('%s is older than %f hours, will be removed' % (folder, time_hour_thr)) io_function.delete_file_or_dir(folder) grid_tmp_dir = io_function.get_file_list_by_pattern('./', 'grid*files') print('Grid tmp folder count: %d in %s' % (len(grid_tmp_dir), './')) for folder in grid_tmp_dir: if check_file_or_dir_is_old(folder, time_hour_thr): print('%s is older than %f hours, will be removed' % (folder, time_hour_thr)) io_function.delete_file_or_dir(folder) time.sleep(60) # wait pass
def check_files_existence(dir, pre_name): file_pattern = ['*dem_reg.tif', '*reg_dem.tif'] # Arctic strip and tile (mosaic) version for pattern in file_pattern: file_list = io_function.get_file_list_by_pattern( dir, pre_name + pattern) if len(file_list) > 0: return True else: # check if in the archived dir if os.path.isdir(arcticDEM_reg_tif_dir): file_list_archived = io_function.get_file_list_by_pattern( arcticDEM_reg_tif_dir, pre_name + pattern) if len(file_list_archived) > 0: return True if os.path.isdir(arcticDEM_mosaic_reg_tif_dir): file_list_archived = io_function.get_file_list_by_pattern( arcticDEM_mosaic_reg_tif_dir, pre_name + pattern) if len(file_list_archived) > 0: return True return False
def get_file_path_parameter(parafile, data_dir, data_name_or_pattern): data_dir = parameters.get_directory_None_if_absence(parafile, data_dir) data_name_or_pattern = parameters.get_string_parameters_None_if_absence(parafile, data_name_or_pattern) if data_dir is None or data_name_or_pattern is None: return None file_list = io_function.get_file_list_by_pattern(data_dir,data_name_or_pattern) if len(file_list) < 1: raise IOError('NO file in %s with name or pattern: %s'%(data_dir, data_name_or_pattern)) if len(file_list) == 1: return file_list[0] else: # return multiple files return file_list
def main(options, args): # process_num = multiprocessing.cpu_count() process_num = options.process_num data_folder = arcticDEM_reg_tif_dir if len(args) > 0: data_folder = args[0] tifs = io_function.get_file_list_by_pattern( data_folder, '*.tif') # _dem_reg check all tifs save_invalid_txt_path = os.path.basename(data_folder) + '_invalid_list.txt' save_good_txt_path = os.path.basename(data_folder) + '_good_list.txt' tif_count = len(tifs) basic.outputlogMessage('get %d tif files in %s' % (tif_count, data_folder)) good_tifs = [] if os.path.isfile(save_good_txt_path): good_tifs.extend(io_function.read_list_from_txt(save_good_txt_path)) invalid_tif = [] # remove good one for the list if len(good_tifs) > 0: tifs = [ item for item in tifs if os.path.basename(item) not in good_tifs ] if process_num == 1: for idx, tif in enumerate(tifs): if check_one_tif(idx, tif_count, tif, good_tifs): good_tifs.append(os.path.basename(tif)) else: invalid_tif.append(os.path.basename(tif)) else: theadPool = Pool(process_num) # multi processes parameters_list = [(idx, tif_count, tif, good_tifs) for idx, tif in enumerate(tifs)] results = theadPool.starmap(check_one_tif, parameters_list) # need python3 for tif, res in zip(tifs, results): if res: good_tifs.append(os.path.basename(tif)) else: invalid_tif.append(os.path.basename(tif)) theadPool.close() io_function.save_list_to_txt(save_invalid_txt_path, invalid_tif) io_function.save_list_to_txt(save_good_txt_path, good_tifs)
def select_polygons_by_ids_in_filenames(in_shp, folder, save_path,id_name='id'): '''get ids from files name, then select polygons from the shapefile ''' if os.path.isfile(save_path): basic.outputlogMessage('warning, %s exists, skip'%save_path) return True files = io_function.get_file_list_by_pattern(folder,'*') manu_sel_ids = [get_id_from_filepath(item) for item in files] ## check if there are duplicated ones check_duplicated_ids(manu_sel_ids) pol_ids = vector_gpd.read_attribute_values_list(in_shp, id_name) # find match index select_idx = [ pol_ids.index(sel_id) for sel_id in manu_sel_ids ] return vector_gpd.save_shapefile_subset_as(select_idx,in_shp,save_path)
def get_tifs_bounding_boxes(image_dir): tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif') for idx, tif in enumerate(tif_list): print('get bounding box: %d/%d tif' % (idx + 1, len(tif_list))) basename = io_function.get_name_no_ext(tif) save_path = os.path.join(image_dir, basename + '_bound.geojson') if os.path.isfile(save_path): print('%s exists, skip' % save_path) continue command_str = imgExt + " %s -o tmp.gpkg" % tif basic.os_system_exit_code(command_str) command_str = "ogr2ogr -f GeoJSON -t_srs EPSG:3413 %s tmp.gpkg" % save_path # note: projection is EPSG:3413 basic.os_system_exit_code(command_str) io_function.delete_file_or_dir('tmp.gpkg')
def main(): # for the user defined module in code_dir, need to be imported in functions # sys.path.insert(0, code_dir) # import parameters # import basic_src.io_function as io_function # import workflow.whole_procedure as whole_procedure # from utility.eva_report_to_tables import read_accuracy_multi_reports loc_dir = "./ray_results" tune_name = "tune_dataAug_para_tesia" augment_options = get_augment_options() file_folders = io_function.get_file_list_by_pattern( os.path.join(loc_dir, tune_name), '*') if len(file_folders) > 1: b_resume = True else: b_resume = False analysis = tune.run( training_function, resources_per_trial={"gpu": 1}, # use one GPUs, local_dir=loc_dir, name=tune_name, # fail_fast=True, # Stopping after the first failure log_to_file=("stdout.log", "stderr.log"), #Redirecting stdout and stderr to files trial_name_creator=tune.function(trial_name_string), trial_dirname_creator=tune.function(trial_dir_string), resume=b_resume, config={"data_augmentation": tune.grid_search(augment_options)}) print("Best config: ", analysis.get_best_config(metric="overall_miou", mode="max")) # Get a dataframe for analyzing trial results. df = analysis.results_df output_file = 'training_dataAug_ray_tune_%s.xlsx' % ( datetime.now().strftime("%Y%m%d_%H%M%S")) with pd.ExcelWriter(output_file) as writer: df.to_excel(writer) # , sheet_name='accuracy table' # set format # workbook = writer.book # format = workbook.add_format({'num_format': '#0.000'}) # acc_talbe_sheet = writer.sheets['accuracy table'] # acc_talbe_sheet.set_column('G:I',None,format) print('write trial results to %s' % output_file)