def mask_nodata_regions_surface_water(ref_raster, in_raster, out_raster, ref_nodata=0, out_nodata=0): if os.path.isfile(out_raster): print('%s already exists'%out_raster) return True # get nodata mask from ref_raster command_str = 'gdal_calc.py --calc="(A>%d)" --outfile=tmp.tif -A %s --NoDataValue 0 --type=Byte '%(ref_nodata,ref_raster) res = os.system(command_str) if res != 0: print(res) sys.exit(1) # apply the mask to in_raster # in the surface water, 1 is water, 0 are other, so mask water outside extent to zero as well, eventually, set 0 as nodata command_str = 'gdal_calc.py --calc="B*A" --outfile=%s -A tmp.tif -B %s --NoDataValue %d --type=Byte ' % (out_raster,in_raster,out_nodata) res = os.system(command_str) if res != 0: print(res) sys.exit(1) io_function.delete_file_or_dir('tmp.tif') return True
def remove_previous_data_or_results(para_file): # remove previous data or result if necessary if os.path.isfile(time_txt): io_function.delete_file_or_dir(time_txt) # command_string = os.path.join(eo_dir, 'workflow', 'remove_previous_data.py') + ' ' + para_file # basic.os_system_exit_code(command_string) from remove_previous_data import remove_previous_data return remove_previous_data(para_file)
def process_arcticDEM_tiles(tar_list, save_dir, inter_format, resample_method, o_res, extent_poly, extent_id, pre_name, b_rm_inter=True): ''' process the mosaic (not multi-temporal) version of ArcticDEM :param tar_list: :param save_dir: :param inter_format: :param resample_method: :param o_res: output resolution :param extent_poly: extent polygons, in the same projection of ArcticDEM :param extent_id: extent id :param pre_name: :param b_rm_inter: :return: ''' # unpackage and crop to extent dem_tif_list, tar_folders = process_dem_tarball(tar_list, save_dir, inter_format, o_res, extent_poly=extent_poly, poly_id=extent_id, process_num=4) if len(dem_tif_list) < 1: raise ValueError('No DEM extracted from tarballs') dem_name = os.path.basename(tar_folders[0])[-7:] save_path = os.path.join( save_dir, pre_name + '_' + dem_name + '_ArcticTileDEM_sub_%d.tif' % extent_id) RSImageProcess.mosaic_crop_images_gdalwarp( dem_tif_list, save_path, resampling_method=resample_method, o_format=inter_format, xres=o_res, yres=o_res, compress='lzw', tiled='yes', bigtiff='if_safer') # remove intermediate files if b_rm_inter: basic.outputlogMessage('remove intermediate files') for folder in tar_folders: io_function.delete_file_or_dir(folder) return True
def move_files(save_dir, out_fig, out_hist_info): if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) trim_fig = io_function.get_name_by_adding_tail(out_fig, 'trim') os.system('convert -trim %s %s' % (out_fig, trim_fig)) io_function.movefiletodir(trim_fig, save_dir, overwrite=True) io_function.delete_file_or_dir(out_fig) # io_function.movefiletodir(out_fig,save_dir,overwrite=True) io_function.movefiletodir(out_hist_info, save_dir, overwrite=True)
def postProcess_total_F1(minimum_area, min_slope, dem_diff_uplimit, dem_diff_buffer_size, min_demD_area, IOU_threshold): # when ray start a process, we need to add code_dir again and import user-defined modules sys.path.insert(0, code_dir) sys.path.insert( 0, os.path.join(code_dir, 'workflow') ) # require in ray when import other modules in workflow folder import basic_src.io_function as io_function import workflow.whole_procedure as whole_procedure para_file = 'main_para.ini' # ray tune will change current folder to its logdir, change it back # os.chdir(curr_dir_before_ray) print('\n\n\n current folder', os.getcwd(), '\n\n\n') # allow ray to change current folder to its logdir, then we can run parallel work_dir = os.getcwd() copy_original_mapped_polygons(curr_dir_before_ray, work_dir) # work_dir = curr_dir_before_ray # create a training folder inf_post_note = str(minimum_area) + '_' + str(min_slope) + '_' + str( dem_diff_uplimit) + '_' + str(dem_diff_buffer_size) + '_' + str( IOU_threshold) # copy copy_ini_files copy_ini_files(curr_dir_before_ray, work_dir) # change para_file modify_parameter(para_file, 'minimum_area', minimum_area) modify_parameter(para_file, 'minimum_slope', min_slope) modify_parameter(para_file, 'dem_difference_range', 'None,' + str(dem_diff_uplimit)) modify_parameter(para_file, 'buffer_size_dem_diff', dem_diff_buffer_size) modify_parameter(para_file, 'minimum_dem_reduction_area', min_demD_area) modify_parameter(para_file, 'IOU_threshold', IOU_threshold) # run training # whole_procedure.run_whole_procedure(para_file,working_dir=work_dir) test_id = 'multiArea_deeplabV3+_5_exp6' whole_procedure.post_processing_backup(para_file, inf_post_note=inf_post_note, b_skip_getshp=True, test_id=test_id) io_function.delete_file_or_dir( 'multi_inf_results') # remove a folder to save storage # calculate the F1 score across all regions (total F1) totalF1 = get_total_F1score(work_dir) return totalF1
def get_tifs_bounding_boxes(image_dir): tif_list = io_function.get_file_list_by_pattern(image_dir, '*/*.tif') for idx, tif in enumerate(tif_list): print('get bounding box: %d/%d tif' % (idx + 1, len(tif_list))) basename = io_function.get_name_no_ext(tif) save_path = os.path.join(image_dir, basename + '_bound.geojson') if os.path.isfile(save_path): print('%s exists, skip' % save_path) continue command_str = imgExt + " %s -o tmp.gpkg" % tif basic.os_system_exit_code(command_str) command_str = "ogr2ogr -f GeoJSON -t_srs EPSG:3413 %s tmp.gpkg" % save_path # note: projection is EPSG:3413 basic.os_system_exit_code(command_str) io_function.delete_file_or_dir('tmp.gpkg')
def monitor_process_failed_grids(): tasks = ['dem_diff', 'dem_headwall_grid'] while True: all_fail_list = [] for task in tasks: all_fail_list.extend(get_failed_grid_ids(task)) if len(all_fail_list) < 1: print( datetime.now(), 'there is no failed jobs for %s to process, wait 10 minutes' % machine_name) time.sleep(600) continue for task in tasks: fail_id_txt_list = get_failed_grid_ids(task) if len(fail_id_txt_list) < 1: continue print('\n', task, 'fail_id_txt_list:', fail_id_txt_list, '\n') fail_ids_txt = merge_grid_ids_txt(task, fail_id_txt_list) # start processing, res = os.system('./run.sh %s %s' % (fail_ids_txt, task)) if res != 0: sys.exit(1) # wait all job done? Yes, in "parallel_processing_curc.py" # check 'done.txt' in each folder fail_ids = [ int(item) for item in io_function.read_list_from_txt(fail_ids_txt) ] for idx, grid_id in enumerate(fail_ids): job_folder = '%s_%s' % (task_2_job_folder[task], str(idx).zfill(5)) print('job_folder for grid_id: %d, is' % grid_id, job_folder) if os.path.isfile(job_folder + '/done.txt'): os.system('rm -r %s' % job_folder) else: sys.exit(1) # remove fail txt anyway if all "done.txt" exists for txt in fail_id_txt_list: io_function.delete_file_or_dir(txt)
def process_one_dem(idx, count, tif, product_list, arcticDEM_slope_dir, arcticDEM_slope_8bit_dir, arcticDEM_hillshade_dir, arcticDEM_tpi_8bit_dir): print('%d/%d convert %s to slope (8bit) and hillshade' % (idx + 1, count, tif)) try: slope_file = os.path.basename( io_function.get_name_by_adding_tail(tif, 'slope')) slope_file_bak = os.path.join(arcticDEM_slope_dir, os.path.basename(slope_file)) if 'slope' in product_list or 'slope_8bit' in product_list: slope_out = dem_to_slope(tif, slope_file, slope_file_bak) if slope_out is False: pass else: if 'slope_8bit' in product_list: slope_8bit = io_function.get_name_by_adding_tail( tif, 'slope8bit') slope_8bit = os.path.join(arcticDEM_slope_8bit_dir, os.path.basename(slope_8bit)) slope_to_8bit(slope_file, slope_8bit) # delete or move the slope file if 'slope' in product_list: io_function.move_file_to_dst(slope_file, slope_file_bak) else: io_function.delete_file_or_dir(slope_file) if 'hillshade' in product_list: hillshapde = io_function.get_name_by_adding_tail(tif, 'hillshade') hillshapde = os.path.join(arcticDEM_hillshade_dir, os.path.basename(hillshapde)) dem_to_hillshade(tif, hillshapde) if 'tpi' in product_list: tip_8bit = io_function.get_name_by_adding_tail(tif, 'TPI8bit') tip_8bit = os.path.join(arcticDEM_tpi_8bit_dir, os.path.basename(tip_8bit)) dem_to_tpi_save_8bit(tif, tip_8bit) return True except: print('failed in process %s' % tif) return tif
def main(): basic.setlogfile('scp_log.txt') while True: # get remote dir basic.outputlogMessage('get remote folders') remote_folders = get_remote_folder(remote_dir, folder_pattern) basic.outputlogMessage("%d remote folders" % len(remote_folders)) folder_list = get_local_folder(local_dir, folder_pattern) basic.outputlogMessage("%d local folders" % len(folder_list)) folder_name_list = [os.path.basename(item) for item in folder_list] for idx, r_folders in enumerate(remote_folders): folder_name = os.path.basename(r_folders) if folder_name in folder_name_list: continue basic.outputlogMessage('copy trained folder in %s' % folder_name) command_str = 'scp -r ${tesia_host}:%s %s/%s' % ( r_folders, local_dir, folder_name) print(command_str) status, result = basic.getstatusoutput(command_str) if status != 0: sys.exit(1) folder_list = get_local_folder(local_dir, folder_pattern) # update local folder # reomve incomplete folders for folder in folder_list: res_json = os.path.join(folder, 'result.json') if os.path.isfile(res_json) and os.path.getsize(res_json) > 0: continue else: basic.outputlogMessage('remove incomplete folder: %s' % os.path.basename(folder)) io_function.delete_file_or_dir(folder) basic.outputlogMessage('wait five hours') time.sleep(3600 * 5) # wait five hours pass
class TestdeeplabTrainclass(): if os.path.isdir('split_images'): io_function.delete_file_or_dir('split_images') if os.path.isdir('split_labels'): io_function.delete_file_or_dir('split_labels') io_function.mkdir('split_images') io_function.mkdir('split_labels') def test_split_a_pair_sub_image_label(self): ### split the training image to many small patch (480*480) patch_w= 160 # parameters.get_string_parameters(para_file,'train_patch_width') patch_h= 160 #parameters.get_string_parameters(para_file,'train_patch_height') # notes # set overlay as 80, then width or height of patches range from 240 to 320. # so it will generate more patches than 160 ones # overlay_x= 80 # parameters.get_string_parameters(para_file,'train_pixel_overlay_x') # overlay_y= 80 #parameters.get_string_parameters(para_file,'train_pixel_overlay_y') overlay_x= 160 # parameters.get_string_parameters(para_file,'train_pixel_overlay_x') overlay_y= 160 #parameters.get_string_parameters(para_file,'train_pixel_overlay_y') split_image_format= '.png' # parameters.get_string_parameters(para_file,'split_image_format') trainImg_dir= 'subImages' # parameters.get_string_parameters(para_file,'input_train_dir') labelImg_dir= 'subLabels' # parameters.get_string_parameters(para_file,'input_label_dir') if os.path.isdir(trainImg_dir) is False: raise IOError('%s not in the current folder, please get subImages first'%trainImg_dir) if os.path.isdir(labelImg_dir) is False: raise IOError('%s not in the current folder, please get subImages first'%labelImg_dir) # sub_img_label_txt = 'sub_images_labels_list_test.txt' sub_img_label_txt = 'sub_images_labels_list_1.txt' if os.path.isfile(sub_img_label_txt) is False: raise IOError('%s not in the current folder, please get subImages first' % sub_img_label_txt) with open(sub_img_label_txt) as txt_obj: line_list = [name.strip() for name in txt_obj.readlines()] for line in line_list: split_sub_images.split_a_pair_sub_image_label(line, patch_w, patch_h, overlay_x, overlay_y, split_image_format)
def main(options, args): img_path = args[0] # if nodata is not set, it will try to read from images. img_nodata = options.nodata output_shp = options.output_shp if output_shp is None: output_shp = os.path.splitext(os.path.basename(img_path))[0] + '_valid.shp' # out_mask = options.output_mask valid_mask = get_valid_pixel_mask(img_path,img_nodata,out_mask=out_mask) # to shapefile raster_io.raster2shapefile(valid_mask,out_shp=output_shp,nodata=0) # the nodata for valid_mask is 0 if valid_mask=='tmp.tif': io_function.delete_file_or_dir(valid_mask)
def predict_remoteSensing_image(para_file, image_path, save_dir,model, config_file, yolo_data, batch_size=1, b_python_api=True): ''' run prediction of a remote sensing using yolov4 :param image_path: :param model: :param config_file: :param yolo_data: :param batch_size: :param b_python_api: if true, use the python API of yolo :return: ''' patch_w = parameters.get_digit_parameters(para_file, "inf_patch_width", 'int') patch_h = parameters.get_digit_parameters(para_file, "inf_patch_height", 'int') overlay_x = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_x", 'int') overlay_y = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_y", 'int') if b_python_api: # using the python API predict_rs_image_yolo_poythonAPI(image_path, save_dir, model, config_file, yolo_data, patch_w, patch_h, overlay_x, overlay_y, batch_size=batch_size) # for each patch has a json file, may end up with a lot of json files, affect I/O # try to merge them to one json file. res_json_files = io_function.get_file_list_by_ext('.json', save_dir, bsub_folder=False) merge_josn_path = os.path.join(save_dir,'all_patches.json') merge_patch_json_files_to_one(res_json_files,merge_josn_path) for f_json in res_json_files: io_function.delete_file_or_dir(f_json) else: # divide image the many patches, then run prediction. patch_list_txt = split_an_image(para_file,image_path,save_dir,patch_w,patch_h,overlay_x,overlay_y) if patch_list_txt is None: return False result_json = save_dir + '_result.json' commond_str = 'darknet detector test ' + yolo_data + ' ' + config_file + ' ' + model + ' -dont_show ' commond_str += ' -ext_output -out ' + result_json + ' < ' + patch_list_txt print(commond_str) res = os.system(commond_str) if res !=0: sys.exit(1)
def dem_to_tpi_save_8bit(input, output): if os.path.isfile(output): basic.outputlogMessage('%s exists, skip' % output) return True if os.path.isfile(input) is False: basic.outputlogMessage('Waring, %s does not exist' % input) return False # Topographic Position Index tpi_file = os.path.basename( io_function.get_name_by_adding_tail(input, 'tpi')) command_str = 'gdaldem TPI %s %s -of GTiff -co compress=lzw -co tiled=yes -co bigtiff=if_safer -b 1 ' % ( input, tpi_file) basic.os_system_exit_code(command_str) # to 8bit if tpi_to_8bit(tpi_file, output) is True: io_function.delete_file_or_dir(tpi_file) return True
def zonal_stats_one_polygon(idx, polygon, image_tiles, img_tile_polygons, stats, nodata=None,range=None, band = 1,all_touched=True): overlap_index = vector_gpd.get_poly_index_within_extent(img_tile_polygons, polygon) image_list = [image_tiles[item] for item in overlap_index] if len(image_list) == 1: out_image, out_tran,nodata = raster_io.read_raster_in_polygons_mask(image_list[0], polygon, nodata=nodata, all_touched=all_touched,bands=band) elif len(image_list) > 1: # for the case it overlap more than one raster, need to produce a mosaic tmp_saved_files = [] for k_img, image_path in enumerate(image_list): # print(image_path) tmp_save_path = os.path.splitext(os.path.basename(image_path))[0] + '_subset_poly%d'%idx +'.tif' _, _,nodata = raster_io.read_raster_in_polygons_mask(image_path, polygon,all_touched=all_touched,nodata=nodata, bands=band, save_path=tmp_save_path) tmp_saved_files.append(tmp_save_path) # mosaic files in tmp_saved_files save_path = 'raster_for_poly%d.tif'%idx mosaic_args_list = ['gdal_merge.py', '-o', save_path,'-n',str(nodata),'-a_nodata',str(nodata)] mosaic_args_list.extend(tmp_saved_files) if basic.exec_command_args_list_one_file(mosaic_args_list,save_path) is False: raise IOError('error, obtain a mosaic (%s) failed'%save_path) # read the raster out_image, out_nodata = raster_io.read_raster_one_band_np(save_path,band=band) # remove temporal raster tmp_saved_files.append(save_path) for item in tmp_saved_files: io_function.delete_file_or_dir(item) else: basic.outputlogMessage('warning, cannot find raster for %d (start=0) polygon'%idx) return None # do calculation return array_stats(out_image, stats, nodata,range=range)
def main(): reg_tif_dir = 'arcticdem_registration_tifs' while True: print(str(datetime.now()), 'start moving or removing files or folders\n\n') reg_files = io_function.get_file_list_by_pattern(reg_tif_dir, '*') print('registration file count: %d in %s' % (len(reg_files), reg_tif_dir)) for file in reg_files: if check_file_or_dir_is_old(file, time_hour_thr): print( '%s is older than %f hours, will be moved to archieved dir' % (file, time_hour_thr)) io_function.movefiletodir(file, arcticDEM_reg_tif_dir, overwrite=True) SETSM_dir = io_function.get_file_list_by_pattern( './', 'SETSM_*2m_v3.0') print('SETSM folder count: %d in %s' % (len(SETSM_dir), './')) for folder in SETSM_dir: if check_file_or_dir_is_old(folder, time_hour_thr): print('%s is older than %f hours, will be removed' % (folder, time_hour_thr)) io_function.delete_file_or_dir(folder) grid_tmp_dir = io_function.get_file_list_by_pattern('./', 'grid*files') print('Grid tmp folder count: %d in %s' % (len(grid_tmp_dir), './')) for folder in grid_tmp_dir: if check_file_or_dir_is_old(folder, time_hour_thr): print('%s is older than %f hours, will be removed' % (folder, time_hour_thr)) io_function.delete_file_or_dir(folder) time.sleep(60) # wait pass
def get_sub_images_multi_regions(para_file): print( "extract sub-images and sub-labels for a given shape file (training polygons)" ) if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd())) get_subImage_script = os.path.join(code_dir, 'datasets', 'get_subImages.py') SECONDS = time.time() # get name of training areas multi_training_regions = parameters.get_string_list_parameters_None_if_absence( para_file, 'training_regions') if multi_training_regions is None or len(multi_training_regions) < 1: raise ValueError('No training area is set in %s' % para_file) # multi_training_files = parameters.get_string_parameters_None_if_absence(para_file, 'multi_training_files') dstnodata = parameters.get_string_parameters(para_file, 'dst_nodata') buffersize = parameters.get_string_parameters(para_file, 'buffer_size') rectangle_ext = parameters.get_string_parameters(para_file, 'b_use_rectangle') process_num = parameters.get_digit_parameters(para_file, 'process_num', 'int') b_no_label_image = parameters.get_bool_parameters_None_if_absence( para_file, 'b_no_label_image') if os.path.isfile('sub_images_labels_list.txt'): io_function.delete_file_or_dir('sub_images_labels_list.txt') subImage_dir = parameters.get_string_parameters_None_if_absence( para_file, 'input_train_dir') subLabel_dir = parameters.get_string_parameters_None_if_absence( para_file, 'input_label_dir') # loop each training regions for idx, area_ini in enumerate(multi_training_regions): input_image_dir = parameters.get_directory_None_if_absence( area_ini, 'input_image_dir') # it is ok consider a file name as pattern and pass it the following functions to get file list input_image_or_pattern = parameters.get_string_parameters( area_ini, 'input_image_or_pattern') b_sub_images_json = parameters.get_bool_parameters( area_ini, 'b_sub_images_json') if b_sub_images_json is True: # copy sub-images, then covert json files to label images. object_names = parameters.get_string_list_parameters( para_file, 'object_names') get_subImages_json.get_subimages_label_josn( input_image_dir, input_image_or_pattern, subImage_dir, subLabel_dir, object_names, b_no_label_image=b_no_label_image, process_num=process_num) pass else: all_train_shp = parameters.get_file_path_parameters_None_if_absence( area_ini, 'training_polygons') train_shp = parameters.get_string_parameters( area_ini, 'training_polygons_sub') # get subImage and subLabel for one training polygons print( 'extract training data from image folder (%s) and polgyons (%s)' % (input_image_dir, train_shp)) if b_no_label_image is True: get_subImage_one_shp(get_subImage_script, all_train_shp, buffersize, dstnodata, rectangle_ext, train_shp, input_image_dir, file_pattern=input_image_or_pattern, process_num=process_num) else: get_subImage_subLabel_one_shp( get_subImage_script, all_train_shp, buffersize, dstnodata, rectangle_ext, train_shp, input_image_dir, file_pattern=input_image_or_pattern, process_num=process_num) # check black sub-images or most part of the sub-images is black (nodata) new_sub_image_label_list = [] delete_sub_image_label_list = [] subImage_dir_delete = subImage_dir + '_delete' subLabel_dir_delete = subLabel_dir + '_delete' io_function.mkdir(subImage_dir_delete) if b_no_label_image is None or b_no_label_image is False: io_function.mkdir(subLabel_dir_delete) get_valid_percent_entropy.plot_valid_entropy(subImage_dir) with open('sub_images_labels_list.txt', 'r') as f_obj: lines = f_obj.readlines() for line in lines: image_path, label_path = line.strip().split(':') # valid_per = raster_io.get_valid_pixel_percentage(image_path) valid_per, entropy = raster_io.get_valid_percent_shannon_entropy( image_path) # base=10 if valid_per > 60 and entropy >= 0.5: new_sub_image_label_list.append(line) else: delete_sub_image_label_list.append(line) io_function.movefiletodir(image_path, subImage_dir_delete) if os.path.isfile(label_path): io_function.movefiletodir(label_path, subLabel_dir_delete) if len(delete_sub_image_label_list) > 0: with open('sub_images_labels_list.txt', 'w') as f_obj: for line in new_sub_image_label_list: f_obj.writelines(line) # check weather they have the same subImage and subLabel if b_no_label_image is None or b_no_label_image is False: sub_image_list = io_function.get_file_list_by_pattern( subImage_dir, '*.tif') sub_label_list = io_function.get_file_list_by_pattern( subLabel_dir, '*.tif') if len(sub_image_list) != len(sub_label_list): raise ValueError( 'the count of subImage (%d) and subLabel (%d) is different' % (len(sub_image_list), len(sub_label_list))) # save brief information of sub-images height_list = [] width_list = [] band_count = 0 dtype = 'unknown' for line in new_sub_image_label_list: image_path, label_path = line.strip().split(':') height, width, band_count, dtype = raster_io.get_height_width_bandnum_dtype( image_path) height_list.append(height) width_list.append(width) # save info to file, if it exists, it will be overwritten img_count = len(new_sub_image_label_list) with open('sub_images_patches_info.txt', 'w') as f_obj: f_obj.writelines('information of sub-images: \n') f_obj.writelines('number of sub-images : %d \n' % img_count) f_obj.writelines('band count : %d \n' % band_count) f_obj.writelines('data type : %s \n' % dtype) f_obj.writelines('maximum width and height: %d, %d \n' % (max(width_list), max(height_list))) f_obj.writelines('minimum width and height: %d, %d \n' % (min(width_list), min(height_list))) f_obj.writelines( 'mean width and height: %.2f, %.2f \n\n' % (sum(width_list) / img_count, sum(height_list) / img_count)) duration = time.time() - SECONDS os.system( 'echo "$(date): time cost of getting sub images and labels: %.2f seconds">>time_cost.txt' % duration)
def segment_subsidence_grey_image(dem_diff_grey_8bit, dem_diff, save_dir,process_num, subsidence_thr_m=-0.5, min_area=40, max_area=100000000, b_rm_files=False): ''' segment subsidence areas based on 8bit dem difference :param dem_diff_grey_8bit: :param dem_diff: :param save_dir: :param process_num: :param subsidence_thr_m: mean value less than this one consider as subsidence (in meter) :param min_area: min size in m^2 (defualt is 40 m^2, 10 pixels on ArcticDEM) :param max_area: min size in m^2 (default is 10km by 10 km) :return: ''' io_function.is_file_exist(dem_diff_grey_8bit) out_pre = os.path.splitext(os.path.basename(dem_diff_grey_8bit))[0] segment_shp_path = os.path.join(save_dir, out_pre + '.shp') # check if the final result exist final_shp_path = io_function.get_name_by_adding_tail(segment_shp_path, 'post') if os.path.isfile(final_shp_path): basic.outputlogMessage('Warning, Final results (%s) of subsidence shapefile exists, skip'%final_shp_path) return True # get initial polygons # because the label from segmentation for superpixels are not unique, so we may need to get mean dem diff based on polygons, set org_raster=None label_path = segment_a_grey_image(dem_diff_grey_8bit,save_dir,process_num, org_raster=None) if os.path.isfile(segment_shp_path) and vector_gpd.is_field_name_in_shp(segment_shp_path,'demD_mean'): basic.outputlogMessage('%s exists, skip'%segment_shp_path) else: # remove segment_shp_path if it exist, but don't have demD_mean if os.path.isfile(segment_shp_path): io_function.delete_shape_file(segment_shp_path) # remove nodato (it was copy from the input image) command_str = 'gdal_edit.py -unsetnodata ' + label_path basic.os_system_exit_code(command_str) # convert the label to shapefile # remove -8 (to use 4 connectedness.) command_string = 'gdal_polygonize.py %s -b 1 -f "ESRI Shapefile" %s' % (label_path, segment_shp_path) res = os.system(command_string) if res != 0: sys.exit(1) # get dem elevation information for each polygon raster_statistic.zonal_stats_multiRasters(segment_shp_path, dem_diff, tile_min_overlap=tile_min_overlap, stats=['mean', 'std','count'], prefix='demD',process_num=process_num) # get DEM diff information for each polygon. dem_diff_shp = get_dem_subscidence_polygons(segment_shp_path, dem_diff, dem_diff_thread_m=subsidence_thr_m, min_area=min_area, max_area=max_area, process_num=process_num,b_rm_files=b_rm_files) if dem_diff_shp is None: id_str = re.findall('grid\d+', os.path.basename(dem_diff))[0][4:] if len(id_str) > 1: grid_id = int(id_str) save_id_grid_no_subsidence(grid_id) else: basic.outputlogMessage('obtain elevation reduction polygons: %s'%dem_diff_shp) ## remove files, only keep the final results. if b_rm_files: io_function.delete_file_or_dir(label_path) IDrange_txt = os.path.splitext(label_path)[0] + '_IDrange.txt' io_function.delete_file_or_dir(IDrange_txt) io_function.delete_shape_file(segment_shp_path) # other intermediate files other_shp_names = ['merged','surrounding','rmreldemD','rmshapeinfo','rmslope'] for name in other_shp_names: rm_shp = io_function.get_name_by_adding_tail(segment_shp_path, name) io_function.delete_shape_file(rm_shp) return True
def train_kfold_cross_val(multi_training_files_allPolygons, multi_training_files, k_value, test_num): ################################################################## # get subset of polygons training_shp_all = [] with open(multi_training_files_allPolygons, 'r') as f_obj: training_lines = f_obj.readlines() for line in training_lines: line = line.strip() training_shp_all.append( line.split(':')[-1]) # the last one is the shape file for training_shpAll in training_shp_all: dir = os.path.dirname(training_shpAll) file_name = os.path.basename(training_shpAll) file_name_no_ext = os.path.splitext(file_name)[0] dir_sub = os.path.join( dir, '%s_%d-fold_cross_val_t%d' % (file_name_no_ext, k_value, test_num)) if os.path.isdir(dir_sub) is False: # will save to dir_sub} io_function.mkdir(dir_sub) create_shp_subset_polygons(dir_sub, training_shpAll, file_name, k_value) else: # check shape file existence sub_shps = io_function.get_file_list_by_pattern(dir_sub, '*.shp') if len(sub_shps) == k_value: print2file( log, "subset of shapefile already exist, skip creating new") else: create_shp_subset_polygons(dir_sub, training_shpAll, file_name, k_value) ################################################################## # training on k subset for idx in range(1, k_value + 1): # remove previous trained model (the setting are the same to exp10) if os.path.isdir(trained_model_dir): io_function.delete_file_or_dir(trained_model_dir) print2file(log, "run training and inference of the %d_th fold" % idx) # replace shape file path in "multi_training_files" io_function.copy_file_to_dst(multi_training_files_allPolygons, multi_training_files, overwrite=True) # replace shape file path in multi_training_files for training_shpAll in training_shp_all: dir = os.path.dirname(training_shpAll) file_name_no_ext = os.path.splitext( os.path.basename(training_shpAll))[0] dir_sub = os.path.join( dir, '%s_%d-fold_cross_val_t%d' % (file_name_no_ext, k_value, test_num)) new_shp_path = os.path.join( dir_sub, '%s_%dfold_%d.shp' % (file_name_no_ext, k_value, idx)) repalce_string_in_file(multi_training_files, training_shpAll, new_shp_path) # modify exe.sh io_function.copy_file_to_dst('exe_template_kfold.sh', 'exe_qtp.sh', overwrite=True) new_line = '%dfold_%d_t%d' % (k_value, idx, test_num) repalce_string_in_file('exe_qtp.sh', 'x_test_num', new_line) # check results existence result_shp = io_function.get_file_list_by_pattern( 'result_backup', '*' + new_line + '*/*.shp') if len(result_shp) > 0: print2file(log, "results of test: %s already exist, skip" % new_line) else: # run training print2file(log, "start: test:%d the %d_th fold" % (test_num, idx)) argslist = ['./exe_qtp.sh'] return_code = basic.exec_command_args_list(argslist) # exit code is not 0, means something wrong, then quit if return_code != 0: sys.exit(return_code) pass
def remove_no_need_dem_files(b_remove=True): # if os.path.isfile(grid_complete_list_txt): # completed_id_list = [int(item) for item in io_function.read_list_from_txt(grid_complete_list_txt)] # else: # print(datetime.now(), 'no complete grids') # return True # # if os.path.isfile(grid_excluded_list_txt): # exclude_id_list = [int(item) for item in io_function.read_list_from_txt(grid_excluded_list_txt)] # completed_id_list.extend(exclude_id_list) completed_id_list = get_complete_ignore_grid_ids() if len(completed_id_list) < 1: print(datetime.now(), 'no complete grids') return True if len(completed_id_list) < 1: return True completed_id_set = set(completed_id_list) # check four folders: arcticDEM_tile_tarball_dir,arcticDEM_tile_reg_tif_dir,tarball_dir,arcticDEM_reg_tif_dir strip_dem_cover_grids = io_function.read_dict_from_txt_json( strip_dem_cover_grids_txt) strip_no_need_list = [ strip for strip in strip_dem_cover_grids.keys() if set(strip_dem_cover_grids[strip]).issubset(completed_id_set) ] tile_dem_cover_grids = io_function.read_dict_from_txt_json( tile_dem_cover_grids_txt) tile_no_need_list = [ tile for tile in tile_dem_cover_grids.keys() if set(tile_dem_cover_grids[tile]).issubset(completed_id_set) ] if b_remove is False: save_list_no_need_dem_files('no_need_ArcticDEM_strip_names.txt', strip_no_need_list) save_list_no_need_dem_files('no_need_ArcticDEM_mosaic_names.txt', tile_no_need_list) else: # remove basic.outputlogMessage( 'there are %d no need strip DEM, downloaded files will be or have been removed' % len(strip_no_need_list)) for strip in strip_no_need_list: file_list = io_function.get_file_list_by_pattern( tarball_dir, strip + '*') file_list_2 = io_function.get_file_list_by_pattern( arcticDEM_reg_tif_dir, strip + '*') file_list.extend(file_list_2) if len(file_list) > 0: for path in file_list: basic.outputlogMessage('removing %s' % path) io_function.delete_file_or_dir(path) basic.outputlogMessage( 'there are %d no need tile DEM, downloaded files will be or have been removed' % len(tile_no_need_list)) for tile in tile_no_need_list: file_list = io_function.get_file_list_by_pattern( arcticDEM_tile_tarball_dir, tile + '*') file_list_2 = io_function.get_file_list_by_pattern( arcticDEM_tile_reg_tif_dir, tile + '*') file_list.extend(file_list_2) # remove slope file derived ArcticDEM (mosaic) file_list_3 = io_function.get_file_list_by_pattern( arcticDEM_tile_slope_dir, tile + '*') file_list.extend(file_list_3) if len(file_list) > 0: for path in file_list: basic.outputlogMessage('removing %s' % path) io_function.delete_file_or_dir(path)
def convert_planet_to_rgb_images(tif_path, save_dir='RGB_images', sr_min=0, sr_max=3000, save_org_dir=None, sharpen=True, rgb_nodata=0): #if multiple processes try to derive the same rgb images, it may have problem. # save output to 'RGB_images' + processID if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) if save_org_dir is not None and os.path.isdir(save_org_dir) is False: io_function.mkdir(save_org_dir) if save_org_dir is not None: copied_org_img_path = os.path.join(save_org_dir, os.path.basename(tif_path)) io_function.copy_file_to_dst(tif_path, copied_org_img_path) # filename_no_ext output = os.path.splitext(os.path.basename(tif_path))[0] if sharpen: fin_output = os.path.join(save_dir, output + '_8bit_rgb_sharpen.tif') else: fin_output = os.path.join(save_dir, output + '_8bit_rgb.tif') if os.path.isfile(fin_output): basic.outputlogMessage( "Skip, because File %s exists in current folder: %s" % (fin_output, os.getcwd())) return fin_output # use fix min and max to make the color be consistent to sentinel-images src_min = sr_min src_max = sr_max dst_min = 1 # 0 is the nodata, so set as 1 dst_max = 255 # gdal_translate -ot Byte -scale ${src_min} ${src_max} ${dst_min} ${dst_max} ${image_path} ${output}_8bit.tif if 'SR.tif' in tif_path: cmd_str = 'gdal_translate -ot Byte -scale %d %d %d %d -of VRT %s %s_8bit.tif' % ( src_min, src_max, dst_min, dst_max, tif_path, output) else: # gdal_contrast_stretch -percentile-range 0.01 0.99 ${output}.tif ${output}_8bit.tif cmd_str = 'gdal_contrast_stretch -percentile-range 0.01 0.99 %s %s_8bit.tif' % ( tif_path, output) status, result = basic.exec_command_string(cmd_str) if status != 0: print(result) sys.exit(status) # the third band is red, second is green, and first is blue #gdal_translate -b 3 -b 2 -b 1 ${output}_8bit.tif ${output}_8bit_rgb.tif cmd_str = 'gdal_translate -b 3 -b 2 -b 1 -of VRT %s_8bit.tif %s_8bit_rgb.tif' % ( output, output) status, result = basic.exec_command_string(cmd_str) if status != 0: print(result) sys.exit(status) # python ${code_dir}/planetScripts/prePlanetImage.py ${output}_8bit_rgb.tif ${fin_output} if sharpen: cmd_str = 'python %s %s_8bit_rgb.tif %s' % (prePlanetImage, output, fin_output) else: # convert from VRT format to tif format cmd_str = 'gdal_translate -of GTiff %s_8bit_rgb.tif %s' % (output, fin_output) status, result = basic.exec_command_string(cmd_str) if status != 0: print(result) sys.exit(status) # set nodata # gdal_edit.py -a_nodata 0 ${fin_output} cmd_str = 'gdal_edit.py -a_nodata %d %s' % (rgb_nodata, fin_output) status, result = basic.exec_command_string(cmd_str) if status != 0: print(result) sys.exit(status) io_function.delete_file_or_dir('%s_8bit.tif' % output) io_function.delete_file_or_dir('%s_8bit_rgb.tif' % output) return fin_output
def get_sub_image(idx,selected_polygon, image_tile_list, image_tile_bounds, save_path, dstnodata, brectangle ): ''' get a mask image based on a selected polygon, it may cross two image tiles :param selected_polygon: selected polygons :param image_tile_list: image list :param image_tile_bounds: the boxes of images in the list :param save_path: save path :param brectangle: if brectangle is True, crop the raster using bounds, else, use the polygon :return: True is successful, False otherwise ''' # find the images which the center polygon overlap (one or two images) img_index = get_overlap_image_index([selected_polygon], image_tile_bounds) if len(img_index) < 1: basic.outputlogMessage( 'Warning, %dth polygon do not overlap any image tile, please check ' #and its buffer area '(1) the shape file and raster have the same projection' 'and (2) this polygon is in the extent of images' % idx) return False image_list = [image_tile_list[item] for item in img_index] # check it cross two or more images if len(image_list) == 1: # for the case that the polygon only overlap one raster with rasterio.open(image_list[0]) as src: polygon_json = mapping(selected_polygon) # not necessary # overlap_win = rasterio.features.geometry_window(src, [polygon_json], pad_x=0, pad_y=0, north_up=True, rotated=False, # pixel_precision=3) if brectangle: # polygon_box = selected_polygon.bounds polygon_json = mapping(selected_polygon.envelope) #shapely.geometry.Polygon([polygon_box]) # crop image and saved to disk out_image, out_transform = mask(src, [polygon_json], nodata=dstnodata, all_touched=True, crop=True) # test: save it to disk out_meta = src.meta.copy() out_meta.update({"driver": "GTiff", "height": out_image.shape[1], "width": out_image.shape[2], "transform": out_transform}) # note that, the saved image have a small offset compared to the original ones (~0.5 pixel) with rasterio.open(save_path, "w", **out_meta) as dest: dest.write(out_image) pass else: # for the case it overlap more than one raster, need to produce a mosaic tmp_saved_files = [] for k_img,image_path in enumerate(image_list): with rasterio.open(image_path) as src: polygon_json = mapping(selected_polygon) if brectangle: # polygon_box = selected_polygon.bounds polygon_json = mapping(selected_polygon.envelope) # shapely.geometry.Polygon([polygon_box]) # crop image and saved to disk out_image, out_transform = mask(src, [polygon_json], nodata=dstnodata, all_touched=True, crop=True) tmp_saved = os.path.splitext(save_path)[0] +'_%d'%k_img + os.path.splitext(save_path)[1] # test: save it to disk out_meta = src.meta.copy() out_meta.update({"driver": "GTiff", "height": out_image.shape[1], "width": out_image.shape[2], "transform": out_transform}) # note that, the saved image have a small offset compared to the original ones (~0.5 pixel) with rasterio.open(tmp_saved, "w", **out_meta) as dest: dest.write(out_image) tmp_saved_files.append(tmp_saved) # mosaic files in tmp_saved_files mosaic_args_list = ['gdal_merge.py', '-o', save_path,'-n',str(dstnodata),'-a_nodata',str(dstnodata)] mosaic_args_list.extend(tmp_saved_files) if basic.exec_command_args_list_one_file(mosaic_args_list,save_path) is False: raise IOError('error, obtain a mosaic (%s) failed'%save_path) # # for test # if idx==13: # raise ValueError('for test') # remove the tmp files for tmp_file in tmp_saved_files: io_function.delete_file_or_dir(tmp_file) # if it will output a very large image (10000 by 10000 pixels), then raise a error return True
basic.setlogfile('parallel_predict_rtsLog.txt') predict_script = HOME + '/codes/PycharmProjects/Landuse_DL/sentinelScripts/predict_rts_oneImg.sh' import GPUtil import datetime from multiprocessing import Process machine_name = os.uname()[1] start_time = datetime.datetime.now() # remove previous results outdir = 'multi_inf_results' if os.path.isdir(outdir) and 'chpc' not in machine_name: # on ITSC service, need to manually deleted previous results io_function.delete_file_or_dir(outdir) os.system('mkdir -p '+ outdir) # get GPU information on the machine # https://github.com/anderskm/gputil # deviceIDs = GPUtil.getAvailable(order = 'first', limit = 100, maxLoad = 0.5, # maxMemory = 0.5, includeNan=False, excludeID=[], excludeUUID=[]) # print('available GPUs:',deviceIDs) with open('inf_image_list.txt','r') as inf_obj: inf_img_list = [name.strip() for name in inf_obj.readlines()] img_count = len(inf_img_list) if img_count < 1:
def remove_previous_data(para_file): print("remove previous data or results to run again") if os.path.isfile(para_file) is False: raise IOError('File %s does not exists in current folder: %s' % (para_file, os.getcwd())) subImage_dir = parameters.get_string_parameters_None_if_absence( para_file, 'input_train_dir') subLabel_dir = parameters.get_string_parameters_None_if_absence( para_file, 'input_label_dir') if os.path.isdir(subImage_dir): io_function.delete_file_or_dir(subImage_dir) print('remove %s' % subImage_dir) if os.path.isdir(subLabel_dir): io_function.delete_file_or_dir(subLabel_dir) print('remove %s' % subLabel_dir) subImage_dir_delete = subImage_dir + '_delete' subLabel_dir_delete = subLabel_dir + '_delete' if os.path.isdir(subImage_dir_delete): io_function.delete_file_or_dir(subImage_dir_delete) print('remove %s' % subImage_dir_delete) if os.path.isdir(subLabel_dir_delete): io_function.delete_file_or_dir(subLabel_dir_delete) print('remove %s ' % subLabel_dir_delete) if os.path.isdir('split_images'): io_function.delete_file_or_dir('split_images') print('remove %s ' % 'split_images') if os.path.isdir('split_labels'): io_function.delete_file_or_dir('split_labels') print('remove %s ' % 'split_labels') images_including_aug = os.path.join('list', 'images_including_aug.txt') if os.path.isfile(images_including_aug): io_function.delete_file_or_dir(images_including_aug) print('remove %s ' % 'list/images_including_aug.txt') if os.path.isdir('tfrecord'): io_function.delete_file_or_dir('tfrecord') print('remove %s ' % 'tfrecord')
def split_sub_images(para_file): print("split sub-images and sub-labels") if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s'%(para_file, os.getcwd())) SECONDS = time.time() if os.path.isdir('split_images'): io_function.delete_file_or_dir('split_images') if os.path.isdir('split_labels'): io_function.delete_file_or_dir('split_labels') io_function.mkdir('split_images') ### split the training image to many small patch (480*480) patch_w=parameters.get_string_parameters(para_file,'train_patch_width') patch_h=parameters.get_string_parameters(para_file,'train_patch_height') overlay_x=parameters.get_string_parameters(para_file,'train_pixel_overlay_x') overlay_y=parameters.get_string_parameters(para_file,'train_pixel_overlay_y') split_image_format=parameters.get_string_parameters(para_file,'split_image_format') trainImg_dir=parameters.get_string_parameters(para_file,'input_train_dir') labelImg_dir=parameters.get_string_parameters(para_file,'input_label_dir') proc_num = parameters.get_digit_parameters(para_file,'process_num','int') if os.path.isdir(trainImg_dir) is False: raise IOError('%s not in the current folder, please get subImages first'%trainImg_dir) if os.path.isdir(labelImg_dir) is False: print('warning, %s not in the current folder'%labelImg_dir) else: io_function.mkdir('split_labels') sub_img_label_txt = 'sub_images_labels_list.txt' if os.path.isfile(sub_img_label_txt) is False: raise IOError('%s not in the current folder, please get subImages first' % sub_img_label_txt) with open(sub_img_label_txt) as txt_obj: line_list = [name.strip() for name in txt_obj.readlines()] # for line in line_list: # sub_image, sub_label = line.split(':') # # # split sub image # split_to_patches(sub_image, 'split_images', patch_w, patch_h, overlay, overlay, split_image_format) # # # split sub label (change the file name to be the same as sub_image name) # pre_name = os.path.splitext(os.path.basename(sub_image))[0] # split_to_patches(sub_label, 'split_labels', patch_w, patch_h, overlay, overlay, split_image_format, file_pre_name=pre_name) parameters_list = [(line, patch_w, patch_h, overlay_x, overlay_y, split_image_format) for line in line_list] theadPool = Pool(proc_num) # multi processes results = theadPool.starmap(split_a_pair_sub_image_label, parameters_list) # need python3 # output trainval.txt and val.txt file files_list = io_function.get_file_list_by_ext(split_image_format, 'split_images',bsub_folder=False) io_function.mkdir('list') trainval = os.path.join('list','trainval.txt') val = os.path.join('list','val.txt') with open(trainval,'w') as w_obj: for file_name in files_list: w_obj.writelines(os.path.splitext(os.path.basename(file_name))[0] + '\n') io_function.copy_file_to_dst(trainval,val,overwrite=True) split_train_val.get_image_with_height_list(trainval, split_image_format, info_type='(no data augmentation)') duration= time.time() - SECONDS os.system('echo "$(date): time cost of splitting sub images and labels: %.2f seconds">>time_cost.txt'%duration)
if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s'%(para_file, os.getcwd())) deeplabRS=os.path.expanduser('~/codes/PycharmProjects/DeeplabforRS') sys.path.insert(0, deeplabRS) import parameters import basic_src.io_function as io_function eo_dir=os.path.expanduser("~/codes/PycharmProjects/Landuse_DL") # split_image_script=os.path.join(eo_dir,'grss_data_fusion', 'split_image.py') sys.path.insert(0, eo_dir) import grss_data_fusion.split_image as split_image if os.path.isdir('split_images'): io_function.delete_file_or_dir('split_images') if os.path.isdir('split_labels'): io_function.delete_file_or_dir('split_labels') io_function.mkdir('split_images') io_function.mkdir('split_labels') ### split the training image to many small patch (480*480) patch_w=parameters.get_string_parameters(para_file,'train_patch_width') patch_h=parameters.get_string_parameters(para_file,'train_patch_height') overlay=parameters.get_string_parameters(para_file,'train_pixel_overlay_x') split_image_format=parameters.get_string_parameters(para_file,'split_image_format') trainImg_dir=parameters.get_string_parameters(para_file,'input_train_dir') labelImg_dir=parameters.get_string_parameters(para_file,'input_label_dir')
def produce_products_dem_subsidence(b_remove_job_folder=True): # run segment jobs in local workstations. task = 'segment' max_list_count = 20 if 'donostia' in machine_name: max_list_count = 8 # donostia is really slow, assigined less task to it job_list_pre = 'job_seg_dem_diff_list_' if os.path.isdir(dem_common.process_log_dir) is False: io_function.mkdir(dem_common.process_log_dir) dem_list_txt = os.path.join(dem_common.process_log_dir, job_list_pre + machine_name + '.txt') # when submit segment of dem_diff, no need ext_shp ext_shp = "monitor_fail_segment_jobs" while True: dem_diff_list = get_dem_diff_list_to_seg() # only handle file are old enough dem_diff_list = get_dem_diff_old_enough(dem_diff_list) dem_diff_ids = [get_grid_id_from_path(item) for item in dem_diff_list] print('dem_diff_ids') print(dem_diff_ids) # remove dem_diff already assigined for other machine if os.path.isfile(dem_list_txt): io_function.delete_file_or_dir(dem_list_txt) dem_diff_assigned = read_dem_diff_assigned_to_other_machine( job_list_pre) assigned_ids = [ get_grid_id_from_path(item) for item in dem_diff_assigned ] print('assigned_ids') print(assigned_ids) keep_idx = [ idx for idx, id in enumerate(dem_diff_ids) if id not in assigned_ids ] dem_diff_list = [dem_diff_list[item] for item in keep_idx] if len(dem_diff_list) < 1: print( datetime.now(), 'there is no DEM_diff for %s to seg, wait 10 minutes' % machine_name) time.sleep(600) # wait 10 min continue # save some of them to txt, for "parallel_processing_curc.py" dem_diff_list = dem_diff_list[:max_list_count] save_ids = [get_grid_id_from_path(item) for item in dem_diff_list] print('save_ids') print(save_ids) io_function.save_list_to_txt(dem_list_txt, dem_diff_list) res = os.system('./run.sh %s %s' % (ext_shp, task)) if res != 0: sys.exit(1) copy_segment_result_to_curc(save_ids) if b_remove_job_folder: os.system('rm -r seg_dem_diff_*') io_function.delete_file_or_dir(dem_list_txt)
def process_dem_tarball(tar_list, work_dir, save_dir, remove_inter_data=False, rm_tarball=False, apply_registration=False): ''' process dem tarball, unpack, apply registration :param tar_list: :param work_dir: :param save_dir: :param remove_inter_data: :param apply_registration: :return: ''' if os.path.isdir(save_dir) is False: io_function.mkdir(save_dir) if os.path.isfile('no_registration_strips.txt'): no_registration_strips = io_function.read_list_from_txt( 'no_registration_strips.txt') else: no_registration_strips = [] out_dir_list = [] out_reg_tifs = [] for idx, targz in enumerate(tar_list): tar_base = os.path.basename(targz)[:-7] # check if no registraion information for this tarball if './' + tar_base in no_registration_strips: continue if check_files_existence(save_dir, tar_base): print("registration result of %s already exists, skip" % targz) continue # check free disk space free_GB = io_function.get_free_disk_space_GB(work_dir) total_wait_time = 0 while free_GB < 50 and total_wait_time < 60 * 60 * 12: basic.outputlogMessage( ' The free disk space (%.4f) is less than 50 GB, wait 60 seconds' % free_GB) time.sleep(60) total_wait_time += 60 free_GB = io_function.get_free_disk_space_GB(work_dir) out_tif, out_dir = process_dem_one_tarball(targz, work_dir, apply_registration) if out_tif is None: if rm_tarball: io_function.delete_file_or_dir(targz) continue out_dir_list.append(out_dir) # move file to save_dir io_function.movefiletodir(out_tif, save_dir) dem_log = os.path.join(out_dir, tar_base + '_dem.log') if os.path.isfile(dem_log): io_function.movefiletodir(dem_log, save_dir) matchtag_tif = os.path.join(out_dir, tar_base + '_matchtag_reg.tif') if os.path.isfile(matchtag_tif): io_function.movefiletodir(matchtag_tif, save_dir) matchtag_tif_log = os.path.join(out_dir, tar_base + '_matchtag.log') if os.path.isfile(matchtag_tif_log): io_function.movefiletodir(matchtag_tif_log, save_dir) out_reg_tifs.append(os.path.join(save_dir, os.path.basename(out_tif))) # remove folder if remove_inter_data: io_function.delete_file_or_dir(out_dir) if rm_tarball: io_function.delete_file_or_dir(targz) # remove folder (in case failed in the previous step) if remove_inter_data: for dir in out_dir_list: if os.path.isdir(dir): io_function.delete_file_or_dir(dir) return out_reg_tifs
def proc_ArcticDEM_strip_one_grid_polygon(tar_dir, dem_polygons, dem_urls, o_res, save_dir, inter_format, b_mosaic_id, b_mosaic_date, b_rm_inter, b_dem_diff, extent_poly, extent_id, keep_dem_percent, process_num, pre_name, resample_method='average', same_extent=False): if check_dem_diff_results(save_dir, pre_name, extent_id): return True # get file in the tar_dir tar_list = get_tar_list_sub(tar_dir, dem_polygons, dem_urls, extent_poly) if len(tar_list) < 1: basic.outputlogMessage( 'Warning, no tarball for the extent (id=%d) in %s' % (extent_id, tar_dir)) return False # unpackage and crop to extent b_apply_registration = True dem_tif_list, tar_folders = process_dem_tarball( tar_list, save_dir, inter_format, o_res, extent_poly=extent_poly, poly_id=extent_id, same_extent=same_extent, process_num=process_num, apply_registration=b_apply_registration) if len(dem_tif_list) < 1: raise ValueError('No DEM extracted from tarballs') proc_dem_mosaic_diff(dem_tif_list, save_dir, extent_id, extent_poly, b_mosaic_id, b_mosaic_date, process_num, keep_dem_percent, o_res, b_dem_diff, pre_name, b_rm_inter, resample_method=resample_method) # remove intermediate files if b_rm_inter: basic.outputlogMessage('remove intermediate files') for folder in tar_folders: io_function.delete_file_or_dir(folder)
eo_dir = os.path.expanduser("~/codes/PycharmProjects/Landuse_DL") get_subImage_script = os.path.join(eo_dir, 'sentinelScripts', 'get_subImages.py') multi_training_files = parameters.get_string_parameters_None_if_absence( para_file, 'multi_training_files') input_image_dir = parameters.get_string_parameters(para_file, 'input_image_dir') dstnodata = parameters.get_string_parameters(para_file, 'dst_nodata') buffersize = parameters.get_string_parameters(para_file, 'buffer_size') rectangle_ext = parameters.get_string_parameters(para_file, 'b_use_rectangle') if os.path.isfile('sub_images_labels_list.txt'): io_function.delete_file_or_dir('sub_images_labels_list.txt') def get_subImage_subLabel_one_shp(all_train_shp, buffersize, dstnodata, rectangle_ext, train_shp, input_image_dir, file_pattern=None): if file_pattern is None: file_pattern = '*.tif' command_string = get_subImage_script + ' -f ' + all_train_shp + ' -b ' + str(buffersize) + ' -e ' + file_pattern + \ ' -o ' + os.getcwd() + ' -n ' + str(dstnodata) + ' ' + rectangle_ext + ' ' + train_shp + ' '+ input_image_dir
def create_moasic_of_each_grid_polygon(id, polygon, polygon_latlon, out_res, cloud_cover_thr, geojson_list, save_dir, new_prj_wkt=None, new_prj_proj4=None, sr_min=0, sr_max=3000, to_rgb=True, nodata=0, save_org_dir=None, resampling_method='min'): ''' create mosaic for Planet images within a grid :param polygon: :param polygon_latlon: :param out_res: :param cloud_cover_thr: :param geojson_list: :param save_dir: :param new_prj_wkt: :param new_prj_proj4: :param sr_min: :param sr_max: :param to_rgb: :param nodata: :return: ''' time0 = time.time() file_name = os.path.basename(save_dir) fin_out = os.path.join(save_dir, file_name + '_sub_%d.tif' % id) if os.path.isfile(fin_out): basic.outputlogMessage( 'Warning, skip %s because it already exists, remove it if want to regenerate it' % fin_out) return fin_out # get image list and cloud cover planet_img_list, cloud_covers = get_Planet_SR_image_list_overlap_a_polygon( polygon_latlon, geojson_list, cloud_cover_thr) if len(planet_img_list) < 1: basic.outputlogMessage('warning, no images within %d grid' % id) return False io_function.mkdir(save_dir) print('images and their cloud cover for %dth grid:' % id) for img, cloud_cover in zip(planet_img_list, cloud_covers): print(img, cloud_cover) proc_id = multiprocessing.current_process().pid # convert to RGB images (for Planet) rgb_image_list = [] rgb_dir = 'RGB_images_' + str(proc_id) if to_rgb: for tif_path in planet_img_list: rgb_img = convert_planet_to_rgb_images(tif_path, save_dir=rgb_dir, save_org_dir=save_org_dir, sr_min=sr_min, sr_max=sr_max) rgb_image_list.append(rgb_img) if len(rgb_image_list) > 0: planet_img_list = rgb_image_list reproj_img_list = [] # reproject if necessary reproj_dir = 'planet_images_reproj_' + str(proc_id) if new_prj_wkt != None and new_prj_proj4 != None: for tif_path in planet_img_list: prj_out = reproject_planet_image(tif_path, new_prj_wkt, new_prj_proj4, save_dir=reproj_dir) # replace the image if prj_out is not False and os.path.isfile(prj_out): reproj_img_list.append(prj_out) else: # if not reproject, then append the original image. reproj_img_list.append(tif_path) if len(reproj_img_list) > 0: planet_img_list = reproj_img_list # create mosaic using gdal_merge.py # because in gdal_merge.py, a later image will replace one, so we put image with largest cloud cover first out = os.path.join(save_dir, file_name + '_sub_%d_tmp.tif' % id) if os.path.isfile(out): io_function.delete_file_or_dir(out) # reverse=True to make it in descending order img_cloud_list = [ (img_path, cloud) for cloud, img_path in sorted(zip(cloud_covers, planet_img_list), key=lambda pair: pair[0], reverse=True) ] # for checking print('Image and its cloud after sorting:') for (img_path, cloud) in img_cloud_list: print(img_path, cloud) tifs = [img_path for (img_path, cloud) in img_cloud_list] tifs_str = ' '.join(tifs) # cmd_str = 'gdal_merge.py -o %s -n %d -init %d -ps %d %d %s'%(out,nodata,nodata,out_res,out_res,tifs_str) cmd_str = 'gdalbuildvrt -resolution user -tr %d %d -srcnodata %d -vrtnodata %d %s %s' % ( out_res, out_res, nodata, nodata, out, tifs_str) status, result = basic.exec_command_string(cmd_str) if status != 0: print(result) sys.exit(status) # # # polygon.exterior.coords # minx, miny, maxx, maxy = polygon.bounds # (minx, miny, maxx, maxy) # print(minx, miny, maxx, maxy) # results = RSImageProcess.subset_image_projwin(fin_out,out,minx, maxy, maxx, miny, xres=out_res,yres=out_res) # print(results) results = RSImageProcess.subset_image_by_polygon_box_image_min( fin_out, out, polygon, xres=out_res, yres=out_res, compress='lzw', tiled='yes', bigtiff='if_safer') if results is False: basic.outputlogMessage( 'Warning, Crop %s failed, keep the one without cropping' % out) io_function.move_file_to_dst(out, fin_out) else: io_function.delete_file_or_dir(out) # ## mosaic and crop at the same time together # minx, miny, maxx, maxy = polygon.bounds # (minx, miny, maxx, maxy) # print(minx, miny, maxx, maxy) # results = RSImageProcess.mosaic_crop_images_gdalwarp(tifs,fin_out,src_nodata=nodata,min_x=minx,min_y=miny,max_x=maxx,max_y=maxy, # xres=out_res,yres=out_res,resampling_method=resampling_method) # # if results is False: # basic.outputlogMessage('Warning, create %s failed' % fin_out) # return False # sys.exit(0) cost_time_sec = time.time() - time0 basic.outputlogMessage( 'finished creating %s cost %.2f seconds (%.2f minutes)' % (fin_out, cost_time_sec, cost_time_sec / 60)) return fin_out