def read_para_values(work_dir,para_file,train_output): para_path = os.path.join(work_dir,para_file) backbone = parameters.get_string_parameters(para_path,'network_setting_ini') train_output['network_setting_ini'].append(backbone) net_ini_path = os.path.join(work_dir,backbone) lr = parameters.get_digit_parameters(net_ini_path,'base_learning_rate','float') train_output['base_learning_rate'].append(lr) iter_num = parameters.get_digit_parameters(net_ini_path,'iteration_num','int') train_output['iteration_num'].append(iter_num) batch_size = parameters.get_digit_parameters(net_ini_path,'batch_size','int') train_output['batch_size'].append(batch_size) buffer_size = parameters.get_digit_parameters(para_path,'buffer_size','int') train_output['buffer_size'].append(buffer_size) training_data_per = parameters.get_digit_parameters(para_path,'training_data_per','float') train_output['training_data_per'].append(training_data_per) data_augmentation = parameters.get_string_parameters(para_path,'data_augmentation') train_output['data_augmentation'].append(data_augmentation) data_aug_ignore_classes = parameters.get_string_parameters(para_path,'data_aug_ignore_classes') train_output['data_aug_ignore_classes'].append(data_aug_ignore_classes) return True
def generate_image_CUT(python_path, generate_script, gan_para_file, gpu_ids, image_list, save_folder): if os.path.isfile('generate.txt_done'): basic.outputlogMessage( 'generate of new images using GAN in %s has completed previoulsy, please remove them if necessary' % os.getcwd()) return True time0 = time.time() generate_tile_width = parameters.get_digit_parameters( gan_para_file, 'generate_tile_width', 'int') generate_tile_height = parameters.get_digit_parameters( gan_para_file, 'generate_tile_height', 'int') generate_overlay_x = parameters.get_digit_parameters( gan_para_file, 'generate_overlay_x', 'int') generate_overlay_y = parameters.get_digit_parameters( gan_para_file, 'generate_overlay_y', 'int') folder = os.path.basename(os.getcwd()) img_list_txt = 'image_to_generate_list.txt' io_function.save_list_to_txt(img_list_txt, image_list) command_string = python_path + ' ' + generate_script \ + ' --dataset_mode '+'satelliteimage' \ + ' --model '+ 'generate' \ + ' --image_A_dir_txt ' + img_list_txt \ + ' --tile_width ' + str(generate_tile_width) \ + ' --tile_height ' + str(generate_tile_height) \ + ' --overlay_x ' + str(generate_overlay_x) \ + ' --overlay_y ' + str(generate_overlay_y) \ + ' --name ' + folder \ + ' --results_dir ' + save_folder \ + ' --gpu_ids ' + ','.join([str(item) for item in gpu_ids]) train_max_dataset_size = parameters.get_digit_parameters_None_if_absence( gan_para_file, 'gen_max_dataset_size', 'int') if train_max_dataset_size is not None: command_string += ' --max_dataset_size ' + str(train_max_dataset_size) # if it's cycleGAN, need to assign A generator gan_model = parameters.get_string_parameters(gan_para_file, 'gan_model') if gan_model == 'cycle_gan': command_string += ' --model_suffix _A ' # from A to B # status, result = basic.exec_command_string(command_string) # this will wait command finished # os.system(command_string + "&") # don't know when it finished res = os.system(command_string) # this work # print('command_string deeplab_inf_script: res',res) if res != 0: sys.exit(1) duration = time.time() - time0 os.system( 'echo "$(date): time cost of generate images using a GAN : %.2f seconds">>"time_cost.txt"' % (duration)) # write a file to indicate that the process has completed. os.system('echo done > generate.txt_done') return True
def get_sub_images_from_prediction_results(para_file, polygons_shp, image_folder_or_path, image_pattern, saved_dir): class_names = parameters.get_string_list_parameters( para_file, 'object_names') dstnodata = parameters.get_digit_parameters(para_file, 'dst_nodata', 'int') bufferSize = parameters.get_digit_parameters(para_file, 'buffer_size', 'int') rectangle_ext = parameters.get_string_parameters_None_if_absence( para_file, 'b_use_rectangle') if rectangle_ext is not None: b_rectangle = True else: b_rectangle = False process_num = parameters.get_digit_parameters(para_file, 'process_num', 'int') get_sub_images_pixel_json_files(polygons_shp, image_folder_or_path, image_pattern, class_names, bufferSize, dstnodata, saved_dir, b_rectangle, process_num) pass
def get_iteration_num(WORK_DIR, para_file, network_setting_ini): ''' get iteration num from setting (iteration_num) or calculate from train_epoch_num, batch_size and training sample count :param WORK_DIR: :param para_file: :param network_setting_ini: :return: ''' train_epoch_num = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_epoch_num', 'int') if train_epoch_num is None: iteration_num = parameters.get_digit_parameters( network_setting_ini, 'iteration_num', 'int') with open('iteration_num.txt', 'w') as f_obj: f_obj.writelines('iteration num is read from setting: %d\n' % iteration_num) else: train_count, val_count = get_train_val_sample_count( WORK_DIR, para_file) batch_size = parameters.get_digit_parameters(network_setting_ini, 'batch_size', 'int') iteration_num = math.ceil(train_epoch_num * train_count / batch_size) with open('iteration_num.txt', 'w') as f_obj: f_obj.writelines( 'iteration num (%d) is calculated from train_epoch_num: %d, training_sample_count: %d, ' 'batch_size: %d\n' % (iteration_num, train_epoch_num, train_count, batch_size)) return iteration_num
def predict_one_image_mmseg(para_file, image_path, img_save_dir, inf_list_file, gpuid, trained_model): """ run prediction of one image """ expr_name = parameters.get_string_parameters(para_file, 'expr_name') network_ini = parameters.get_string_parameters(para_file, 'network_setting_ini') base_config_file = parameters.get_string_parameters( network_ini, 'base_config') config_file = osp.basename( io_function.get_name_by_adding_tail(base_config_file, expr_name)) inf_batch_size = parameters.get_digit_parameters(network_ini, 'inf_batch_size', 'int') patch_width = parameters.get_digit_parameters(para_file, 'inf_patch_width', 'int') patch_height = parameters.get_digit_parameters(para_file, 'inf_patch_height', 'int') adj_overlay_x = parameters.get_digit_parameters(para_file, 'inf_pixel_overlay_x', 'int') adj_overlay_y = parameters.get_digit_parameters(para_file, 'inf_pixel_overlay_y', 'int') done_indicator = '%s_done' % inf_list_file if os.path.isfile(done_indicator): basic.outputlogMessage('warning, %s exist, skip prediction' % done_indicator) return if os.path.isdir(img_save_dir) is False: io_function.mkdir(img_save_dir) # use a specific GPU for prediction, only inference one image time0 = time.time() if gpuid is None: gpuid = 0 predict_rsImage_mmseg(config_file, trained_model, image_path, img_save_dir, batch_size=inf_batch_size, gpuid=gpuid, tile_width=patch_width, tile_height=patch_height, overlay_x=adj_overlay_x, overlay_y=adj_overlay_y) duration = time.time() - time0 os.system( 'echo "$(date): time cost of inference for image in %s: %.2f seconds">>"time_cost.txt"' % (inf_list_file, duration)) # write a file to indicate that the prediction has done. os.system('echo %s > %s_done' % (inf_list_file, inf_list_file)) return
def updated_config_file(WORK_DIR, expr_name, base_config_file, save_path, para_file, network_setting_ini, gpu_num): ''' update mmseg config file and save to a new file in local working folder :param WORK_DIR: :param expr_name: :param base_config_file: :param save_path: :param para_file: :param network_setting_ini: :return: ''' # read the config, then modifi some, them dump to disk cfg = Config.fromfile(base_config_file) # 4 basic component: dataset, model, schedule, default_runtime # change model (no need): when we choose a base_config_file, we already choose a model, including backbone) # change dataset modify_dataset(cfg, para_file, network_setting_ini, gpu_num) # change num_classes # NUM_CLASSES_noBG = parameters.get_digit_parameters(para_file,'NUM_CLASSES_noBG','int') # cfg.model.decode_head['num_classes'] = NUM_CLASSES_noBG + 1 # cfg.model.auxiliary_head['num_classes'] = NUM_CLASSES_noBG + 1 # change schedule iteration_num = get_iteration_num(WORK_DIR, para_file, network_setting_ini) cfg.runner['max_iters'] = iteration_num checkpoint_interval = parameters.get_digit_parameters( network_setting_ini, 'checkpoint_interval', 'int') cfg.checkpoint_config['interval'] = checkpoint_interval evaluation_interval = parameters.get_digit_parameters( network_setting_ini, 'evaluation_interval', 'int') cfg.evaluation['interval'] = evaluation_interval # change runtime (log level, resume_from or load_from) cfg.work_dir = os.path.join(WORK_DIR, expr_name) # update parameters for testing (used in later step of prediction) updated_config_file_for_test(cfg) # dump config cfg.dump(save_path) return True
def predict_remoteSensing_image(para_file, image_path, save_dir,model, config_file, yolo_data, batch_size=1, b_python_api=True): ''' run prediction of a remote sensing using yolov4 :param image_path: :param model: :param config_file: :param yolo_data: :param batch_size: :param b_python_api: if true, use the python API of yolo :return: ''' patch_w = parameters.get_digit_parameters(para_file, "inf_patch_width", 'int') patch_h = parameters.get_digit_parameters(para_file, "inf_patch_height", 'int') overlay_x = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_x", 'int') overlay_y = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_y", 'int') if b_python_api: # using the python API predict_rs_image_yolo_poythonAPI(image_path, save_dir, model, config_file, yolo_data, patch_w, patch_h, overlay_x, overlay_y, batch_size=batch_size) # for each patch has a json file, may end up with a lot of json files, affect I/O # try to merge them to one json file. res_json_files = io_function.get_file_list_by_ext('.json', save_dir, bsub_folder=False) merge_josn_path = os.path.join(save_dir,'all_patches.json') merge_patch_json_files_to_one(res_json_files,merge_josn_path) for f_json in res_json_files: io_function.delete_file_or_dir(f_json) else: # divide image the many patches, then run prediction. patch_list_txt = split_an_image(para_file,image_path,save_dir,patch_w,patch_h,overlay_x,overlay_y) if patch_list_txt is None: return False result_json = save_dir + '_result.json' commond_str = 'darknet detector test ' + yolo_data + ' ' + config_file + ' ' + model + ' -dont_show ' commond_str += ' -ext_output -out ' + result_json + ' < ' + patch_list_txt print(commond_str) res = os.system(commond_str) if res !=0: sys.exit(1)
def modify_dataset(cfg, para_file, network_setting_ini, gpu_num): datetype = 'RSImagePatches' cfg.dataset_type = datetype cfg.data_root = './' ## There are still two more crop_size in pipleline, after choosing the base_config_file, we already choose the crop size # image_crop_size = parameters.get_string_list_parameters(para_file, 'image_crop_size') # image_crop_size = [ int(item) for item in image_crop_size] # if len(image_crop_size) != 2 and image_crop_size[0].isdigit() and image_crop_size[1].isdigit(): # raise ValueError('image_crop_size should be height,width') # cfg.crop_size = (image_crop_size[0],image_crop_size[1]) training_sample_list_txt = parameters.get_string_parameters( para_file, 'training_sample_list_txt') validation_sample_list_txt = parameters.get_string_parameters( para_file, 'validation_sample_list_txt') split_list = ['train', 'val', 'test'] for split in split_list: # dataset in train cfg.data[split]['type'] = datetype cfg.data[split]['data_root'] = './' cfg.data[split]['img_dir'] = 'split_images' cfg.data[split]['ann_dir'] = 'split_labels' if split == 'train': cfg.data[split]['split'] = [ osp.join('list', training_sample_list_txt) ] else: # set val and test to validation, when run real test (prediction) for entire RS images, we will set test again. cfg.data[split]['split'] = [ osp.join('list', validation_sample_list_txt) ] # set None for test cfg.data['test']['img_dir'] = None cfg.data['test']['ann_dir'] = None cfg.data['test']['split'] = None # setting based on batch size batch_size = parameters.get_digit_parameters(network_setting_ini, 'batch_size', 'int') if batch_size % gpu_num != 0: raise ValueError('Batch size (%d) cannot be divided by gpu num (%d)' % (batch_size, gpu_num)) cfg.data['samples_per_gpu'] = int(batch_size / gpu_num) cfg.data['workers_per_gpu'] = int( batch_size / gpu_num) + 2 # set worker a litter higher to utilize CPU return True
def training_img_augment(para_file): print("start data augmentation") if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s'%(para_file, os.getcwd())) # augscript = os.path.join(code_dir,'datasets','image_augment.py') img_ext = parameters.get_string_parameters_None_if_absence(para_file,'split_image_format') print("image format: %s"% img_ext) proc_num = parameters.get_digit_parameters(para_file, 'process_num', 'int') SECONDS=time.time() from datasets.image_augment import image_augment_main #augment training images print("image augmentation on image patches") img_list_aug_txt = 'list/images_including_aug.txt' # command_string = augscript + ' -p ' + para_file + ' -d ' + 'split_images' + ' -e ' + img_ext + ' -n ' + str(proc_num) + \ # ' -o ' + 'split_images' + ' -l ' + img_list_aug_txt + ' ' + 'list/trainval.txt' # res = os.system(command_string) # if res!=0: # sys.exit(1) image_augment_main(para_file,'list/trainval.txt',img_list_aug_txt,'split_images','split_images',img_ext,False,proc_num) #augment training lables print("image augmentation on label patches") # command_string = augscript + ' -p ' + para_file + ' -d ' + 'split_labels' + ' -e ' + img_ext + ' -n ' + str(proc_num) + \ # ' -o ' + 'split_labels' + ' -l ' + img_list_aug_txt + ' ' + 'list/trainval.txt' + ' --is_ground_truth ' # # res = os.system(command_string) # if res!=0: # sys.exit(1) # save the result to the same file (redundant, they have the same filename) image_augment_main(para_file, 'list/trainval.txt', img_list_aug_txt, 'split_labels', 'split_labels', img_ext, True,proc_num) if os.path.isfile(img_list_aug_txt): os.system(' cp %s list/trainval.txt'%img_list_aug_txt) os.system(' cp %s list/val.txt'%img_list_aug_txt) else: print('list/images_including_aug.txt does not exist because no data augmentation strings') # output the number of image patches (ls may failed if there are a lot of files, so remove these two lines) # os.system('echo "count of class 0 ":$(ls split_images/*class_0*${img_ext} |wc -l) >> time_cost.txt') # os.system('echo "count of class 1 ":$(ls split_images/*class_1*${img_ext} |wc -l) >> time_cost.txt') duration= time.time() - SECONDS os.system('echo "$(date): time cost of data augmentation: %.2f seconds">>time_cost.txt'%duration)
def main(unused_argv): # dataset_splits = glob.glob(os.path.join(FLAGS.list_file, '*.txt')) # # for dataset_split in dataset_splits: # _convert_dataset(dataset_split) #how about the mean value? #split images data_root = FLAGS.image_folder list_txt = FLAGS.list_file ############## dataset processing parameters.set_saved_parafile_path(FLAGS.para_file) patch_w = parameters.get_digit_parameters("", "train_patch_width", None, 'int') patch_h = parameters.get_digit_parameters("", "train_patch_height", None, 'int') overlay_x = parameters.get_digit_parameters("", "train_pixel_overlay_x", None, 'int') overlay_y = parameters.get_digit_parameters("", "train_pixel_overlay_y", None, 'int') patches = make_dataset(data_root, list_txt, patch_w, patch_h, overlay_x, overlay_y, train=FLAGS.is_training) os.system("mkdir -p " + FLAGS.output_dir) #convert images patches_1d = [item for alist in patches for item in alist] # convert 2D list to 1D _convert_dataset(patches_1d, train=FLAGS.is_training) pass
def main(options, args): input_file = args[0] img_patch_dir = options.image_dir label_patch_dir = options.label_dir basic.outputlogMessage( 'check split image patches and label patches in %s, especially after data augmentation' % input_file) b_diff = False num_classes_noBG = parameters.get_digit_parameters(options.para_file, 'NUM_CLASSES_noBG', None, 'int') with open(input_file, 'r') as f_obj: dir = os.path.dirname(input_file) files_list = f_obj.readlines() for file_name in files_list: file_name = file_name.strip() img_path = os.path.join(img_patch_dir, file_name + '.png') label_path = os.path.join(label_patch_dir, file_name + '.png') img_data = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) h_w = img_data.shape[:2] label_data = cv2.imread(label_path, cv2.IMREAD_UNCHANGED) if h_w != label_data.shape: b_diff = True basic.outputlogMessage( '%s have different size in image (%s) and label (%s) patch' % (file_name, str(h_w), str(label_data.shape))) max_label = np.max(label_data) unique_value = np.unique(label_data) if max_label > num_classes_noBG: b_diff = True basic.outputlogMessage( '%s: maximum pixel value (%d) in label images > num_class (%d)' % (file_name, max_label, num_classes_noBG)) if len(unique_value) > num_classes_noBG + 1: b_diff = True basic.outputlogMessage( '%s: the count of unique pixel value (%s) in label images is greater than num_class (%d)' % (file_name, str(unique_value), num_classes_noBG)) if b_diff is False: basic.outputlogMessage('all the patches are good')
def add_boxes_attributes(input, output, para_file=None,data_para_file=None): if io_function.is_file_exist(input) is False: return False # copy output if io_function.copy_shape_file(input, output) is False: raise IOError('copy shape file %s failed'%input) # calculate area, perimeter of polygons if cal_add_area_length_of_polygon(output) is False: return False # calculate the width, height, and width_height ratio boxes = vector_gpd.read_polygons_gpd(input) bounding_boxes = [ vector_gpd.get_polygon_bounding_box(item) for item in boxes ] # bounding box (minx, miny, maxx, maxy) width_list = [ bound[2] - bound[0] for bound in bounding_boxes] height_list = [ bound[3] - bound[1] for bound in bounding_boxes] ratio_w_h_list = [] for w, h in zip(width_list, height_list): if w > h: ratio = h / w else: ratio = w / h ratio_w_h_list.append(ratio) add_attributes = {'WIDTH':width_list,'HEIGHT':height_list, 'ratio_w_h':ratio_w_h_list} vector_gpd.add_attributes_to_shp(output,add_attributes) # add topography of each box (consider the box as a polygon) if data_para_file is not None and para_file is not None: io_function.is_file_exist(data_para_file) io_function.is_file_exist(para_file) dem_files, slope_files, aspect_files, dem_diff_files = get_topographic_files(data_para_file) process_num = parameters.get_digit_parameters(para_file,'process_num','int') if calculate_polygon_topography(output,para_file,dem_files,slope_files,aspect_files=aspect_files,dem_diffs=dem_diff_files, process_num=process_num) is False: basic.outputlogMessage('Warning: calculate information of topography failed') # return False # don't return return output
def main(options, args): input_shp = args[0] output_raster = args[1] if io_function.is_file_exist(input_shp) is False: return False all_class_raster = io_function.get_name_by_adding_tail( output_raster, 'AllClass') num_class = parameters.get_digit_parameters(options.para_file, 'NUM_CLASSES_noBG', None, 'int') if convert_training_examples_from_shp_to_raster(input_shp, all_class_raster) is False: basic.outputlogMessage( "Producing the label images from training polygons is Falild") return False else: basic.outputlogMessage( "Done: Producing the label images from training polygons, output: %s" % all_class_raster) if num_class == 1: #only keep target (gully or others) label one_class_raster = io_function.get_name_by_adding_tail( output_raster, 'oneClass') if only_keep_one_class( all_class_raster, one_class_raster, class_index=1) is False: return False else: one_class_raster = all_class_raster # crop the label image to have the same 2D dimension with the training images baseimage = parameters.get_input_image_path() if RSImageProcess.subset_image_baseimage(output_raster, one_class_raster, baseimage) is False: basic.outputlogMessage("Error: subset_image_baseimage Failed") return False return True
def predict_one_image_yolo(para_file, image_path, img_save_dir, inf_list_file, gpuid, trained_model): config_file = parameters.get_string_parameters( para_file, 'network_setting_ini') # 'yolov4_obj.cfg' yolo_data = os.path.join('data', 'obj.data') # b_python_api = False inf_batch_size = parameters.get_digit_parameters(para_file, 'inf_batch_size', 'int') b_python_api = parameters.get_bool_parameters(para_file, 'b_inf_use_python_api') done_indicator = '%s_done' % inf_list_file if os.path.isfile(done_indicator): basic.outputlogMessage('warning, %s exist, skip prediction' % done_indicator) return # use a specific GPU for prediction, only inference one image time0 = time.time() if gpuid is not None: os.environ['CUDA_VISIBLE_DEVICES'] = str(gpuid) predict_remoteSensing_image(para_file, image_path, img_save_dir, trained_model, config_file, yolo_data, batch_size=inf_batch_size, b_python_api=b_python_api) duration = time.time() - time0 os.system( 'echo "$(date): time cost of inference for image in %s: %.2f seconds">>"time_cost.txt"' % (inf_list_file, duration)) # write a file to indicate that the prediction has done. os.system('echo %s > %s_done' % (inf_list_file, inf_list_file)) return
def split_sub_images(para_file): print("split sub-images and sub-labels") if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s'%(para_file, os.getcwd())) SECONDS = time.time() if os.path.isdir('split_images'): io_function.delete_file_or_dir('split_images') if os.path.isdir('split_labels'): io_function.delete_file_or_dir('split_labels') io_function.mkdir('split_images') ### split the training image to many small patch (480*480) patch_w=parameters.get_string_parameters(para_file,'train_patch_width') patch_h=parameters.get_string_parameters(para_file,'train_patch_height') overlay_x=parameters.get_string_parameters(para_file,'train_pixel_overlay_x') overlay_y=parameters.get_string_parameters(para_file,'train_pixel_overlay_y') split_image_format=parameters.get_string_parameters(para_file,'split_image_format') trainImg_dir=parameters.get_string_parameters(para_file,'input_train_dir') labelImg_dir=parameters.get_string_parameters(para_file,'input_label_dir') proc_num = parameters.get_digit_parameters(para_file,'process_num','int') if os.path.isdir(trainImg_dir) is False: raise IOError('%s not in the current folder, please get subImages first'%trainImg_dir) if os.path.isdir(labelImg_dir) is False: print('warning, %s not in the current folder'%labelImg_dir) else: io_function.mkdir('split_labels') sub_img_label_txt = 'sub_images_labels_list.txt' if os.path.isfile(sub_img_label_txt) is False: raise IOError('%s not in the current folder, please get subImages first' % sub_img_label_txt) with open(sub_img_label_txt) as txt_obj: line_list = [name.strip() for name in txt_obj.readlines()] # for line in line_list: # sub_image, sub_label = line.split(':') # # # split sub image # split_to_patches(sub_image, 'split_images', patch_w, patch_h, overlay, overlay, split_image_format) # # # split sub label (change the file name to be the same as sub_image name) # pre_name = os.path.splitext(os.path.basename(sub_image))[0] # split_to_patches(sub_label, 'split_labels', patch_w, patch_h, overlay, overlay, split_image_format, file_pre_name=pre_name) parameters_list = [(line, patch_w, patch_h, overlay_x, overlay_y, split_image_format) for line in line_list] theadPool = Pool(proc_num) # multi processes results = theadPool.starmap(split_a_pair_sub_image_label, parameters_list) # need python3 # output trainval.txt and val.txt file files_list = io_function.get_file_list_by_ext(split_image_format, 'split_images',bsub_folder=False) io_function.mkdir('list') trainval = os.path.join('list','trainval.txt') val = os.path.join('list','val.txt') with open(trainval,'w') as w_obj: for file_name in files_list: w_obj.writelines(os.path.splitext(os.path.basename(file_name))[0] + '\n') io_function.copy_file_to_dst(trainval,val,overwrite=True) split_train_val.get_image_with_height_list(trainval, split_image_format, info_type='(no data augmentation)') duration= time.time() - SECONDS os.system('echo "$(date): time cost of splitting sub images and labels: %.2f seconds">>time_cost.txt'%duration)
def get_sub_images_multi_regions(para_file): print( "extract sub-images and sub-labels for a given shape file (training polygons)" ) if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd())) get_subImage_script = os.path.join(code_dir, 'datasets', 'get_subImages.py') SECONDS = time.time() # get name of training areas multi_training_regions = parameters.get_string_list_parameters_None_if_absence( para_file, 'training_regions') if multi_training_regions is None or len(multi_training_regions) < 1: raise ValueError('No training area is set in %s' % para_file) # multi_training_files = parameters.get_string_parameters_None_if_absence(para_file, 'multi_training_files') dstnodata = parameters.get_string_parameters(para_file, 'dst_nodata') buffersize = parameters.get_string_parameters(para_file, 'buffer_size') rectangle_ext = parameters.get_string_parameters(para_file, 'b_use_rectangle') process_num = parameters.get_digit_parameters(para_file, 'process_num', 'int') b_no_label_image = parameters.get_bool_parameters_None_if_absence( para_file, 'b_no_label_image') if os.path.isfile('sub_images_labels_list.txt'): io_function.delete_file_or_dir('sub_images_labels_list.txt') subImage_dir = parameters.get_string_parameters_None_if_absence( para_file, 'input_train_dir') subLabel_dir = parameters.get_string_parameters_None_if_absence( para_file, 'input_label_dir') # loop each training regions for idx, area_ini in enumerate(multi_training_regions): input_image_dir = parameters.get_directory_None_if_absence( area_ini, 'input_image_dir') # it is ok consider a file name as pattern and pass it the following functions to get file list input_image_or_pattern = parameters.get_string_parameters( area_ini, 'input_image_or_pattern') b_sub_images_json = parameters.get_bool_parameters( area_ini, 'b_sub_images_json') if b_sub_images_json is True: # copy sub-images, then covert json files to label images. object_names = parameters.get_string_list_parameters( para_file, 'object_names') get_subImages_json.get_subimages_label_josn( input_image_dir, input_image_or_pattern, subImage_dir, subLabel_dir, object_names, b_no_label_image=b_no_label_image, process_num=process_num) pass else: all_train_shp = parameters.get_file_path_parameters_None_if_absence( area_ini, 'training_polygons') train_shp = parameters.get_string_parameters( area_ini, 'training_polygons_sub') # get subImage and subLabel for one training polygons print( 'extract training data from image folder (%s) and polgyons (%s)' % (input_image_dir, train_shp)) if b_no_label_image is True: get_subImage_one_shp(get_subImage_script, all_train_shp, buffersize, dstnodata, rectangle_ext, train_shp, input_image_dir, file_pattern=input_image_or_pattern, process_num=process_num) else: get_subImage_subLabel_one_shp( get_subImage_script, all_train_shp, buffersize, dstnodata, rectangle_ext, train_shp, input_image_dir, file_pattern=input_image_or_pattern, process_num=process_num) # check black sub-images or most part of the sub-images is black (nodata) new_sub_image_label_list = [] delete_sub_image_label_list = [] subImage_dir_delete = subImage_dir + '_delete' subLabel_dir_delete = subLabel_dir + '_delete' io_function.mkdir(subImage_dir_delete) if b_no_label_image is None or b_no_label_image is False: io_function.mkdir(subLabel_dir_delete) get_valid_percent_entropy.plot_valid_entropy(subImage_dir) with open('sub_images_labels_list.txt', 'r') as f_obj: lines = f_obj.readlines() for line in lines: image_path, label_path = line.strip().split(':') # valid_per = raster_io.get_valid_pixel_percentage(image_path) valid_per, entropy = raster_io.get_valid_percent_shannon_entropy( image_path) # base=10 if valid_per > 60 and entropy >= 0.5: new_sub_image_label_list.append(line) else: delete_sub_image_label_list.append(line) io_function.movefiletodir(image_path, subImage_dir_delete) if os.path.isfile(label_path): io_function.movefiletodir(label_path, subLabel_dir_delete) if len(delete_sub_image_label_list) > 0: with open('sub_images_labels_list.txt', 'w') as f_obj: for line in new_sub_image_label_list: f_obj.writelines(line) # check weather they have the same subImage and subLabel if b_no_label_image is None or b_no_label_image is False: sub_image_list = io_function.get_file_list_by_pattern( subImage_dir, '*.tif') sub_label_list = io_function.get_file_list_by_pattern( subLabel_dir, '*.tif') if len(sub_image_list) != len(sub_label_list): raise ValueError( 'the count of subImage (%d) and subLabel (%d) is different' % (len(sub_image_list), len(sub_label_list))) # save brief information of sub-images height_list = [] width_list = [] band_count = 0 dtype = 'unknown' for line in new_sub_image_label_list: image_path, label_path = line.strip().split(':') height, width, band_count, dtype = raster_io.get_height_width_bandnum_dtype( image_path) height_list.append(height) width_list.append(width) # save info to file, if it exists, it will be overwritten img_count = len(new_sub_image_label_list) with open('sub_images_patches_info.txt', 'w') as f_obj: f_obj.writelines('information of sub-images: \n') f_obj.writelines('number of sub-images : %d \n' % img_count) f_obj.writelines('band count : %d \n' % band_count) f_obj.writelines('data type : %s \n' % dtype) f_obj.writelines('maximum width and height: %d, %d \n' % (max(width_list), max(height_list))) f_obj.writelines('minimum width and height: %d, %d \n' % (min(width_list), min(height_list))) f_obj.writelines( 'mean width and height: %.2f, %.2f \n\n' % (sum(width_list) / img_count, sum(height_list) / img_count)) duration = time.time() - SECONDS os.system( 'echo "$(date): time cost of getting sub images and labels: %.2f seconds">>time_cost.txt' % duration)
def inf_remoteSensing_image(model, image_path=None): ''' input a remote sensing image, then split to many small patches, inference each patch and merge than at last :param model: trained model :param image_path: :return: False if unsuccessful. ''' # split images inf_image_dir = parameters.get_string_parameters(para_file, 'inf_images_dir') patch_w = parameters.get_digit_parameters(para_file, "inf_patch_width", None, 'int') patch_h = parameters.get_digit_parameters(para_file, "inf_patch_height", None, 'int') overlay_x = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_x", None, 'int') overlay_y = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_y", None, 'int') inf_batch_size = parameters.get_digit_parameters(para_file, "inf_batch_size", None, 'int') global inf_list_file if image_path is not None: with open('inf_image_list.txt', 'w') as f_obj: f_obj.writelines(image_path) inf_list_file = 'inf_image_list.txt' data_patches_2d = build_RS_data.make_dataset(inf_image_dir, inf_list_file, patch_w, patch_h, overlay_x, overlay_y, train=False) if len(data_patches_2d) < 1: return False total_patch_count = 0 for img_idx, aImage_patches in enumerate(data_patches_2d): patch_num = len(aImage_patches) total_patch_count += patch_num print('number of patches on Image %d: %d' % (img_idx, patch_num)) print('total number of patches: %d' % total_patch_count) ## inference multiple image patches at the same time # for img_idx, aImage_patches in enumerate(data_patches_2d): # # print('start inference on Image %d' % img_idx) # # idx = 0 #index of all patches on this image # patch_batches = build_RS_data.split_patches_into_batches(aImage_patches,inf_batch_size) # # for a_batch_of_patches in patch_batches: # # # Since it required a constant of batch size for the frozen graph, we copy (duplicate) the first patch # org_patch_num = len(a_batch_of_patches) # while len(a_batch_of_patches) < inf_batch_size: # a_batch_of_patches.append(a_batch_of_patches[0]) # # # read image data and stack at 0 dimension # multi_image_data = [] # for img_patch in a_batch_of_patches: # img_data = build_RS_data.read_patch(img_patch) # (nband, height,width) # img_data = np.transpose(img_data, (1, 2, 0)) # keras and tf require (height,width,nband) # multi_image_data.append(img_data) # # multi_images = np.stack(multi_image_data, axis=0) # # # modify the BATCH_Size # model.config.BATCH_SIZE = len(multi_image_data) # # # inference them # results = model.detect(multi_image_data, verbose=0) # # r = results[0] # # visualize.display_instances(image_data, r['rois'], r['masks'], r['class_ids'], # # ['BG', 'thawslump'], r['scores'], ax=get_ax()) # # #save # for num,(mrcc_r,img_patch) in enumerate(zip(results,a_batch_of_patches)): # # # ignore the duplicated ones # if num >= org_patch_num: # break # # # # convert mask to a map of classification # masks = mrcc_r['masks'] # shape: (height, width, num_classes?) # height, width, nclass = masks.shape # # seg_map = np.zeros((height, width),dtype=np.uint8) # for n_id in range(0,nclass): # seg_map[ masks[:,:,n_id] == True ] = n_id + 1 # # print('Save segmentation result of Image:%d patch:%4d, shape:(%d,%d)' % # (img_idx, idx, seg_map.shape[0], seg_map.shape[1])) # # # short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29 # file_name = "I%d_%d" % (img_idx, idx) # # save_path = os.path.join(inf_output_dir, file_name + '.tif') # if build_RS_data.save_patch_oneband_8bit(img_patch,seg_map.astype(np.uint8),save_path) is False: # return False # # idx += 1 # inference image patches one by one for img_idx, aImage_patches in enumerate(data_patches_2d): print('start inference on Image %d' % img_idx) for idx, img_patch in enumerate(aImage_patches): # # test debug # if not idx in [3359, 3360,3361,3476,3477,3478,3593,3594,3595]: # continue # if not idx in [4700, 4817, 4818, 4819, 4934, 4935, 4936, 5051, 5052, 5053]: # continue # if not idx in [2602]: # a false positive # continue img_data = build_RS_data.read_patch( img_patch) # (nband, height,width) # test # save_path = "I%d_%d_org.tif" % (img_idx, idx) # build_RS_data.save_patch(img_patch, img_data, save_path) img_data = np.transpose( img_data, (1, 2, 0)) # keras and tf require (height,width,nband) # inference them results = model.detect([img_data], verbose=0) mrcc_r = results[0] # mrcc_r['scores'] # mrcc_r['rois'] masks = mrcc_r['masks'] # shape: (height, width, num_instance) height, width, ncount = masks.shape class_ids = mrcc_r['class_ids'] seg_map = np.zeros((height, width), dtype=np.uint8) for inst in range(0, ncount): # instance one by one seg_map[masks[:, :, inst] == True] = class_ids[inst] print( 'Save segmentation result of Image:%d patch:%4d, shape:(%d,%d)' % (img_idx, idx, seg_map.shape[0], seg_map.shape[1])) # short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29 file_name = "I%d_%d" % (img_idx, idx) save_path = os.path.join(inf_output_dir, file_name + '.tif') if build_RS_data.save_patch_oneband_8bit( img_patch, seg_map.astype(np.uint8), save_path) is False: return False
def muti_inf_remoteSensing_image(model, image_path=None): ''' use multiple scale (different patch size) for inference, then merge them using non_max_suppression :param model: trained model :param image_path: :return: True if successful, False otherwise ''' # get parameters inf_image_dir = parameters.get_string_parameters(para_file, 'inf_images_dir') muti_patch_w = parameters.get_string_parameters(para_file, "muti_inf_patch_width") muti_patch_h = parameters.get_string_parameters(para_file, "muti_inf_patch_height") muti_overlay_x = parameters.get_string_parameters( para_file, "muti_inf_pixel_overlay_x") muti_overlay_y = parameters.get_string_parameters( para_file, "muti_inf_pixel_overlay_y") final_keep_classes = parameters.get_string_parameters( para_file, "final_keep_classes") nms_iou_threshold = parameters.get_digit_parameters( para_file, "nms_iou_threshold", None, 'float') patch_w_list = [int(item) for item in muti_patch_w.split(',')] patch_h_list = [int(item) for item in muti_patch_h.split(',')] overlay_x_list = [int(item) for item in muti_overlay_x.split(',')] overlay_y_list = [int(item) for item in muti_overlay_y.split(',')] if final_keep_classes == '': final_keep_classes = None else: final_keep_classes = [ int(item) for item in final_keep_classes.split(',') ] # inference and save to json files for patch_w, patch_h, overlay_x, overlay_y in zip(patch_w_list, patch_h_list, overlay_x_list, overlay_y_list): inf_rs_image_json(model, patch_w, patch_h, overlay_x, overlay_y, inf_image_dir) # load all boxes of images with open(inf_list_file) as file_obj: files_list = file_obj.readlines() for img_idx, image_name in enumerate(files_list): file_pattern = os.path.join( inf_output_dir, 'I%d_patches_*_*_*' % img_idx) # e.g., I0_patches_320_320_80_80 proc = subprocess.Popen('ls -d ' + file_pattern, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) profiles, err = proc.communicate() json_folder_list = profiles.split() if len(json_folder_list) < 1: raise IOError('No folder containing json files in %s' % inf_output_dir) # bytes to str if isinstance(json_folder_list[0], bytes): json_folder_list = [item.decode() for item in json_folder_list] print('loading json files of image :%d, in %s' % (img_idx, ','.join(json_folder_list))) # load all the boxes and scores mrcnn_r_list = [ ] # the results dict from rcnn, contains all the information mask_files = [] # mask file for instance, each box has a mask file boxes = [] # boxes of instance class_ids = [] # class id of each box patch_indices = [] # index of patch, on which contains the box scores = [] # scores of boxes json_files_list = [] # json files of patches for json_folder in json_folder_list: file_list = io_function.get_file_list_by_ext('.txt', json_folder, bsub_folder=False) json_files_list.extend(file_list) # load and convert coordinates, don't load mask images in this stage patch_idx = 0 for json_file in json_files_list: mrcnn_r = build_RS_data.load_instances_patch( json_file, bNMS=True, bReadMaks=False, final_classes=final_keep_classes) mrcnn_r_list.append( mrcnn_r) # this corresponds to json_files_list if mrcnn_r is not None: # this will ignore the patches without instances, it is fine hlc 2018-11-25 mask_files.extend(mrcnn_r['masks']) boxes.extend(mrcnn_r['rois']) scores.extend(mrcnn_r['scores']) class_ids.extend(mrcnn_r['class_ids']) patch_indices.extend([patch_idx] * len(mrcnn_r['rois'])) patch_idx += 1 # Apply non-max suppression keep_idxs = utils.non_max_suppression(np.array(boxes), np.array(scores), nms_iou_threshold) # boxes_keep = [r for i, r in enumerate(boxes) if i in keep_ixs] # convert kept patches to label images for idx, keep_idx in enumerate(keep_idxs): patch_idx = patch_indices[ keep_idx] # the index in original patches # load mask (in the previous step, we did not load masks) mask_file = mask_files[keep_idx] patch_dir = os.path.dirname(json_files_list[patch_idx]) org_img_name = mrcnn_r_list[patch_idx]['org_img'] b_dict = mrcnn_r_list[patch_idx]['patch_boundary'] patch_boundary = (b_dict['xoff'], b_dict['yoff'], b_dict['xsize'], b_dict['ysize']) img_patch = build_RS_data.patchclass( os.path.join(inf_image_dir, org_img_name), patch_boundary) # the mask only contains one instances # masks can overlap each others, but instances should not overlap each other # non-instance pixels are zeros,which will be set as non-data when performing gdal_merge.pyg mask = cv2.imread(os.path.join(patch_dir, mask_file), cv2.IMREAD_UNCHANGED) mask[mask == 255] = class_ids[ keep_idx] # when save mask, mask*255 for display print('Save mask of instances:%d on Image:%d , shape:(%d,%d)' % (idx, img_idx, mask.shape[0], mask.shape[1])) # short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29 file_name = "I%d_%d" % (img_idx, idx) save_path = os.path.join(inf_output_dir, file_name + '.tif') if build_RS_data.save_patch_oneband_8bit( img_patch, mask.astype(np.uint8), save_path) is False: return False return True
def train_evaluation_deeplab_separate(WORK_DIR, deeplab_dir, expr_name, para_file, network_setting_ini, gpu_num): ''' in "train_evaluation_deeplab", run training, stop, then evaluation, then traininng, make learning rate strange, and the results worse. so in this function, we start two process, one for training, another for evaluation (run on CPU) ''' # prepare training folder EXP_FOLDER = expr_name INIT_FOLDER = os.path.join(WORK_DIR, EXP_FOLDER, 'init_models') TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train') EVAL_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'eval') VIS_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'vis') EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export') io_function.mkdir(INIT_FOLDER) io_function.mkdir(TRAIN_LOGDIR) io_function.mkdir(EVAL_LOGDIR) io_function.mkdir(VIS_LOGDIR) io_function.mkdir(EXPORT_DIR) # prepare the tensorflow check point (pretrained model) for training pre_trained_dir = parameters.get_directory_None_if_absence( network_setting_ini, 'pre_trained_model_folder') pre_trained_tar = parameters.get_string_parameters(network_setting_ini, 'TF_INIT_CKPT') pre_trained_path = os.path.join(pre_trained_dir, pre_trained_tar) if os.path.isfile(pre_trained_path) is False: print('pre-trained model: %s not exist, try to download' % pre_trained_path) # try to download the file pre_trained_url = parameters.get_string_parameters_None_if_absence( network_setting_ini, 'pre_trained_model_url') res = os.system('wget %s ' % pre_trained_url) if res != 0: sys.exit(1) io_function.movefiletodir(pre_trained_tar, pre_trained_dir) # unpack pre-trained model to INIT_FOLDER os.chdir(INIT_FOLDER) res = os.system('tar -xf %s' % pre_trained_path) if res != 0: raise IOError('failed to unpack %s' % pre_trained_path) os.chdir(WORK_DIR) dataset_dir = os.path.join(WORK_DIR, 'tfrecord') batch_size = parameters.get_digit_parameters(network_setting_ini, 'batch_size', 'int') # maximum iteration number iteration_num = parameters.get_digit_parameters(network_setting_ini, 'iteration_num', 'int') base_learning_rate = parameters.get_digit_parameters( network_setting_ini, 'base_learning_rate', 'float') train_output_stride = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_output_stride', 'int') train_atrous_rates1 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_atrous_rates1', 'int') train_atrous_rates2 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_atrous_rates2', 'int') train_atrous_rates3 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_atrous_rates3', 'int') inf_output_stride = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_output_stride', 'int') inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates1', 'int') inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates2', 'int') inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates3', 'int') # depth_multiplier default is 1.0. depth_multiplier = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'depth_multiplier', 'float') decoder_output_stride = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'decoder_output_stride', 'int') aspp_convs_filters = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'aspp_convs_filters', 'int') train_script = os.path.join(deeplab_dir, 'train.py') train_split = os.path.splitext( parameters.get_string_parameters(para_file, 'training_sample_list_txt'))[0] model_variant = parameters.get_string_parameters(network_setting_ini, 'model_variant') checkpoint = parameters.get_string_parameters(network_setting_ini, 'tf_initial_checkpoint') init_checkpoint_files = io_function.get_file_list_by_pattern( INIT_FOLDER, checkpoint + '*') if len(init_checkpoint_files) < 1: raise IOError('No initial checkpoint in %s with pattern: %s' % (INIT_FOLDER, checkpoint)) init_checkpoint = os.path.join(INIT_FOLDER, checkpoint) b_early_stopping = parameters.get_bool_parameters(para_file, 'b_early_stopping') b_initialize_last_layer = parameters.get_bool_parameters( para_file, 'b_initialize_last_layer') dataset = parameters.get_string_parameters(para_file, 'dataset_name') num_classes_noBG = parameters.get_digit_parameters_None_if_absence( para_file, 'NUM_CLASSES_noBG', 'int') assert num_classes_noBG != None if b_initialize_last_layer is True: if pre_trained_tar in pre_trained_tar_21_classes: print( 'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21' % pre_trained_tar) num_classes_noBG = 20 if pre_trained_tar in pre_trained_tar_19_classes: print( 'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19' % pre_trained_tar) num_classes_noBG = 18 num_of_classes = num_classes_noBG + 1 image_crop_size = parameters.get_string_list_parameters( para_file, 'image_crop_size') if len(image_crop_size) != 2 and image_crop_size[0].isdigit( ) and image_crop_size[1].isdigit(): raise ValueError('image_crop_size should be height,width') crop_size_str = ','.join(image_crop_size) # validation interval (epoch), do # validation_interval = parameters.get_digit_parameters_None_if_absence(para_file,'validation_interval','int') train_count, val_count = get_train_val_sample_count(WORK_DIR, para_file) iter_per_epoch = math.ceil(train_count / batch_size) total_epoches = math.ceil(iteration_num / iter_per_epoch) already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR) if already_trained_iteration >= iteration_num: basic.outputlogMessage('Training already run %d iterations, skip' % already_trained_iteration) return True save_interval_secs = 1200 # default is 1200 second for saving model save_summaries_secs = 600 # default is 600 second for saving summaries eval_interval_secs = save_interval_secs # default is 300 second for running evaluation, if no new saved model, no need to run evaluation? train_process = Process( target=train_deeplab, args=(train_script, dataset, train_split, num_of_classes, base_learning_rate, model_variant, init_checkpoint, TRAIN_LOGDIR, dataset_dir, gpu_num, train_atrous_rates1, train_atrous_rates2, train_atrous_rates3, train_output_stride, crop_size_str, batch_size, iteration_num, depth_multiplier, decoder_output_stride, aspp_convs_filters, b_initialize_last_layer)) train_process.start() time.sleep(60) # wait if train_process.exitcode is not None and train_process.exitcode != 0: sys.exit(1) # eval_process.start() # time.sleep(10) # wait # if eval_process.exitcode is not None and eval_process.exitcode != 0: # sys.exit(1) while True: # only run evaluation when there is new trained model already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR) miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes) basic.outputlogMessage( 'Already trained iteration: %d, latest evaluation at %d step' % (already_trained_iteration, miou_dict['step'][-1])) if already_trained_iteration > miou_dict['step'][-1]: # run evaluation and wait until it finished gpuid = "" # set gpuid to empty string, making evaluation run on CPU evl_script = os.path.join(deeplab_dir, 'eval.py') evl_split = os.path.splitext( parameters.get_string_parameters( para_file, 'validation_sample_list_txt'))[0] # max_eva_number = -1 # run as many evaluation as possible, --eval_interval_secs (default is 300 seconds) max_eva_number = 1 # only run once inside the while loop, use while loop to control multiple evaluation eval_process = Process( target=evaluation_deeplab, args=(evl_script, dataset, evl_split, num_of_classes, model_variant, inf_atrous_rates1, inf_atrous_rates2, inf_atrous_rates3, inf_output_stride, TRAIN_LOGDIR, EVAL_LOGDIR, dataset_dir, crop_size_str, max_eva_number, depth_multiplier, decoder_output_stride, aspp_convs_filters, gpuid, eval_interval_secs)) eval_process.start( ) # put Process inside while loop to avoid error: AssertionError: cannot start a process twice while eval_process.is_alive(): time.sleep(5) # check if need early stopping if b_early_stopping: print(datetime.now(), 'check early stopping') miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes) if 'overall' in miou_dict.keys() and len( miou_dict['overall']) >= 5: # if the last five miou did not improve, then stop training if np.all(np.diff(miou_dict['overall'][-5:]) < 0.005 ): # 0.0001 (%0.01) # 0.5 % basic.outputlogMessage( 'early stopping: stop training because overall miou did not improved in the last five evaluation' ) output_early_stopping_message(TRAIN_LOGDIR) # train_process.kill() # this one seems not working # subprocess pid different from ps output # https://stackoverflow.com/questions/4444141/subprocess-pid-different-from-ps-output # os.system('kill ' + str(train_process.pid)) # still not working. train_process.pid is not the one output by ps -aux # train_process.terminate() # Note that descendant processes of the process will not be terminated # train_process.join() # Wait until child process terminates with open('train_py_pid.txt', 'r') as f_obj: lines = f_obj.readlines() train_pid = int(lines[0].strip()) os.system('kill ' + str(train_pid)) basic.outputlogMessage( 'kill training processing with id: %d' % train_pid) break # this breaks the while loop, making that it may not evaluate on some new saved model. # if the evaluation step is less than saved model iteration, run another iteration again immediately already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR) miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes) if already_trained_iteration > miou_dict['step'][-1]: continue # if finished training if train_process.is_alive() is False: break # # if eval_process exit, then quit training as well # if eval_process.is_alive() is False and train_process.is_alive(): # train_process.kill() # break time.sleep(eval_interval_secs) # wait for next evaluation # save loss value to disk get_loss_learning_rate_list(TRAIN_LOGDIR) # get miou again miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes) # eval_process did not exit as expected, kill it again. # os.system('kill ' + str(eval_process.pid)) # get iou and backup iou_path = os.path.join(EVAL_LOGDIR, 'miou.txt') loss_path = os.path.join(TRAIN_LOGDIR, 'loss_learning_rate.txt') patch_info = os.path.join(WORK_DIR, 'sub_images_patches_info.txt') # backup miou and training_loss & learning rate test_id = os.path.basename(WORK_DIR) + '_' + expr_name backup_dir = os.path.join(WORK_DIR, 'result_backup') if os.path.isdir(backup_dir) is False: io_function.mkdir(backup_dir) new_iou_name = os.path.join(backup_dir, test_id + '_' + os.path.basename(iou_path)) io_function.copy_file_to_dst(iou_path, new_iou_name, overwrite=True) loss_new_name = os.path.join(backup_dir, test_id + '_' + os.path.basename(loss_path)) io_function.copy_file_to_dst(loss_path, loss_new_name, overwrite=True) new_patch_info = os.path.join(backup_dir, test_id + '_' + os.path.basename(patch_info)) io_function.copy_file_to_dst(patch_info, new_patch_info, overwrite=True) # plot mIOU, loss, and learnint rate curves, and backup miou_curve_path = plot_miou_loss_curve.plot_miou_loss_main( iou_path, train_count=train_count, val_count=val_count, batch_size=batch_size) loss_curve_path = plot_miou_loss_curve.plot_miou_loss_main( loss_path, train_count=train_count, val_count=val_count, batch_size=batch_size) miou_curve_bakname = os.path.join( backup_dir, test_id + '_' + os.path.basename(miou_curve_path)) io_function.copy_file_to_dst(miou_curve_path, miou_curve_bakname, overwrite=True) loss_curve_bakname = os.path.join( backup_dir, test_id + '_' + os.path.basename(loss_curve_path)) io_function.copy_file_to_dst(loss_curve_path, loss_curve_bakname, overwrite=True)
def inf_remoteSensing_image(model,image_path=None): ''' input a remote sensing image, then split to many small patches, inference each patch and merge than at last :param model: trained model :param image_path: :return: False if unsuccessful. ''' # split images inf_image_dir=parameters.get_string_parameters(FLAGS.inf_para_file,'inf_images_dir') patch_w = parameters.get_digit_parameters(FLAGS.inf_para_file, "inf_patch_width", None, 'int') patch_h = parameters.get_digit_parameters(FLAGS.inf_para_file, "inf_patch_height", None, 'int') overlay_x = parameters.get_digit_parameters(FLAGS.inf_para_file, "inf_pixel_overlay_x", None, 'int') overlay_y = parameters.get_digit_parameters(FLAGS.inf_para_file, "inf_pixel_overlay_y", None, 'int') if image_path is not None: with open('saved_inf_list.txt','w') as f_obj: f_obj.writelines(image_path) FLAGS.inf_list_file = 'saved_inf_list.txt' data_patches_2d = build_RS_data.make_dataset(inf_image_dir,FLAGS.inf_list_file, patch_w,patch_h,overlay_x,overlay_y,train=False) if len(data_patches_2d)< 1: return False total_patch_count = 0 for img_idx, aImage_patches in enumerate(data_patches_2d): patch_num = len(aImage_patches) total_patch_count += patch_num print('number of patches on Image %d: %d' % (img_idx,patch_num)) print('total number of patches: %d'%total_patch_count) for img_idx, aImage_patches in enumerate(data_patches_2d): print('start inference on Image %d' % img_idx) # ## parallel inference patches ## but it turns out not work due to the Pickle.PicklingError # # use multiple thread # num_cores = multiprocessing.cpu_count() # print('number of thread %d' % num_cores) # # theadPool = mp.Pool(num_cores) # multi threads, can not utilize all the CPUs? not sure hlc 2018-4-19 # theadPool = Pool(num_cores) # multi processes # # parameters_list = [(img_idx,idx,img_patch.org_img,img_patch.boundary,model) for (idx,img_patch) in enumerate(aImage_patches)] # results = theadPool.map(inference_one_patch, parameters_list) # print('result_list',results ) # inference patches batch by batch, but it turns out that the frozen graph only accept one patch each time # Oct 30,2018 # split to many batches (groups) idx = 0 #index of all patches on this image patch_batches = build_RS_data.split_patches_into_batches(aImage_patches,FLAGS.inf_batch_size) for a_batch_of_patches in patch_batches: # Since it required a constant of batch size for the frozen graph, we copy (duplicate) the first patch org_patch_num = len(a_batch_of_patches) while len(a_batch_of_patches) < FLAGS.inf_batch_size: a_batch_of_patches.append(a_batch_of_patches[0]) # read image data and stack at 0 dimension multi_image_data = [] for img_patch in a_batch_of_patches: img_data = build_RS_data.read_patch(img_patch) ## ignore image patch are all black or white if np.std(img_data) < 0.0001: print('Image:%d patch:%4d is black or white, ignore' %(img_idx, idx)) idx += 1 continue multi_image_data.append(img_data) # ignore image patch are all black or white if len(multi_image_data) < 1: continue multi_images = np.stack(multi_image_data, axis=0) # inference them a_batch_seg_map = model.run_rsImg_multi_patches(multi_images) #save for num,(seg_map,img_patch) in enumerate(zip(a_batch_seg_map,a_batch_of_patches)): # ignore the duplicated ones if num >= org_patch_num: break print('Save segmentation result of Image:%d patch:%4d, shape:(%d,%d)' % (img_idx, idx, seg_map.shape[0], seg_map.shape[1])) # short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29 file_name = "I%d_%d" % (img_idx, idx) save_path = os.path.join(FLAGS.inf_output_dir, file_name + '.tif') # if os.path.isfile(save_path): # print('already exist, skip') # idx += 1 # continue if build_RS_data.save_patch_oneband_8bit(img_patch,seg_map.astype(np.uint8),save_path) is False: return False idx += 1
parser.add_argument('--cuda', action='store_true', help='enables cuda') parser.add_argument('--resume', default='', type=str, metavar='PATH', help='path to latest checkpoint (default: none)') parser.add_argument('--useBN', action='store_true', help='enalbes batch normalization') args = parser.parse_args() print(args) parameters.set_saved_parafile_path(args.para) patch_w = parameters.get_digit_parameters("", "inf_patch_width", None, 'int') patch_h = parameters.get_digit_parameters("", "inf_patch_height", None, 'int') overlay = parameters.get_digit_parameters("", "inf_pixel_overlay", None, 'int') dataset = RemoteSensingImg(args.dataroot, args.list, patch_w, patch_h, overlay, train=False) train_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batchSize, num_workers=args.workers, shuffle=False) model = Net(args.useBN)
def main(options, args): if options.out_dir is None: out_dir = "extract_dir" else: out_dir = options.out_dir if os.path.isdir(out_dir) is False: os.makedirs(out_dir) img_dir = options.img_dir extension = options.extension is_groud_true = options.ground_truth # print(options.para_file) augmentation = parameters.get_string_parameters(options.para_file, 'data_augmentation') augmentation = [item.lower().strip() for item in augmentation.split(',')] augmentation = list(filter( None, augmentation)) # remove empty str (in case only have empty string) if len(augmentation) < 1: print('No input augmentation requirement (e.g. flip)') return True # print(augmentation) # sys.exit(1) # number of classes num_classes_noBG = parameters.get_digit_parameters(options.para_file, 'NUM_CLASSES_noBG', None, 'int') global num_classes num_classes = num_classes_noBG + 1 # ignore_classes = parameters.get_string_parameters( options.para_file, 'data_aug_ignore_classes') ignore_classes = [item.strip() for item in ignore_classes.split(',')] ignore_classes = list(filter( None, ignore_classes)) # remove empty str (in case only have empty string) img_list_txt = args[0] if os.path.isfile(img_list_txt) is False: print("Error, File %s not exist" % img_list_txt) return False f_obj = open(img_list_txt) index = 1 files_list = f_obj.readlines() for line in files_list: # ignore_classes if len(ignore_classes) > 0: found_class = [ line.find(ignore_class) >= 0 for ignore_class in ignore_classes ] if True in found_class: continue file_path = line.strip() file_path = os.path.join(img_dir, file_path + extension) print("Augmentation of image (%d / %d)" % (index, len(files_list))) if image_augment( file_path, out_dir, is_groud_true, augment=augmentation) is False: print('Error, Failed in image augmentation') return False index += 1 f_obj.close() if index < len(files_list): basic.outputlogMessage('Some of the images belong to %s are ingnore' % ','.join(ignore_classes)) # update img_list_txt new_files = io_function.get_file_list_by_ext(extension, '.', bsub_folder=False) new_files_noext = [ os.path.splitext(os.path.basename(item))[0] + '\n' for item in new_files ] basic.outputlogMessage('save new file list to %s' % options.save_list) with open(options.save_list, 'w') as f_obj: f_obj.writelines(new_files_noext)
def image_label_to_yolo_format(para_file): print("Image labels (semantic segmentation) to YOLO object detection") if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd())) img_ext = parameters.get_string_parameters_None_if_absence( para_file, 'split_image_format') proc_num = parameters.get_digit_parameters(para_file, 'process_num', 'int') SECONDS = time.time() # get image and label path image_list = [] label_list = [] with open(os.path.join('list', 'trainval.txt'), 'r') as f_obj: lines = [item.strip() for item in f_obj.readlines()] for line in lines: image_list.append(os.path.join('split_images', line + img_ext)) label_list.append(os.path.join('split_labels', line + img_ext)) num_classes_noBG = parameters.get_digit_parameters_None_if_absence( para_file, 'NUM_CLASSES_noBG', 'int') b_ignore_edge_objects = parameters.get_bool_parameters_None_if_absence( para_file, 'b_ignore_edge_objects') if b_ignore_edge_objects is None: b_ignore_edge_objects = False # get boxes total_count = len(image_list) for idx, (img, label) in enumerate(zip(image_list, label_list)): get_yolo_boxes_one_img(idx, total_count, img, label, num_classes_noBG, rm_edge_obj=b_ignore_edge_objects) # write obj.data file train_sample_txt = parameters.get_string_parameters( para_file, 'training_sample_list_txt') val_sample_txt = parameters.get_string_parameters( para_file, 'validation_sample_list_txt') train_img_list = get_image_list('list', train_sample_txt, 'split_images', img_ext) val_img_list = get_image_list('list', val_sample_txt, 'split_images', img_ext) expr_name = parameters.get_string_parameters(para_file, 'expr_name') object_names = parameters.get_string_list_parameters( para_file, 'object_names') io_function.mkdir('data') io_function.mkdir(expr_name) with open(os.path.join('data', 'obj.data'), 'w') as f_obj: f_obj.writelines('classes = %d' % num_classes_noBG + '\n') train_txt = os.path.join('data', 'train.txt') io_function.save_list_to_txt(train_txt, train_img_list) f_obj.writelines('train = %s' % train_txt + '\n') val_txt = os.path.join('data', 'val.txt') io_function.save_list_to_txt(val_txt, val_img_list) f_obj.writelines('valid = %s' % val_txt + '\n') obj_name_txt = os.path.join('data', 'obj.names') io_function.save_list_to_txt(obj_name_txt, object_names) f_obj.writelines('names = %s' % obj_name_txt + '\n') f_obj.writelines('backup = %s' % expr_name + '\n') duration = time.time() - SECONDS os.system( 'echo "$(date): time cost of converting to yolo format: %.2f seconds">>time_cost.txt' % duration) pass
print("Model: ", args.model) # print("Dataset: ", args.dataset) print("the parameter file: ", args.para_file) # print("Logs: ", args.logs) # print("Auto Download: ", args.download) para_file = args.para_file inf_list_file = args.inf_list_file inf_output_dir = args.inf_output_dir expr_name = parameters.get_string_parameters(args.para_file, 'expr_name') if os.path.isdir(expr_name) is False: os.mkdir(expr_name) NO_DATA = parameters.get_digit_parameters(args.para_file, 'dst_nodata', None, 'int') num_class_noBG = parameters.get_digit_parameters(args.para_file, 'NUM_CLASSES_noBG', None, 'int') # modify default setting according to different machine gpu_count = parameters.get_digit_parameters(args.para_file, 'gpu_count', None, 'int') images_per_gpu = parameters.get_digit_parameters(args.para_file, 'images_per_gpu', None, 'int') PlanetConfig.GPU_COUNT = gpu_count PlanetConfig.IMAGES_PER_GPU = images_per_gpu # Number of training and validation steps per epoch, same as nucleus sample # https://github.com/matterport/Mask_RCNN/issues/1092
def __init__(self, args): self.args = args # Define Saver self.saver = Saver(args) self.saver.save_experiment_config() # Define Tensorboard Summary self.summary = TensorboardSummary(self.saver.experiment_dir) self.writer = self.summary.create_summary() # Define Dataloader kwargs = {'num_workers': args.workers, 'pin_memory': True} parameters.set_saved_parafile_path(args.para) patch_w = parameters.get_digit_parameters("", "train_patch_width", None, 'int') patch_h = parameters.get_digit_parameters("", "train_patch_height", None, 'int') overlay_x = parameters.get_digit_parameters("", "train_pixel_overlay_x", None, 'int') overlay_y = parameters.get_digit_parameters("", "train_pixel_overlay_y", None, 'int') crop_height = parameters.get_digit_parameters("", "crop_height", None, 'int') crop_width = parameters.get_digit_parameters("", "crop_width", None, 'int') dataset = RemoteSensingImg(args.dataroot, args.list, patch_w, patch_h, overlay_x, overlay_y) #train_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, # num_workers=args.workers, shuffle=True) train_length = int(len(dataset) * 0.9) validation_length = len(dataset) - train_length #print ("totol data len is %d , train_length is %d"%(len(train_loader),train_length)) [self.train_dataset, self.val_dataset] = torch.utils.data.random_split(dataset, (train_length, validation_length)) print("len of train dataset is %d and val dataset is %d and total datalen is %d"%(len(self.train_dataset),len(self.val_dataset),len(dataset))) self.train_loader=torch.utils.data.DataLoader(self.train_dataset, batch_size=args.batch_size,num_workers=args.workers, shuffle=True,drop_last=True) self.val_loader=torch.utils.data.DataLoader(self.val_dataset, batch_size=args.batch_size,num_workers=args.workers, shuffle=True,drop_last=True) print("len of train loader is %d and val loader is %d"%(len(self.train_loader),len(self.val_loader))) #self.train_loader, self.val_loader, self.test_loader, self.nclass = make_data_loader(args, **kwargs) # Define network model = DeepLab(num_classes=1, backbone=args.backbone, output_stride=args.out_stride, sync_bn=args.sync_bn, freeze_bn=args.freeze_bn) train_params = [{'params': model.get_1x_lr_params(), 'lr': args.lr}, {'params': model.get_10x_lr_params(), 'lr': args.lr * 10}] # Define Optimizer optimizer = torch.optim.SGD(train_params, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=args.nesterov) # whether to use class balanced weights # if args.use_balanced_weights: # classes_weights_path = os.path.join(Path.db_root_dir(args.dataset), args.dataset+'_classes_weights.npy') # if os.path.isfile(classes_weights_path): # weight = np.load(classes_weights_path) # else: # weight = calculate_weigths_labels(args.dataset, self.train_loader, self.nclass) # weight = torch.from_numpy(weight.astype(np.float32)) # else: # weight = None # Define Criterion #self.criterion = SegmentationLosses(weight=weight, cuda=args.cuda).build_loss(mode=args.loss_type) self.criterion=nn.BCELoss() if args.cuda: self.criterion=self.criterion.cuda() self.model, self.optimizer = model, optimizer # Define Evaluator self.evaluator = Evaluator(2) # Define lr scheduler print("lenght of train_loader is %d"%(len(self.train_loader))) self.scheduler = LR_Scheduler(args.lr_scheduler, args.lr, args.epochs, len(self.train_loader)) # Using cuda if args.cuda: #self.model = torch.nn.DataParallel(self.model, device_ids=self.args.gpu_ids) self.model = torch.nn.DataParallel(self.model) patch_replication_callback(self.model) self.model = self.model.cuda() # Resuming checkpoint self.best_pred = 0.0 if args.resume is not None: if not os.path.isfile(args.resume): raise RuntimeError("=> no checkpoint found at '{}'" .format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] if args.cuda: self.model.module.load_state_dict(checkpoint['state_dict']) else: self.model.load_state_dict(checkpoint['state_dict']) if not args.ft: self.optimizer.load_state_dict(checkpoint['optimizer']) self.best_pred = checkpoint['best_pred'] print("=> loaded checkpoint '{}' (epoch {}) with best mIoU {}" .format(args.resume, checkpoint['epoch'], checkpoint['best_pred'])) # Clear start epoch if fine-tuning if args.ft: args.start_epoch = 0 self.best_pred=0
def evaluation_result(para_file, result_shp, val_shp, evaluation_txt=None): """ evaluate the result based on IoU :param result_shp: result shape file contains detected polygons :param val_shp: shape file contains validation polygons :return: True is successful, False otherwise """ basic.outputlogMessage("evaluation result") IoUs = vector_features.calculate_IoU_scores(result_shp, val_shp) if IoUs is False: return False #save IoU to result shapefile operation_obj = shape_opeation() operation_obj.add_one_field_records_to_shapefile(result_shp, IoUs, 'IoU') iou_threshold = parameters.get_digit_parameters(para_file, 'IOU_threshold', 'float') true_pos_count = 0 false_pos_count = 0 val_polygon_count = operation_obj.get_shapes_count(val_shp) # calculate precision, recall, F1 score for iou in IoUs: if iou > iou_threshold: true_pos_count += 1 else: false_pos_count += 1 false_neg_count = val_polygon_count - true_pos_count if false_neg_count < 0: basic.outputlogMessage( 'warning, false negative count is smaller than 0, recall can not be trusted' ) precision = float(true_pos_count) / (float(true_pos_count) + float(false_pos_count)) recall = float(true_pos_count) / (float(true_pos_count) + float(false_neg_count)) if (true_pos_count > 0): F1score = 2.0 * precision * recall / (precision + recall) else: F1score = 0 #output evaluation reslult if evaluation_txt is None: evaluation_txt = "evaluation_report.txt" f_obj = open(evaluation_txt, 'w') f_obj.writelines('true_pos_count: %d\n' % true_pos_count) f_obj.writelines('false_pos_count: %d\n' % false_pos_count) f_obj.writelines('false_neg_count: %d\n' % false_neg_count) f_obj.writelines('precision: %.6f\n' % precision) f_obj.writelines('recall: %.6f\n' % recall) f_obj.writelines('F1score: %.6f\n' % F1score) f_obj.close() ########################################################################################## ## another method for calculating false_neg_count base on IoU value # calculate the IoU for validation polygons (ground truths) IoUs = vector_features.calculate_IoU_scores(val_shp, result_shp) if IoUs is False: return False # if the IoU of a validation polygon smaller than threshold, then it's false negative false_neg_count = 0 idx_of_false_neg = [] for idx, iou in enumerate(IoUs): if iou < iou_threshold: false_neg_count += 1 idx_of_false_neg.append(idx + 1) # index start from 1 precision = float(true_pos_count) / (float(true_pos_count) + float(false_pos_count)) recall = float(true_pos_count) / (float(true_pos_count) + float(false_neg_count)) if (true_pos_count > 0): F1score = 2.0 * precision * recall / (precision + recall) else: F1score = 0 # output evaluation reslult # evaluation_txt = "evaluation_report.txt" f_obj = open(evaluation_txt, 'a') # add to "evaluation_report.txt" f_obj.writelines('\n\n** Count false negative by IoU**\n') f_obj.writelines('true_pos_count: %d\n' % true_pos_count) f_obj.writelines('false_pos_count: %d\n' % false_pos_count) f_obj.writelines('false_neg_count_byIoU: %d\n' % false_neg_count) f_obj.writelines('precision: %.6f\n' % precision) f_obj.writelines('recall: %.6f\n' % recall) f_obj.writelines('F1score: %.6f\n' % F1score) # output the index of false negative f_obj.writelines('\nindex (start from 1) of false negatives: %s\n' % ','.join([str(item) for item in idx_of_false_neg])) f_obj.close() pass
def train_evaluation_deeplab(WORK_DIR, deeplab_dir, expr_name, para_file, network_setting_ini, gpu_num): # prepare training folder EXP_FOLDER = expr_name INIT_FOLDER = os.path.join(WORK_DIR, EXP_FOLDER, 'init_models') TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train') EVAL_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'eval') VIS_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'vis') EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export') io_function.mkdir(INIT_FOLDER) io_function.mkdir(TRAIN_LOGDIR) io_function.mkdir(EVAL_LOGDIR) io_function.mkdir(VIS_LOGDIR) io_function.mkdir(EXPORT_DIR) # prepare the tensorflow check point (pretrained model) for training pre_trained_dir = parameters.get_directory_None_if_absence( network_setting_ini, 'pre_trained_model_folder') pre_trained_tar = parameters.get_string_parameters(network_setting_ini, 'TF_INIT_CKPT') pre_trained_path = os.path.join(pre_trained_dir, pre_trained_tar) if os.path.isfile(pre_trained_path) is False: print('pre-trained model: %s not exist, try to download' % pre_trained_path) # try to download the file pre_trained_url = parameters.get_string_parameters_None_if_absence( network_setting_ini, 'pre_trained_model_url') res = os.system('wget %s ' % pre_trained_url) if res != 0: sys.exit(1) io_function.movefiletodir(pre_trained_tar, pre_trained_dir) # unpack pre-trained model to INIT_FOLDER os.chdir(INIT_FOLDER) res = os.system('tar -xf %s' % pre_trained_path) if res != 0: raise IOError('failed to unpack %s' % pre_trained_path) os.chdir(WORK_DIR) dataset_dir = os.path.join(WORK_DIR, 'tfrecord') batch_size = parameters.get_digit_parameters(network_setting_ini, 'batch_size', 'int') # maximum iteration number iteration_num = parameters.get_digit_parameters(network_setting_ini, 'iteration_num', 'int') base_learning_rate = parameters.get_digit_parameters( network_setting_ini, 'base_learning_rate', 'float') train_output_stride = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_output_stride', 'int') train_atrous_rates1 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_atrous_rates1', 'int') train_atrous_rates2 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_atrous_rates2', 'int') train_atrous_rates3 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'train_atrous_rates3', 'int') inf_output_stride = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_output_stride', 'int') inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates1', 'int') inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates2', 'int') inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates3', 'int') # depth_multiplier default is 1.0. depth_multiplier = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'depth_multiplier', 'float') decoder_output_stride = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'decoder_output_stride', 'int') aspp_convs_filters = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'aspp_convs_filters', 'int') train_script = os.path.join(deeplab_dir, 'train.py') train_split = os.path.splitext( parameters.get_string_parameters(para_file, 'training_sample_list_txt'))[0] model_variant = parameters.get_string_parameters(network_setting_ini, 'model_variant') checkpoint = parameters.get_string_parameters(network_setting_ini, 'tf_initial_checkpoint') init_checkpoint_files = io_function.get_file_list_by_pattern( INIT_FOLDER, checkpoint + '*') if len(init_checkpoint_files) < 1: raise IOError('No initial checkpoint in %s with pattern: %s' % (INIT_FOLDER, checkpoint)) init_checkpoint = os.path.join(INIT_FOLDER, checkpoint) b_early_stopping = parameters.get_bool_parameters(para_file, 'b_early_stopping') b_initialize_last_layer = parameters.get_bool_parameters( para_file, 'b_initialize_last_layer') dataset = parameters.get_string_parameters(para_file, 'dataset_name') num_classes_noBG = parameters.get_digit_parameters_None_if_absence( para_file, 'NUM_CLASSES_noBG', 'int') assert num_classes_noBG != None if b_initialize_last_layer is True: if pre_trained_tar in pre_trained_tar_21_classes: print( 'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21' % pre_trained_tar) num_classes_noBG = 20 if pre_trained_tar in pre_trained_tar_19_classes: print( 'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19' % pre_trained_tar) num_classes_noBG = 18 num_of_classes = num_classes_noBG + 1 image_crop_size = parameters.get_string_list_parameters( para_file, 'image_crop_size') if len(image_crop_size) != 2 and image_crop_size[0].isdigit( ) and image_crop_size[1].isdigit(): raise ValueError('image_crop_size should be height,width') crop_size_str = ','.join(image_crop_size) evl_script = os.path.join(deeplab_dir, 'eval.py') evl_split = os.path.splitext( parameters.get_string_parameters(para_file, 'validation_sample_list_txt'))[0] max_eva_number = 1 # validation interval (epoch) validation_interval = parameters.get_digit_parameters_None_if_absence( para_file, 'validation_interval', 'int') train_count, val_count = get_train_val_sample_count(WORK_DIR, para_file) iter_per_epoch = math.ceil(train_count / batch_size) total_epoches = math.ceil(iteration_num / iter_per_epoch) already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR) if already_trained_iteration >= iteration_num: basic.outputlogMessage('Training already run %d iterations, skip' % already_trained_iteration) return True if validation_interval is None: basic.outputlogMessage( 'No input validation_interval, so training to %d, then evaluating in the end' % iteration_num) # run training train_deeplab(train_script, dataset, train_split, num_of_classes, base_learning_rate, model_variant, init_checkpoint, TRAIN_LOGDIR, dataset_dir, gpu_num, train_atrous_rates1, train_atrous_rates2, train_atrous_rates3, train_output_stride, crop_size_str, batch_size, iteration_num, depth_multiplier, decoder_output_stride, aspp_convs_filters, b_initialize_last_layer) # run evaluation evaluation_deeplab(evl_script, dataset, evl_split, num_of_classes, model_variant, inf_atrous_rates1, inf_atrous_rates2, inf_atrous_rates3, inf_output_stride, TRAIN_LOGDIR, EVAL_LOGDIR, dataset_dir, crop_size_str, max_eva_number, depth_multiplier, decoder_output_stride, aspp_convs_filters) miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes) get_loss_learning_rate_list(TRAIN_LOGDIR) else: basic.outputlogMessage( 'training to the maximum iteration of %d, and evaluating very %d epoch(es)' % (iteration_num, validation_interval)) for epoch in range(validation_interval, total_epoches + validation_interval, validation_interval): to_iter_num = min(epoch * iter_per_epoch, iteration_num) if to_iter_num <= already_trained_iteration: continue basic.outputlogMessage( 'training and evaluating to %d epoches (to iteration: %d)' % (epoch, to_iter_num)) # run training train_deeplab(train_script, dataset, train_split, num_of_classes, base_learning_rate, model_variant, init_checkpoint, TRAIN_LOGDIR, dataset_dir, gpu_num, train_atrous_rates1, train_atrous_rates2, train_atrous_rates3, train_output_stride, crop_size_str, batch_size, to_iter_num, depth_multiplier, decoder_output_stride, aspp_convs_filters, b_initialize_last_layer) # run evaluation evaluation_deeplab(evl_script, dataset, evl_split, num_of_classes, model_variant, inf_atrous_rates1, inf_atrous_rates2, inf_atrous_rates3, inf_output_stride, TRAIN_LOGDIR, EVAL_LOGDIR, dataset_dir, crop_size_str, max_eva_number, depth_multiplier, decoder_output_stride, aspp_convs_filters) # get miou miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes) # save loss value to disk get_loss_learning_rate_list(TRAIN_LOGDIR) # check if need to early stopping if b_early_stopping: if len(miou_dict['overall']) >= 5: # if the last five miou did not improve, then stop training if np.all(np.diff(miou_dict['overall'][-5:]) < 0.005 ): # 0.0001 (%0.01) # 0.5 % basic.outputlogMessage( 'early stopping: stop training because overall miou did not improved in the last five evaluation' ) output_early_stopping_message(TRAIN_LOGDIR) break # plot mIOU, loss, and learnint rate curves iou_path = os.path.join(EVAL_LOGDIR, 'miou.txt') loss_path = os.path.join(TRAIN_LOGDIR, 'loss_learning_rate.txt') miou_curve_path = plot_miou_loss_curve.plot_miou_loss_main( iou_path, train_count=train_count, val_count=val_count, batch_size=batch_size) loss_curve_path = plot_miou_loss_curve.plot_miou_loss_main( loss_path, train_count=train_count, val_count=val_count, batch_size=batch_size) # backup miou and training_loss & learning rate test_id = os.path.basename(WORK_DIR) + '_' + expr_name backup_dir = os.path.join(WORK_DIR, 'result_backup') if os.path.isdir(backup_dir) is False: io_function.mkdir(backup_dir) new_iou_name = os.path.join(backup_dir, test_id + '_' + os.path.basename(iou_path)) io_function.copy_file_to_dst(iou_path, new_iou_name, overwrite=True) miou_curve_bakname = os.path.join( backup_dir, test_id + '_' + os.path.basename(miou_curve_path)) io_function.copy_file_to_dst(miou_curve_path, miou_curve_bakname, overwrite=True) loss_new_name = os.path.join(backup_dir, test_id + '_' + os.path.basename(loss_path)) io_function.copy_file_to_dst(loss_path, loss_new_name, overwrite=True) loss_curve_bakname = os.path.join( backup_dir, test_id + '_' + os.path.basename(loss_curve_path)) io_function.copy_file_to_dst(loss_curve_path, loss_curve_bakname, overwrite=True)
def yolo_postProcess(para_file, inf_post_note, b_skip_getshp=False, test_id=None): # test_id is the related to training if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd())) # the test string in 'exe.sh' test_note = inf_post_note WORK_DIR = os.getcwd() SECONDS = time.time() expr_name = parameters.get_string_parameters(para_file, 'expr_name') network_setting_ini = parameters.get_string_parameters( para_file, 'network_setting_ini') inf_dir = parameters.get_directory(para_file, 'inf_output_dir') nms_overlap_thr = parameters.get_digit_parameters(para_file, 'nms_overlapThresh', 'float') if test_id is None: test_id = os.path.basename(WORK_DIR) + '_' + expr_name # get name of inference areas multi_inf_regions = parameters.get_string_list_parameters( para_file, 'inference_regions') # run post-processing parallel # max_parallel_postProc_task = 8 backup_dir = os.path.join(WORK_DIR, 'result_backup') io_function.mkdir(backup_dir) # loop each inference regions sub_tasks = [] same_area_time_inis = group_same_area_time_observations(multi_inf_regions) region_eva_reports = {} for key in same_area_time_inis.keys(): multi_observations = same_area_time_inis[key] area_name = parameters.get_string_parameters( multi_observations[0], 'area_name') # they have the same name and time area_time = parameters.get_string_parameters(multi_observations[0], 'area_time') merged_shp_list = [] for area_idx, area_ini in enumerate(multi_observations): area_remark = parameters.get_string_parameters( area_ini, 'area_remark') area_save_dir, shp_pre, _ = get_observation_save_dir_shp_pre( inf_dir, area_name, area_time, area_remark, test_id) # get image list inf_image_dir = parameters.get_directory(area_ini, 'inf_image_dir') # it is ok consider a file name as pattern and pass it the following functions to get file list inf_image_or_pattern = parameters.get_string_parameters( area_ini, 'inf_image_or_pattern') inf_img_list = io_function.get_file_list_by_pattern( inf_image_dir, inf_image_or_pattern) img_count = len(inf_img_list) if img_count < 1: raise ValueError( 'No image for inference, please check inf_image_dir and inf_image_or_pattern in %s' % area_ini) merged_shp = os.path.join(WORK_DIR, area_save_dir, shp_pre + '.shp') if b_skip_getshp: pass else: # post image one by one result_shp_list = [] for img_idx, img_path in enumerate(inf_img_list): out_shp = yolo_results_to_shapefile( WORK_DIR, img_idx, area_save_dir, nms_overlap_thr, test_id) if out_shp is not None: #TODO: remove some boxes based on DEM difference? result_shp_list.append(os.path.join(WORK_DIR, out_shp)) # merge shapefiles merge_shape_files(result_shp_list, merged_shp, b_create_id=True) merged_shp_list.append(merged_shp) if b_skip_getshp is False: # add occurrence to each polygons get_occurence_for_multi_observation(merged_shp_list) for area_idx, area_ini in enumerate(multi_observations): area_remark = parameters.get_string_parameters( area_ini, 'area_remark') area_save_dir, shp_pre, area_remark_time = get_observation_save_dir_shp_pre( inf_dir, area_name, area_time, area_remark, test_id) merged_shp = os.path.join(WORK_DIR, area_save_dir, shp_pre + '.shp') # add attributes to shapefile (no other attribute to add) shp_attributes = os.path.join(WORK_DIR, area_save_dir, shp_pre + '_post_NOrm.shp') # shp_attributes = merged_shp add_boxes_attributes(merged_shp, shp_attributes) # para_file, area_ini # remove polygons shp_post = os.path.join(WORK_DIR, area_save_dir, shp_pre + '_post.shp') remove_polygons_main(shp_attributes, shp_post, para_file) # evaluate the mapping results out_report = os.path.join(WORK_DIR, area_save_dir, shp_pre + '_evaluation_report.txt') evaluation_polygons(shp_post, para_file, area_ini, out_report) ##### copy and backup files ###### # copy files to result_backup if len(test_note) > 0: backup_dir_area = os.path.join( backup_dir, area_name + '_' + area_remark_time + '_' + test_id + '_' + test_note) else: backup_dir_area = os.path.join( backup_dir, area_name + '_' + area_remark_time + '_' + test_id) io_function.mkdir(backup_dir_area) if len(test_note) > 0: bak_merged_shp = os.path.join( backup_dir_area, '_'.join([shp_pre, test_note]) + '.shp') bak_post_shp = os.path.join( backup_dir_area, '_'.join([shp_pre, 'post', test_note]) + '.shp') bak_eva_report = os.path.join( backup_dir_area, '_'.join([shp_pre, 'eva_report', test_note]) + '.txt') bak_area_ini = os.path.join( backup_dir_area, '_'.join([shp_pre, 'region', test_note]) + '.ini') else: bak_merged_shp = os.path.join(backup_dir_area, '_'.join([shp_pre]) + '.shp') bak_post_shp = os.path.join( backup_dir_area, '_'.join([shp_pre, 'post']) + '.shp') bak_eva_report = os.path.join( backup_dir_area, '_'.join([shp_pre, 'eva_report']) + '.txt') bak_area_ini = os.path.join( backup_dir_area, '_'.join([shp_pre, 'region']) + '.ini') io_function.copy_shape_file(merged_shp, bak_merged_shp) io_function.copy_shape_file(shp_post, bak_post_shp) io_function.copy_file_to_dst(area_ini, bak_area_ini, overwrite=True) if os.path.isfile(out_report): io_function.copy_file_to_dst(out_report, bak_eva_report, overwrite=True) region_eva_reports[shp_pre] = bak_eva_report if len(test_note) > 0: bak_para_ini = os.path.join( backup_dir, '_'.join([test_id, 'para', test_note]) + '.ini') # bak_network_ini = os.path.join(backup_dir, '_'.join([test_id,'network',test_note]) + '.ini' ) bak_time_cost = os.path.join( backup_dir, '_'.join([test_id, 'time_cost', test_note]) + '.txt') else: bak_para_ini = os.path.join(backup_dir, '_'.join([test_id, 'para']) + '.ini') # bak_network_ini = os.path.join(backup_dir, '_'.join([test_id, 'network']) + '.ini') bak_time_cost = os.path.join(backup_dir, '_'.join([test_id, 'time_cost']) + '.txt') io_function.copy_file_to_dst(para_file, bak_para_ini) # io_function.copy_file_to_dst(network_setting_ini, bak_network_ini) if os.path.isfile('time_cost.txt'): io_function.copy_file_to_dst('time_cost.txt', bak_time_cost) # output the evaluation report to screen for key in region_eva_reports.keys(): report = region_eva_reports[key] print('evaluation report for %s:' % key) os.system('head -n 7 %s' % report) # output evaluation report to table if len(test_note) > 0: out_table = os.path.join( backup_dir, '_'.join([test_id, 'accuracy_table', test_note]) + '.xlsx') else: out_table = os.path.join( backup_dir, '_'.join([test_id, 'accuracy_table']) + '.xlsx') eva_reports = [region_eva_reports[key] for key in region_eva_reports] if len(eva_reports) > 0: eva_report_to_tables.eva_reports_to_table(eva_reports, out_table) duration = time.time() - SECONDS os.system( 'echo "$(date): time cost of post-procesing: %.2f seconds">>time_cost.txt' % duration) pass
default=0, type=int, help= 'switch to train edge, if this is on batchSize should be 1, 0 is off 1 is on' ) parser.add_argument('--test', action='store_true', help='switch to test model, without train') args = parser.parse_args() print(args) ############## dataset processing parameters.set_saved_parafile_path(args.para) patch_w = parameters.get_digit_parameters("", "train_patch_width", None, 'int') patch_h = parameters.get_digit_parameters("", "train_patch_height", None, 'int') overlay_x = parameters.get_digit_parameters("", "train_pixel_overlay_x", None, 'int') overlay_y = parameters.get_digit_parameters("", "train_pixel_overlay_y", None, 'int') crop_height = parameters.get_digit_parameters("", "crop_height", None, 'int') crop_width = parameters.get_digit_parameters("", "crop_width", None, 'int') dataset = RemoteSensingImg(args.dataroot, args.list, patch_w, patch_h, overlay_x, overlay_y, args.edge) train_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batchSize, num_workers=args.workers, shuffle=True) # torch.backends.cudnn.enabled = True
def calculate_polygon_topography(polygons_shp, para_file, dem_files, slope_files, aspect_files=None, dem_diffs=None): """ calculate the topography information such elevation and slope of each polygon Args: polygons_shp: input shapfe file dem_files: DEM raster file or tiles, should have the same projection of shapefile slope_files: slope raster file or tiles (can be drived from dem file by using QGIS or ArcGIS) aspect_files: aspect raster file or tiles (can be drived from dem file by using QGIS or ArcGIS) Returns: True if successful, False Otherwise """ if io_function.is_file_exist(polygons_shp) is False: return False operation_obj = shape_opeation() ## calculate the topography information from the buffer area # the para file was set in parameters.set_saved_parafile_path(options.para_file) b_use_buffer_area = parameters.get_bool_parameters( para_file, 'b_topo_use_buffer_area') if b_use_buffer_area is True: b_buffer_size = 5 # meters (the same as the shape file) basic.outputlogMessage( "info: calculate the topography information from the buffer area") buffer_polygon_shp = io_function.get_name_by_adding_tail( polygons_shp, 'buffer') # if os.path.isfile(buffer_polygon_shp) is False: if vector_features.get_buffer_polygons( polygons_shp, buffer_polygon_shp, b_buffer_size) is False: basic.outputlogMessage( "error, failed in producing the buffer_polygon_shp") return False # else: # basic.outputlogMessage("warning, buffer_polygon_shp already exist, skip producing it") # replace the polygon shape file polygons_shp_backup = polygons_shp polygons_shp = buffer_polygon_shp else: basic.outputlogMessage( "info: calculate the topography information from the inside of each polygon" ) # all_touched: bool, optional # Whether to include every raster cell touched by a geometry, or only # those having a center point within the polygon. # defaults to `False` # Since the dem usually is coarser, so we set all_touched = True all_touched = True process_num = 4 # #DEM if dem_files is not None: stats_list = ['min', 'max', 'mean', 'median', 'std'] #['min', 'max', 'mean', 'count','median','std'] # if operation_obj.add_fields_from_raster(polygons_shp, dem_file, "dem", band=1,stats_list=stats_list,all_touched=all_touched) is False: # return False if zonal_stats_multiRasters(polygons_shp, dem_files, stats=stats_list, prefix='dem', band=1, all_touched=all_touched, process_num=process_num) is False: return False else: basic.outputlogMessage( "warning, DEM file not exist, skip the calculation of DEM information" ) # #slope if slope_files is not None: stats_list = ['min', 'max', 'mean', 'median', 'std'] if zonal_stats_multiRasters(polygons_shp, slope_files, stats=stats_list, prefix='slo', band=1, all_touched=all_touched, process_num=process_num) is False: return False else: basic.outputlogMessage( "warning, slope file not exist, skip the calculation of slope information" ) # #aspect if aspect_files is not None: stats_list = ['min', 'max', 'mean', 'std'] if zonal_stats_multiRasters(polygons_shp, aspect_files, stats=stats_list, prefix='asp', band=1, all_touched=all_touched, process_num=process_num) is False: return False else: basic.outputlogMessage( 'warning, aspect file not exist, ignore adding aspect information') # elevation difference if dem_diffs is not None: stats_list = ['min', 'max', 'mean', 'median', 'std', 'area'] # only count the pixel within this range when do statistics dem_diff_range_str = parameters.get_string_list_parameters( para_file, 'dem_difference_range') range = [ None if item.upper() == 'NONE' else float(item) for item in dem_diff_range_str ] # expand the polygon when doing dem difference statistics buffer_size_dem_diff = parameters.get_digit_parameters( para_file, 'buffer_size_dem_diff', 'float') if zonal_stats_multiRasters(polygons_shp, dem_diffs, stats=stats_list, prefix='demD', band=1, all_touched=all_touched, process_num=process_num, range=range, buffer=buffer_size_dem_diff) is False: return False else: basic.outputlogMessage( 'warning, dem difference file not exist, ignore adding dem diff information' ) # # hillshape # copy the topography information if b_use_buffer_area is True: operation_obj.add_fields_shape(polygons_shp_backup, buffer_polygon_shp, polygons_shp_backup) return True