def read_para_values(work_dir,para_file,train_output):
para_path = os.path.join(work_dir,para_file)
backbone = parameters.get_string_parameters(para_path,'network_setting_ini')
train_output['network_setting_ini'].append(backbone)
net_ini_path = os.path.join(work_dir,backbone)
lr = parameters.get_digit_parameters(net_ini_path,'base_learning_rate','float')
train_output['base_learning_rate'].append(lr)
iter_num = parameters.get_digit_parameters(net_ini_path,'iteration_num','int')
train_output['iteration_num'].append(iter_num)
batch_size = parameters.get_digit_parameters(net_ini_path,'batch_size','int')
train_output['batch_size'].append(batch_size)
buffer_size = parameters.get_digit_parameters(para_path,'buffer_size','int')
train_output['buffer_size'].append(buffer_size)
training_data_per = parameters.get_digit_parameters(para_path,'training_data_per','float')
train_output['training_data_per'].append(training_data_per)
data_augmentation = parameters.get_string_parameters(para_path,'data_augmentation')
train_output['data_augmentation'].append(data_augmentation)
data_aug_ignore_classes = parameters.get_string_parameters(para_path,'data_aug_ignore_classes')
train_output['data_aug_ignore_classes'].append(data_aug_ignore_classes)
return True
def generate_image_CUT(python_path, generate_script, gan_para_file, gpu_ids,
image_list, save_folder):
if os.path.isfile('generate.txt_done'):
basic.outputlogMessage(
'generate of new images using GAN in %s has completed previoulsy, please remove them if necessary'
% os.getcwd())
return True
time0 = time.time()
generate_tile_width = parameters.get_digit_parameters(
gan_para_file, 'generate_tile_width', 'int')
generate_tile_height = parameters.get_digit_parameters(
gan_para_file, 'generate_tile_height', 'int')
generate_overlay_x = parameters.get_digit_parameters(
gan_para_file, 'generate_overlay_x', 'int')
generate_overlay_y = parameters.get_digit_parameters(
gan_para_file, 'generate_overlay_y', 'int')
folder = os.path.basename(os.getcwd())
img_list_txt = 'image_to_generate_list.txt'
io_function.save_list_to_txt(img_list_txt, image_list)
command_string = python_path + ' ' + generate_script \
+ ' --dataset_mode '+'satelliteimage' \
+ ' --model '+ 'generate' \
+ ' --image_A_dir_txt ' + img_list_txt \
+ ' --tile_width ' + str(generate_tile_width) \
+ ' --tile_height ' + str(generate_tile_height) \
+ ' --overlay_x ' + str(generate_overlay_x) \
+ ' --overlay_y ' + str(generate_overlay_y) \
+ ' --name ' + folder \
+ ' --results_dir ' + save_folder \
+ ' --gpu_ids ' + ','.join([str(item) for item in gpu_ids])
train_max_dataset_size = parameters.get_digit_parameters_None_if_absence(
gan_para_file, 'gen_max_dataset_size', 'int')
if train_max_dataset_size is not None:
command_string += ' --max_dataset_size ' + str(train_max_dataset_size)
# if it's cycleGAN, need to assign A generator
gan_model = parameters.get_string_parameters(gan_para_file, 'gan_model')
if gan_model == 'cycle_gan':
command_string += ' --model_suffix _A ' # from A to B
# status, result = basic.exec_command_string(command_string) # this will wait command finished
# os.system(command_string + "&") # don't know when it finished
res = os.system(command_string) # this work
# print('command_string deeplab_inf_script: res',res)
if res != 0:
sys.exit(1)
duration = time.time() - time0
os.system(
'echo "$(date): time cost of generate images using a GAN : %.2f seconds">>"time_cost.txt"'
% (duration))
# write a file to indicate that the process has completed.
os.system('echo done > generate.txt_done')
return True
def get_sub_images_from_prediction_results(para_file, polygons_shp,
image_folder_or_path, image_pattern,
saved_dir):
class_names = parameters.get_string_list_parameters(
para_file, 'object_names')
dstnodata = parameters.get_digit_parameters(para_file, 'dst_nodata', 'int')
bufferSize = parameters.get_digit_parameters(para_file, 'buffer_size',
'int')
rectangle_ext = parameters.get_string_parameters_None_if_absence(
para_file, 'b_use_rectangle')
if rectangle_ext is not None:
b_rectangle = True
else:
b_rectangle = False
process_num = parameters.get_digit_parameters(para_file, 'process_num',
'int')
get_sub_images_pixel_json_files(polygons_shp, image_folder_or_path,
image_pattern, class_names, bufferSize,
dstnodata, saved_dir, b_rectangle,
process_num)
pass
def get_iteration_num(WORK_DIR, para_file, network_setting_ini):
'''
get iteration num from setting (iteration_num) or calculate from train_epoch_num, batch_size and training sample count
:param WORK_DIR:
:param para_file:
:param network_setting_ini:
:return:
'''
train_epoch_num = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_epoch_num', 'int')
if train_epoch_num is None:
iteration_num = parameters.get_digit_parameters(
network_setting_ini, 'iteration_num', 'int')
with open('iteration_num.txt', 'w') as f_obj:
f_obj.writelines('iteration num is read from setting: %d\n' %
iteration_num)
else:
train_count, val_count = get_train_val_sample_count(
WORK_DIR, para_file)
batch_size = parameters.get_digit_parameters(network_setting_ini,
'batch_size', 'int')
iteration_num = math.ceil(train_epoch_num * train_count / batch_size)
with open('iteration_num.txt', 'w') as f_obj:
f_obj.writelines(
'iteration num (%d) is calculated from train_epoch_num: %d, training_sample_count: %d, '
'batch_size: %d\n' %
(iteration_num, train_epoch_num, train_count, batch_size))
return iteration_num
def predict_one_image_mmseg(para_file, image_path, img_save_dir, inf_list_file,
gpuid, trained_model):
""" run prediction of one image
"""
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
network_ini = parameters.get_string_parameters(para_file,
'network_setting_ini')
base_config_file = parameters.get_string_parameters(
network_ini, 'base_config')
config_file = osp.basename(
io_function.get_name_by_adding_tail(base_config_file, expr_name))
inf_batch_size = parameters.get_digit_parameters(network_ini,
'inf_batch_size', 'int')
patch_width = parameters.get_digit_parameters(para_file, 'inf_patch_width',
'int')
patch_height = parameters.get_digit_parameters(para_file,
'inf_patch_height', 'int')
adj_overlay_x = parameters.get_digit_parameters(para_file,
'inf_pixel_overlay_x',
'int')
adj_overlay_y = parameters.get_digit_parameters(para_file,
'inf_pixel_overlay_y',
'int')
done_indicator = '%s_done' % inf_list_file
if os.path.isfile(done_indicator):
basic.outputlogMessage('warning, %s exist, skip prediction' %
done_indicator)
return
if os.path.isdir(img_save_dir) is False:
io_function.mkdir(img_save_dir)
# use a specific GPU for prediction, only inference one image
time0 = time.time()
if gpuid is None:
gpuid = 0
predict_rsImage_mmseg(config_file,
trained_model,
image_path,
img_save_dir,
batch_size=inf_batch_size,
gpuid=gpuid,
tile_width=patch_width,
tile_height=patch_height,
overlay_x=adj_overlay_x,
overlay_y=adj_overlay_y)
duration = time.time() - time0
os.system(
'echo "$(date): time cost of inference for image in %s: %.2f seconds">>"time_cost.txt"'
% (inf_list_file, duration))
# write a file to indicate that the prediction has done.
os.system('echo %s > %s_done' % (inf_list_file, inf_list_file))
return
def updated_config_file(WORK_DIR, expr_name, base_config_file, save_path,
para_file, network_setting_ini, gpu_num):
'''
update mmseg config file and save to a new file in local working folder
:param WORK_DIR:
:param expr_name:
:param base_config_file:
:param save_path:
:param para_file:
:param network_setting_ini:
:return:
'''
# read the config, then modifi some, them dump to disk
cfg = Config.fromfile(base_config_file)
# 4 basic component: dataset, model, schedule, default_runtime
# change model (no need): when we choose a base_config_file, we already choose a model, including backbone)
# change dataset
modify_dataset(cfg, para_file, network_setting_ini, gpu_num)
# change num_classes
# NUM_CLASSES_noBG = parameters.get_digit_parameters(para_file,'NUM_CLASSES_noBG','int')
# cfg.model.decode_head['num_classes'] = NUM_CLASSES_noBG + 1
# cfg.model.auxiliary_head['num_classes'] = NUM_CLASSES_noBG + 1
# change schedule
iteration_num = get_iteration_num(WORK_DIR, para_file, network_setting_ini)
cfg.runner['max_iters'] = iteration_num
checkpoint_interval = parameters.get_digit_parameters(
network_setting_ini, 'checkpoint_interval', 'int')
cfg.checkpoint_config['interval'] = checkpoint_interval
evaluation_interval = parameters.get_digit_parameters(
network_setting_ini, 'evaluation_interval', 'int')
cfg.evaluation['interval'] = evaluation_interval
# change runtime (log level, resume_from or load_from)
cfg.work_dir = os.path.join(WORK_DIR, expr_name)
# update parameters for testing (used in later step of prediction)
updated_config_file_for_test(cfg)
# dump config
cfg.dump(save_path)
return True
def predict_remoteSensing_image(para_file, image_path, save_dir,model, config_file, yolo_data, batch_size=1, b_python_api=True):
'''
run prediction of a remote sensing using yolov4
:param image_path:
:param model:
:param config_file:
:param yolo_data:
:param batch_size:
:param b_python_api: if true, use the python API of yolo
:return:
'''
patch_w = parameters.get_digit_parameters(para_file, "inf_patch_width", 'int')
patch_h = parameters.get_digit_parameters(para_file, "inf_patch_height", 'int')
overlay_x = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_x", 'int')
overlay_y = parameters.get_digit_parameters(para_file, "inf_pixel_overlay_y", 'int')
if b_python_api:
# using the python API
predict_rs_image_yolo_poythonAPI(image_path, save_dir, model, config_file, yolo_data,
patch_w, patch_h, overlay_x, overlay_y, batch_size=batch_size)
# for each patch has a json file, may end up with a lot of json files, affect I/O
# try to merge them to one json file.
res_json_files = io_function.get_file_list_by_ext('.json', save_dir, bsub_folder=False)
merge_josn_path = os.path.join(save_dir,'all_patches.json')
merge_patch_json_files_to_one(res_json_files,merge_josn_path)
for f_json in res_json_files:
io_function.delete_file_or_dir(f_json)
else:
# divide image the many patches, then run prediction.
patch_list_txt = split_an_image(para_file,image_path,save_dir,patch_w,patch_h,overlay_x,overlay_y)
if patch_list_txt is None:
return False
result_json = save_dir + '_result.json'
commond_str = 'darknet detector test ' + yolo_data + ' ' + config_file + ' ' + model + ' -dont_show '
commond_str += ' -ext_output -out ' + result_json + ' < ' + patch_list_txt
print(commond_str)
res = os.system(commond_str)
if res !=0:
sys.exit(1)
def modify_dataset(cfg, para_file, network_setting_ini, gpu_num):
datetype = 'RSImagePatches'
cfg.dataset_type = datetype
cfg.data_root = './'
## There are still two more crop_size in pipleline, after choosing the base_config_file, we already choose the crop size
# image_crop_size = parameters.get_string_list_parameters(para_file, 'image_crop_size')
# image_crop_size = [ int(item) for item in image_crop_size]
# if len(image_crop_size) != 2 and image_crop_size[0].isdigit() and image_crop_size[1].isdigit():
# raise ValueError('image_crop_size should be height,width')
# cfg.crop_size = (image_crop_size[0],image_crop_size[1])
training_sample_list_txt = parameters.get_string_parameters(
para_file, 'training_sample_list_txt')
validation_sample_list_txt = parameters.get_string_parameters(
para_file, 'validation_sample_list_txt')
split_list = ['train', 'val', 'test']
for split in split_list:
# dataset in train
cfg.data[split]['type'] = datetype
cfg.data[split]['data_root'] = './'
cfg.data[split]['img_dir'] = 'split_images'
cfg.data[split]['ann_dir'] = 'split_labels'
if split == 'train':
cfg.data[split]['split'] = [
osp.join('list', training_sample_list_txt)
]
else:
# set val and test to validation, when run real test (prediction) for entire RS images, we will set test again.
cfg.data[split]['split'] = [
osp.join('list', validation_sample_list_txt)
]
# set None for test
cfg.data['test']['img_dir'] = None
cfg.data['test']['ann_dir'] = None
cfg.data['test']['split'] = None
# setting based on batch size
batch_size = parameters.get_digit_parameters(network_setting_ini,
'batch_size', 'int')
if batch_size % gpu_num != 0:
raise ValueError('Batch size (%d) cannot be divided by gpu num (%d)' %
(batch_size, gpu_num))
cfg.data['samples_per_gpu'] = int(batch_size / gpu_num)
cfg.data['workers_per_gpu'] = int(
batch_size / gpu_num) + 2 # set worker a litter higher to utilize CPU
return True
def training_img_augment(para_file):
print("start data augmentation")
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s'%(para_file, os.getcwd()))
# augscript = os.path.join(code_dir,'datasets','image_augment.py')
img_ext = parameters.get_string_parameters_None_if_absence(para_file,'split_image_format')
print("image format: %s"% img_ext)
proc_num = parameters.get_digit_parameters(para_file, 'process_num', 'int')
SECONDS=time.time()
from datasets.image_augment import image_augment_main
#augment training images
print("image augmentation on image patches")
img_list_aug_txt = 'list/images_including_aug.txt'
# command_string = augscript + ' -p ' + para_file + ' -d ' + 'split_images' + ' -e ' + img_ext + ' -n ' + str(proc_num) + \
# ' -o ' + 'split_images' + ' -l ' + img_list_aug_txt + ' ' + 'list/trainval.txt'
# res = os.system(command_string)
# if res!=0:
# sys.exit(1)
image_augment_main(para_file,'list/trainval.txt',img_list_aug_txt,'split_images','split_images',img_ext,False,proc_num)
#augment training lables
print("image augmentation on label patches")
# command_string = augscript + ' -p ' + para_file + ' -d ' + 'split_labels' + ' -e ' + img_ext + ' -n ' + str(proc_num) + \
# ' -o ' + 'split_labels' + ' -l ' + img_list_aug_txt + ' ' + 'list/trainval.txt' + ' --is_ground_truth '
#
# res = os.system(command_string)
# if res!=0:
# sys.exit(1)
# save the result to the same file (redundant, they have the same filename)
image_augment_main(para_file, 'list/trainval.txt', img_list_aug_txt, 'split_labels', 'split_labels', img_ext, True,proc_num)
if os.path.isfile(img_list_aug_txt):
os.system(' cp %s list/trainval.txt'%img_list_aug_txt)
os.system(' cp %s list/val.txt'%img_list_aug_txt)
else:
print('list/images_including_aug.txt does not exist because no data augmentation strings')
# output the number of image patches (ls may failed if there are a lot of files, so remove these two lines)
# os.system('echo "count of class 0 ":$(ls split_images/*class_0*${img_ext} |wc -l) >> time_cost.txt')
# os.system('echo "count of class 1 ":$(ls split_images/*class_1*${img_ext} |wc -l) >> time_cost.txt')
duration= time.time() - SECONDS
os.system('echo "$(date): time cost of data augmentation: %.2f seconds">>time_cost.txt'%duration)
def main(unused_argv):
# dataset_splits = glob.glob(os.path.join(FLAGS.list_file, '*.txt'))
#
# for dataset_split in dataset_splits:
# _convert_dataset(dataset_split)
#how about the mean value?
#split images
data_root = FLAGS.image_folder
list_txt = FLAGS.list_file
############## dataset processing
parameters.set_saved_parafile_path(FLAGS.para_file)
patch_w = parameters.get_digit_parameters("", "train_patch_width", None,
'int')
patch_h = parameters.get_digit_parameters("", "train_patch_height", None,
'int')
overlay_x = parameters.get_digit_parameters("", "train_pixel_overlay_x",
None, 'int')
overlay_y = parameters.get_digit_parameters("", "train_pixel_overlay_y",
None, 'int')
patches = make_dataset(data_root,
list_txt,
patch_w,
patch_h,
overlay_x,
overlay_y,
train=FLAGS.is_training)
os.system("mkdir -p " + FLAGS.output_dir)
#convert images
patches_1d = [item for alist in patches
for item in alist] # convert 2D list to 1D
_convert_dataset(patches_1d, train=FLAGS.is_training)
pass
def main(options, args):
input_file = args[0]
img_patch_dir = options.image_dir
label_patch_dir = options.label_dir
basic.outputlogMessage(
'check split image patches and label patches in %s, especially after data augmentation'
% input_file)
b_diff = False
num_classes_noBG = parameters.get_digit_parameters(options.para_file,
'NUM_CLASSES_noBG',
None, 'int')
with open(input_file, 'r') as f_obj:
dir = os.path.dirname(input_file)
files_list = f_obj.readlines()
for file_name in files_list:
file_name = file_name.strip()
img_path = os.path.join(img_patch_dir, file_name + '.png')
label_path = os.path.join(label_patch_dir, file_name + '.png')
img_data = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
h_w = img_data.shape[:2]
label_data = cv2.imread(label_path, cv2.IMREAD_UNCHANGED)
if h_w != label_data.shape:
b_diff = True
basic.outputlogMessage(
'%s have different size in image (%s) and label (%s) patch'
% (file_name, str(h_w), str(label_data.shape)))
max_label = np.max(label_data)
unique_value = np.unique(label_data)
if max_label > num_classes_noBG:
b_diff = True
basic.outputlogMessage(
'%s: maximum pixel value (%d) in label images > num_class (%d)'
% (file_name, max_label, num_classes_noBG))
if len(unique_value) > num_classes_noBG + 1:
b_diff = True
basic.outputlogMessage(
'%s: the count of unique pixel value (%s) in label images is greater than num_class (%d)'
% (file_name, str(unique_value), num_classes_noBG))
if b_diff is False:
basic.outputlogMessage('all the patches are good')
def add_boxes_attributes(input, output, para_file=None,data_para_file=None):
if io_function.is_file_exist(input) is False:
return False
# copy output
if io_function.copy_shape_file(input, output) is False:
raise IOError('copy shape file %s failed'%input)
# calculate area, perimeter of polygons
if cal_add_area_length_of_polygon(output) is False:
return False
# calculate the width, height, and width_height ratio
boxes = vector_gpd.read_polygons_gpd(input)
bounding_boxes = [ vector_gpd.get_polygon_bounding_box(item) for item in boxes ]
# bounding box (minx, miny, maxx, maxy)
width_list = [ bound[2] - bound[0] for bound in bounding_boxes]
height_list = [ bound[3] - bound[1] for bound in bounding_boxes]
ratio_w_h_list = []
for w, h in zip(width_list, height_list):
if w > h:
ratio = h / w
else:
ratio = w / h
ratio_w_h_list.append(ratio)
add_attributes = {'WIDTH':width_list,'HEIGHT':height_list, 'ratio_w_h':ratio_w_h_list}
vector_gpd.add_attributes_to_shp(output,add_attributes)
# add topography of each box (consider the box as a polygon)
if data_para_file is not None and para_file is not None:
io_function.is_file_exist(data_para_file)
io_function.is_file_exist(para_file)
dem_files, slope_files, aspect_files, dem_diff_files = get_topographic_files(data_para_file)
process_num = parameters.get_digit_parameters(para_file,'process_num','int')
if calculate_polygon_topography(output,para_file,dem_files,slope_files,aspect_files=aspect_files,dem_diffs=dem_diff_files,
process_num=process_num) is False:
basic.outputlogMessage('Warning: calculate information of topography failed')
# return False # don't return
return output
def main(options, args):
input_shp = args[0]
output_raster = args[1]
if io_function.is_file_exist(input_shp) is False:
return False
all_class_raster = io_function.get_name_by_adding_tail(
output_raster, 'AllClass')
num_class = parameters.get_digit_parameters(options.para_file,
'NUM_CLASSES_noBG', None,
'int')
if convert_training_examples_from_shp_to_raster(input_shp,
all_class_raster) is False:
basic.outputlogMessage(
"Producing the label images from training polygons is Falild")
return False
else:
basic.outputlogMessage(
"Done: Producing the label images from training polygons, output: %s"
% all_class_raster)
if num_class == 1:
#only keep target (gully or others) label
one_class_raster = io_function.get_name_by_adding_tail(
output_raster, 'oneClass')
if only_keep_one_class(
all_class_raster, one_class_raster, class_index=1) is False:
return False
else:
one_class_raster = all_class_raster
# crop the label image to have the same 2D dimension with the training images
baseimage = parameters.get_input_image_path()
if RSImageProcess.subset_image_baseimage(output_raster, one_class_raster,
baseimage) is False:
basic.outputlogMessage("Error: subset_image_baseimage Failed")
return False
return True
def predict_one_image_yolo(para_file, image_path, img_save_dir, inf_list_file,
gpuid, trained_model):
config_file = parameters.get_string_parameters(
para_file, 'network_setting_ini') # 'yolov4_obj.cfg'
yolo_data = os.path.join('data', 'obj.data')
# b_python_api = False
inf_batch_size = parameters.get_digit_parameters(para_file,
'inf_batch_size', 'int')
b_python_api = parameters.get_bool_parameters(para_file,
'b_inf_use_python_api')
done_indicator = '%s_done' % inf_list_file
if os.path.isfile(done_indicator):
basic.outputlogMessage('warning, %s exist, skip prediction' %
done_indicator)
return
# use a specific GPU for prediction, only inference one image
time0 = time.time()
if gpuid is not None:
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpuid)
predict_remoteSensing_image(para_file,
image_path,
img_save_dir,
trained_model,
config_file,
yolo_data,
batch_size=inf_batch_size,
b_python_api=b_python_api)
duration = time.time() - time0
os.system(
'echo "$(date): time cost of inference for image in %s: %.2f seconds">>"time_cost.txt"'
% (inf_list_file, duration))
# write a file to indicate that the prediction has done.
os.system('echo %s > %s_done' % (inf_list_file, inf_list_file))
return
def split_sub_images(para_file):
print("split sub-images and sub-labels")
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s'%(para_file, os.getcwd()))
SECONDS = time.time()
if os.path.isdir('split_images'):
io_function.delete_file_or_dir('split_images')
if os.path.isdir('split_labels'):
io_function.delete_file_or_dir('split_labels')
io_function.mkdir('split_images')
### split the training image to many small patch (480*480)
patch_w=parameters.get_string_parameters(para_file,'train_patch_width')
patch_h=parameters.get_string_parameters(para_file,'train_patch_height')
overlay_x=parameters.get_string_parameters(para_file,'train_pixel_overlay_x')
overlay_y=parameters.get_string_parameters(para_file,'train_pixel_overlay_y')
split_image_format=parameters.get_string_parameters(para_file,'split_image_format')
trainImg_dir=parameters.get_string_parameters(para_file,'input_train_dir')
labelImg_dir=parameters.get_string_parameters(para_file,'input_label_dir')
proc_num = parameters.get_digit_parameters(para_file,'process_num','int')
if os.path.isdir(trainImg_dir) is False:
raise IOError('%s not in the current folder, please get subImages first'%trainImg_dir)
if os.path.isdir(labelImg_dir) is False:
print('warning, %s not in the current folder'%labelImg_dir)
else:
io_function.mkdir('split_labels')
sub_img_label_txt = 'sub_images_labels_list.txt'
if os.path.isfile(sub_img_label_txt) is False:
raise IOError('%s not in the current folder, please get subImages first' % sub_img_label_txt)
with open(sub_img_label_txt) as txt_obj:
line_list = [name.strip() for name in txt_obj.readlines()]
# for line in line_list:
# sub_image, sub_label = line.split(':')
#
# # split sub image
# split_to_patches(sub_image, 'split_images', patch_w, patch_h, overlay, overlay, split_image_format)
#
# # split sub label (change the file name to be the same as sub_image name)
# pre_name = os.path.splitext(os.path.basename(sub_image))[0]
# split_to_patches(sub_label, 'split_labels', patch_w, patch_h, overlay, overlay, split_image_format, file_pre_name=pre_name)
parameters_list = [(line, patch_w, patch_h, overlay_x, overlay_y, split_image_format) for line in line_list]
theadPool = Pool(proc_num) # multi processes
results = theadPool.starmap(split_a_pair_sub_image_label, parameters_list) # need python3
# output trainval.txt and val.txt file
files_list = io_function.get_file_list_by_ext(split_image_format, 'split_images',bsub_folder=False)
io_function.mkdir('list')
trainval = os.path.join('list','trainval.txt')
val = os.path.join('list','val.txt')
with open(trainval,'w') as w_obj:
for file_name in files_list:
w_obj.writelines(os.path.splitext(os.path.basename(file_name))[0] + '\n')
io_function.copy_file_to_dst(trainval,val,overwrite=True)
split_train_val.get_image_with_height_list(trainval, split_image_format, info_type='(no data augmentation)')
duration= time.time() - SECONDS
os.system('echo "$(date): time cost of splitting sub images and labels: %.2f seconds">>time_cost.txt'%duration)
def get_sub_images_multi_regions(para_file):
print(
"extract sub-images and sub-labels for a given shape file (training polygons)"
)
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
get_subImage_script = os.path.join(code_dir, 'datasets',
'get_subImages.py')
SECONDS = time.time()
# get name of training areas
multi_training_regions = parameters.get_string_list_parameters_None_if_absence(
para_file, 'training_regions')
if multi_training_regions is None or len(multi_training_regions) < 1:
raise ValueError('No training area is set in %s' % para_file)
# multi_training_files = parameters.get_string_parameters_None_if_absence(para_file, 'multi_training_files')
dstnodata = parameters.get_string_parameters(para_file, 'dst_nodata')
buffersize = parameters.get_string_parameters(para_file, 'buffer_size')
rectangle_ext = parameters.get_string_parameters(para_file,
'b_use_rectangle')
process_num = parameters.get_digit_parameters(para_file, 'process_num',
'int')
b_no_label_image = parameters.get_bool_parameters_None_if_absence(
para_file, 'b_no_label_image')
if os.path.isfile('sub_images_labels_list.txt'):
io_function.delete_file_or_dir('sub_images_labels_list.txt')
subImage_dir = parameters.get_string_parameters_None_if_absence(
para_file, 'input_train_dir')
subLabel_dir = parameters.get_string_parameters_None_if_absence(
para_file, 'input_label_dir')
# loop each training regions
for idx, area_ini in enumerate(multi_training_regions):
input_image_dir = parameters.get_directory_None_if_absence(
area_ini, 'input_image_dir')
# it is ok consider a file name as pattern and pass it the following functions to get file list
input_image_or_pattern = parameters.get_string_parameters(
area_ini, 'input_image_or_pattern')
b_sub_images_json = parameters.get_bool_parameters(
area_ini, 'b_sub_images_json')
if b_sub_images_json is True:
# copy sub-images, then covert json files to label images.
object_names = parameters.get_string_list_parameters(
para_file, 'object_names')
get_subImages_json.get_subimages_label_josn(
input_image_dir,
input_image_or_pattern,
subImage_dir,
subLabel_dir,
object_names,
b_no_label_image=b_no_label_image,
process_num=process_num)
pass
else:
all_train_shp = parameters.get_file_path_parameters_None_if_absence(
area_ini, 'training_polygons')
train_shp = parameters.get_string_parameters(
area_ini, 'training_polygons_sub')
# get subImage and subLabel for one training polygons
print(
'extract training data from image folder (%s) and polgyons (%s)'
% (input_image_dir, train_shp))
if b_no_label_image is True:
get_subImage_one_shp(get_subImage_script,
all_train_shp,
buffersize,
dstnodata,
rectangle_ext,
train_shp,
input_image_dir,
file_pattern=input_image_or_pattern,
process_num=process_num)
else:
get_subImage_subLabel_one_shp(
get_subImage_script,
all_train_shp,
buffersize,
dstnodata,
rectangle_ext,
train_shp,
input_image_dir,
file_pattern=input_image_or_pattern,
process_num=process_num)
# check black sub-images or most part of the sub-images is black (nodata)
new_sub_image_label_list = []
delete_sub_image_label_list = []
subImage_dir_delete = subImage_dir + '_delete'
subLabel_dir_delete = subLabel_dir + '_delete'
io_function.mkdir(subImage_dir_delete)
if b_no_label_image is None or b_no_label_image is False:
io_function.mkdir(subLabel_dir_delete)
get_valid_percent_entropy.plot_valid_entropy(subImage_dir)
with open('sub_images_labels_list.txt', 'r') as f_obj:
lines = f_obj.readlines()
for line in lines:
image_path, label_path = line.strip().split(':')
# valid_per = raster_io.get_valid_pixel_percentage(image_path)
valid_per, entropy = raster_io.get_valid_percent_shannon_entropy(
image_path) # base=10
if valid_per > 60 and entropy >= 0.5:
new_sub_image_label_list.append(line)
else:
delete_sub_image_label_list.append(line)
io_function.movefiletodir(image_path, subImage_dir_delete)
if os.path.isfile(label_path):
io_function.movefiletodir(label_path, subLabel_dir_delete)
if len(delete_sub_image_label_list) > 0:
with open('sub_images_labels_list.txt', 'w') as f_obj:
for line in new_sub_image_label_list:
f_obj.writelines(line)
# check weather they have the same subImage and subLabel
if b_no_label_image is None or b_no_label_image is False:
sub_image_list = io_function.get_file_list_by_pattern(
subImage_dir, '*.tif')
sub_label_list = io_function.get_file_list_by_pattern(
subLabel_dir, '*.tif')
if len(sub_image_list) != len(sub_label_list):
raise ValueError(
'the count of subImage (%d) and subLabel (%d) is different' %
(len(sub_image_list), len(sub_label_list)))
# save brief information of sub-images
height_list = []
width_list = []
band_count = 0
dtype = 'unknown'
for line in new_sub_image_label_list:
image_path, label_path = line.strip().split(':')
height, width, band_count, dtype = raster_io.get_height_width_bandnum_dtype(
image_path)
height_list.append(height)
width_list.append(width)
# save info to file, if it exists, it will be overwritten
img_count = len(new_sub_image_label_list)
with open('sub_images_patches_info.txt', 'w') as f_obj:
f_obj.writelines('information of sub-images: \n')
f_obj.writelines('number of sub-images : %d \n' % img_count)
f_obj.writelines('band count : %d \n' % band_count)
f_obj.writelines('data type : %s \n' % dtype)
f_obj.writelines('maximum width and height: %d, %d \n' %
(max(width_list), max(height_list)))
f_obj.writelines('minimum width and height: %d, %d \n' %
(min(width_list), min(height_list)))
f_obj.writelines(
'mean width and height: %.2f, %.2f \n\n' %
(sum(width_list) / img_count, sum(height_list) / img_count))
duration = time.time() - SECONDS
os.system(
'echo "$(date): time cost of getting sub images and labels: %.2f seconds">>time_cost.txt'
% duration)
def inf_remoteSensing_image(model, image_path=None):
'''
input a remote sensing image, then split to many small patches, inference each patch and merge than at last
:param model: trained model
:param image_path:
:return: False if unsuccessful.
'''
# split images
inf_image_dir = parameters.get_string_parameters(para_file,
'inf_images_dir')
patch_w = parameters.get_digit_parameters(para_file, "inf_patch_width",
None, 'int')
patch_h = parameters.get_digit_parameters(para_file, "inf_patch_height",
None, 'int')
overlay_x = parameters.get_digit_parameters(para_file,
"inf_pixel_overlay_x", None,
'int')
overlay_y = parameters.get_digit_parameters(para_file,
"inf_pixel_overlay_y", None,
'int')
inf_batch_size = parameters.get_digit_parameters(para_file,
"inf_batch_size", None,
'int')
global inf_list_file
if image_path is not None:
with open('inf_image_list.txt', 'w') as f_obj:
f_obj.writelines(image_path)
inf_list_file = 'inf_image_list.txt'
data_patches_2d = build_RS_data.make_dataset(inf_image_dir,
inf_list_file,
patch_w,
patch_h,
overlay_x,
overlay_y,
train=False)
if len(data_patches_2d) < 1:
return False
total_patch_count = 0
for img_idx, aImage_patches in enumerate(data_patches_2d):
patch_num = len(aImage_patches)
total_patch_count += patch_num
print('number of patches on Image %d: %d' % (img_idx, patch_num))
print('total number of patches: %d' % total_patch_count)
## inference multiple image patches at the same time
# for img_idx, aImage_patches in enumerate(data_patches_2d):
#
# print('start inference on Image %d' % img_idx)
#
# idx = 0 #index of all patches on this image
# patch_batches = build_RS_data.split_patches_into_batches(aImage_patches,inf_batch_size)
#
# for a_batch_of_patches in patch_batches:
#
# # Since it required a constant of batch size for the frozen graph, we copy (duplicate) the first patch
# org_patch_num = len(a_batch_of_patches)
# while len(a_batch_of_patches) < inf_batch_size:
# a_batch_of_patches.append(a_batch_of_patches[0])
#
# # read image data and stack at 0 dimension
# multi_image_data = []
# for img_patch in a_batch_of_patches:
# img_data = build_RS_data.read_patch(img_patch) # (nband, height,width)
# img_data = np.transpose(img_data, (1, 2, 0)) # keras and tf require (height,width,nband)
# multi_image_data.append(img_data)
# # multi_images = np.stack(multi_image_data, axis=0)
#
# # modify the BATCH_Size
# model.config.BATCH_SIZE = len(multi_image_data)
#
# # inference them
# results = model.detect(multi_image_data, verbose=0)
# # r = results[0]
# # visualize.display_instances(image_data, r['rois'], r['masks'], r['class_ids'],
# # ['BG', 'thawslump'], r['scores'], ax=get_ax())
#
# #save
# for num,(mrcc_r,img_patch) in enumerate(zip(results,a_batch_of_patches)):
#
# # ignore the duplicated ones
# if num >= org_patch_num:
# break
#
#
# # convert mask to a map of classification
# masks = mrcc_r['masks'] # shape: (height, width, num_classes?)
# height, width, nclass = masks.shape
#
# seg_map = np.zeros((height, width),dtype=np.uint8)
# for n_id in range(0,nclass):
# seg_map[ masks[:,:,n_id] == True ] = n_id + 1
#
# print('Save segmentation result of Image:%d patch:%4d, shape:(%d,%d)' %
# (img_idx, idx, seg_map.shape[0], seg_map.shape[1]))
#
# # short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29
# file_name = "I%d_%d" % (img_idx, idx)
#
# save_path = os.path.join(inf_output_dir, file_name + '.tif')
# if build_RS_data.save_patch_oneband_8bit(img_patch,seg_map.astype(np.uint8),save_path) is False:
# return False
#
# idx += 1
# inference image patches one by one
for img_idx, aImage_patches in enumerate(data_patches_2d):
print('start inference on Image %d' % img_idx)
for idx, img_patch in enumerate(aImage_patches):
# # test debug
# if not idx in [3359, 3360,3361,3476,3477,3478,3593,3594,3595]:
# continue
# if not idx in [4700, 4817, 4818, 4819, 4934, 4935, 4936, 5051, 5052, 5053]:
# continue
# if not idx in [2602]: # a false positive
# continue
img_data = build_RS_data.read_patch(
img_patch) # (nband, height,width)
# test
# save_path = "I%d_%d_org.tif" % (img_idx, idx)
# build_RS_data.save_patch(img_patch, img_data, save_path)
img_data = np.transpose(
img_data,
(1, 2, 0)) # keras and tf require (height,width,nband)
# inference them
results = model.detect([img_data], verbose=0)
mrcc_r = results[0]
# mrcc_r['scores']
# mrcc_r['rois']
masks = mrcc_r['masks'] # shape: (height, width, num_instance)
height, width, ncount = masks.shape
class_ids = mrcc_r['class_ids']
seg_map = np.zeros((height, width), dtype=np.uint8)
for inst in range(0, ncount): # instance one by one
seg_map[masks[:, :, inst] == True] = class_ids[inst]
print(
'Save segmentation result of Image:%d patch:%4d, shape:(%d,%d)'
% (img_idx, idx, seg_map.shape[0], seg_map.shape[1]))
# short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29
file_name = "I%d_%d" % (img_idx, idx)
save_path = os.path.join(inf_output_dir, file_name + '.tif')
if build_RS_data.save_patch_oneband_8bit(
img_patch, seg_map.astype(np.uint8), save_path) is False:
return False
def muti_inf_remoteSensing_image(model, image_path=None):
'''
use multiple scale (different patch size) for inference, then merge them using non_max_suppression
:param model: trained model
:param image_path:
:return: True if successful, False otherwise
'''
# get parameters
inf_image_dir = parameters.get_string_parameters(para_file,
'inf_images_dir')
muti_patch_w = parameters.get_string_parameters(para_file,
"muti_inf_patch_width")
muti_patch_h = parameters.get_string_parameters(para_file,
"muti_inf_patch_height")
muti_overlay_x = parameters.get_string_parameters(
para_file, "muti_inf_pixel_overlay_x")
muti_overlay_y = parameters.get_string_parameters(
para_file, "muti_inf_pixel_overlay_y")
final_keep_classes = parameters.get_string_parameters(
para_file, "final_keep_classes")
nms_iou_threshold = parameters.get_digit_parameters(
para_file, "nms_iou_threshold", None, 'float')
patch_w_list = [int(item) for item in muti_patch_w.split(',')]
patch_h_list = [int(item) for item in muti_patch_h.split(',')]
overlay_x_list = [int(item) for item in muti_overlay_x.split(',')]
overlay_y_list = [int(item) for item in muti_overlay_y.split(',')]
if final_keep_classes == '':
final_keep_classes = None
else:
final_keep_classes = [
int(item) for item in final_keep_classes.split(',')
]
# inference and save to json files
for patch_w, patch_h, overlay_x, overlay_y in zip(patch_w_list,
patch_h_list,
overlay_x_list,
overlay_y_list):
inf_rs_image_json(model, patch_w, patch_h, overlay_x, overlay_y,
inf_image_dir)
# load all boxes of images
with open(inf_list_file) as file_obj:
files_list = file_obj.readlines()
for img_idx, image_name in enumerate(files_list):
file_pattern = os.path.join(
inf_output_dir, 'I%d_patches_*_*_*' %
img_idx) # e.g., I0_patches_320_320_80_80
proc = subprocess.Popen('ls -d ' + file_pattern,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
profiles, err = proc.communicate()
json_folder_list = profiles.split()
if len(json_folder_list) < 1:
raise IOError('No folder containing json files in %s' %
inf_output_dir)
# bytes to str
if isinstance(json_folder_list[0], bytes):
json_folder_list = [item.decode() for item in json_folder_list]
print('loading json files of image :%d, in %s' %
(img_idx, ','.join(json_folder_list)))
# load all the boxes and scores
mrcnn_r_list = [
] # the results dict from rcnn, contains all the information
mask_files = [] # mask file for instance, each box has a mask file
boxes = [] # boxes of instance
class_ids = [] # class id of each box
patch_indices = [] # index of patch, on which contains the box
scores = [] # scores of boxes
json_files_list = [] # json files of patches
for json_folder in json_folder_list:
file_list = io_function.get_file_list_by_ext('.txt',
json_folder,
bsub_folder=False)
json_files_list.extend(file_list)
# load and convert coordinates, don't load mask images in this stage
patch_idx = 0
for json_file in json_files_list:
mrcnn_r = build_RS_data.load_instances_patch(
json_file,
bNMS=True,
bReadMaks=False,
final_classes=final_keep_classes)
mrcnn_r_list.append(
mrcnn_r) # this corresponds to json_files_list
if mrcnn_r is not None: # this will ignore the patches without instances, it is fine hlc 2018-11-25
mask_files.extend(mrcnn_r['masks'])
boxes.extend(mrcnn_r['rois'])
scores.extend(mrcnn_r['scores'])
class_ids.extend(mrcnn_r['class_ids'])
patch_indices.extend([patch_idx] * len(mrcnn_r['rois']))
patch_idx += 1
# Apply non-max suppression
keep_idxs = utils.non_max_suppression(np.array(boxes),
np.array(scores),
nms_iou_threshold)
# boxes_keep = [r for i, r in enumerate(boxes) if i in keep_ixs]
# convert kept patches to label images
for idx, keep_idx in enumerate(keep_idxs):
patch_idx = patch_indices[
keep_idx] # the index in original patches
# load mask (in the previous step, we did not load masks)
mask_file = mask_files[keep_idx]
patch_dir = os.path.dirname(json_files_list[patch_idx])
org_img_name = mrcnn_r_list[patch_idx]['org_img']
b_dict = mrcnn_r_list[patch_idx]['patch_boundary']
patch_boundary = (b_dict['xoff'], b_dict['yoff'],
b_dict['xsize'], b_dict['ysize'])
img_patch = build_RS_data.patchclass(
os.path.join(inf_image_dir, org_img_name), patch_boundary)
# the mask only contains one instances
# masks can overlap each others, but instances should not overlap each other
# non-instance pixels are zeros,which will be set as non-data when performing gdal_merge.pyg
mask = cv2.imread(os.path.join(patch_dir, mask_file),
cv2.IMREAD_UNCHANGED)
mask[mask == 255] = class_ids[
keep_idx] # when save mask, mask*255 for display
print('Save mask of instances:%d on Image:%d , shape:(%d,%d)' %
(idx, img_idx, mask.shape[0], mask.shape[1]))
# short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29
file_name = "I%d_%d" % (img_idx, idx)
save_path = os.path.join(inf_output_dir, file_name + '.tif')
if build_RS_data.save_patch_oneband_8bit(
img_patch, mask.astype(np.uint8), save_path) is False:
return False
return True
def train_evaluation_deeplab_separate(WORK_DIR, deeplab_dir, expr_name,
para_file, network_setting_ini, gpu_num):
'''
in "train_evaluation_deeplab", run training, stop, then evaluation, then traininng, make learning rate strange, and the results worse.
so in this function, we start two process, one for training, another for evaluation (run on CPU)
'''
# prepare training folder
EXP_FOLDER = expr_name
INIT_FOLDER = os.path.join(WORK_DIR, EXP_FOLDER, 'init_models')
TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train')
EVAL_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'eval')
VIS_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'vis')
EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export')
io_function.mkdir(INIT_FOLDER)
io_function.mkdir(TRAIN_LOGDIR)
io_function.mkdir(EVAL_LOGDIR)
io_function.mkdir(VIS_LOGDIR)
io_function.mkdir(EXPORT_DIR)
# prepare the tensorflow check point (pretrained model) for training
pre_trained_dir = parameters.get_directory_None_if_absence(
network_setting_ini, 'pre_trained_model_folder')
pre_trained_tar = parameters.get_string_parameters(network_setting_ini,
'TF_INIT_CKPT')
pre_trained_path = os.path.join(pre_trained_dir, pre_trained_tar)
if os.path.isfile(pre_trained_path) is False:
print('pre-trained model: %s not exist, try to download' %
pre_trained_path)
# try to download the file
pre_trained_url = parameters.get_string_parameters_None_if_absence(
network_setting_ini, 'pre_trained_model_url')
res = os.system('wget %s ' % pre_trained_url)
if res != 0:
sys.exit(1)
io_function.movefiletodir(pre_trained_tar, pre_trained_dir)
# unpack pre-trained model to INIT_FOLDER
os.chdir(INIT_FOLDER)
res = os.system('tar -xf %s' % pre_trained_path)
if res != 0:
raise IOError('failed to unpack %s' % pre_trained_path)
os.chdir(WORK_DIR)
dataset_dir = os.path.join(WORK_DIR, 'tfrecord')
batch_size = parameters.get_digit_parameters(network_setting_ini,
'batch_size', 'int')
# maximum iteration number
iteration_num = parameters.get_digit_parameters(network_setting_ini,
'iteration_num', 'int')
base_learning_rate = parameters.get_digit_parameters(
network_setting_ini, 'base_learning_rate', 'float')
train_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_output_stride', 'int')
train_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates1', 'int')
train_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates2', 'int')
train_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates3', 'int')
inf_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_output_stride', 'int')
inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates1', 'int')
inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates2', 'int')
inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates3', 'int')
# depth_multiplier default is 1.0.
depth_multiplier = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'depth_multiplier', 'float')
decoder_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'decoder_output_stride', 'int')
aspp_convs_filters = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'aspp_convs_filters', 'int')
train_script = os.path.join(deeplab_dir, 'train.py')
train_split = os.path.splitext(
parameters.get_string_parameters(para_file,
'training_sample_list_txt'))[0]
model_variant = parameters.get_string_parameters(network_setting_ini,
'model_variant')
checkpoint = parameters.get_string_parameters(network_setting_ini,
'tf_initial_checkpoint')
init_checkpoint_files = io_function.get_file_list_by_pattern(
INIT_FOLDER, checkpoint + '*')
if len(init_checkpoint_files) < 1:
raise IOError('No initial checkpoint in %s with pattern: %s' %
(INIT_FOLDER, checkpoint))
init_checkpoint = os.path.join(INIT_FOLDER, checkpoint)
b_early_stopping = parameters.get_bool_parameters(para_file,
'b_early_stopping')
b_initialize_last_layer = parameters.get_bool_parameters(
para_file, 'b_initialize_last_layer')
dataset = parameters.get_string_parameters(para_file, 'dataset_name')
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
assert num_classes_noBG != None
if b_initialize_last_layer is True:
if pre_trained_tar in pre_trained_tar_21_classes:
print(
'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21'
% pre_trained_tar)
num_classes_noBG = 20
if pre_trained_tar in pre_trained_tar_19_classes:
print(
'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19'
% pre_trained_tar)
num_classes_noBG = 18
num_of_classes = num_classes_noBG + 1
image_crop_size = parameters.get_string_list_parameters(
para_file, 'image_crop_size')
if len(image_crop_size) != 2 and image_crop_size[0].isdigit(
) and image_crop_size[1].isdigit():
raise ValueError('image_crop_size should be height,width')
crop_size_str = ','.join(image_crop_size)
# validation interval (epoch), do
# validation_interval = parameters.get_digit_parameters_None_if_absence(para_file,'validation_interval','int')
train_count, val_count = get_train_val_sample_count(WORK_DIR, para_file)
iter_per_epoch = math.ceil(train_count / batch_size)
total_epoches = math.ceil(iteration_num / iter_per_epoch)
already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR)
if already_trained_iteration >= iteration_num:
basic.outputlogMessage('Training already run %d iterations, skip' %
already_trained_iteration)
return True
save_interval_secs = 1200 # default is 1200 second for saving model
save_summaries_secs = 600 # default is 600 second for saving summaries
eval_interval_secs = save_interval_secs # default is 300 second for running evaluation, if no new saved model, no need to run evaluation?
train_process = Process(
target=train_deeplab,
args=(train_script, dataset, train_split, num_of_classes,
base_learning_rate, model_variant, init_checkpoint, TRAIN_LOGDIR,
dataset_dir, gpu_num, train_atrous_rates1, train_atrous_rates2,
train_atrous_rates3, train_output_stride, crop_size_str,
batch_size, iteration_num, depth_multiplier,
decoder_output_stride, aspp_convs_filters,
b_initialize_last_layer))
train_process.start()
time.sleep(60) # wait
if train_process.exitcode is not None and train_process.exitcode != 0:
sys.exit(1)
# eval_process.start()
# time.sleep(10) # wait
# if eval_process.exitcode is not None and eval_process.exitcode != 0:
# sys.exit(1)
while True:
# only run evaluation when there is new trained model
already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR)
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
basic.outputlogMessage(
'Already trained iteration: %d, latest evaluation at %d step' %
(already_trained_iteration, miou_dict['step'][-1]))
if already_trained_iteration > miou_dict['step'][-1]:
# run evaluation and wait until it finished
gpuid = "" # set gpuid to empty string, making evaluation run on CPU
evl_script = os.path.join(deeplab_dir, 'eval.py')
evl_split = os.path.splitext(
parameters.get_string_parameters(
para_file, 'validation_sample_list_txt'))[0]
# max_eva_number = -1 # run as many evaluation as possible, --eval_interval_secs (default is 300 seconds)
max_eva_number = 1 # only run once inside the while loop, use while loop to control multiple evaluation
eval_process = Process(
target=evaluation_deeplab,
args=(evl_script, dataset, evl_split, num_of_classes,
model_variant, inf_atrous_rates1, inf_atrous_rates2,
inf_atrous_rates3, inf_output_stride, TRAIN_LOGDIR,
EVAL_LOGDIR, dataset_dir, crop_size_str, max_eva_number,
depth_multiplier, decoder_output_stride,
aspp_convs_filters, gpuid, eval_interval_secs))
eval_process.start(
) # put Process inside while loop to avoid error: AssertionError: cannot start a process twice
while eval_process.is_alive():
time.sleep(5)
# check if need early stopping
if b_early_stopping:
print(datetime.now(), 'check early stopping')
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
if 'overall' in miou_dict.keys() and len(
miou_dict['overall']) >= 5:
# if the last five miou did not improve, then stop training
if np.all(np.diff(miou_dict['overall'][-5:]) < 0.005
): # 0.0001 (%0.01) # 0.5 %
basic.outputlogMessage(
'early stopping: stop training because overall miou did not improved in the last five evaluation'
)
output_early_stopping_message(TRAIN_LOGDIR)
# train_process.kill() # this one seems not working
# subprocess pid different from ps output
# https://stackoverflow.com/questions/4444141/subprocess-pid-different-from-ps-output
# os.system('kill ' + str(train_process.pid)) # still not working. train_process.pid is not the one output by ps -aux
# train_process.terminate() # Note that descendant processes of the process will not be terminated
# train_process.join() # Wait until child process terminates
with open('train_py_pid.txt', 'r') as f_obj:
lines = f_obj.readlines()
train_pid = int(lines[0].strip())
os.system('kill ' + str(train_pid))
basic.outputlogMessage(
'kill training processing with id: %d' % train_pid)
break # this breaks the while loop, making that it may not evaluate on some new saved model.
# if the evaluation step is less than saved model iteration, run another iteration again immediately
already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR)
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
if already_trained_iteration > miou_dict['step'][-1]:
continue
# if finished training
if train_process.is_alive() is False:
break
# # if eval_process exit, then quit training as well
# if eval_process.is_alive() is False and train_process.is_alive():
# train_process.kill()
# break
time.sleep(eval_interval_secs) # wait for next evaluation
# save loss value to disk
get_loss_learning_rate_list(TRAIN_LOGDIR)
# get miou again
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
# eval_process did not exit as expected, kill it again.
# os.system('kill ' + str(eval_process.pid))
# get iou and backup
iou_path = os.path.join(EVAL_LOGDIR, 'miou.txt')
loss_path = os.path.join(TRAIN_LOGDIR, 'loss_learning_rate.txt')
patch_info = os.path.join(WORK_DIR, 'sub_images_patches_info.txt')
# backup miou and training_loss & learning rate
test_id = os.path.basename(WORK_DIR) + '_' + expr_name
backup_dir = os.path.join(WORK_DIR, 'result_backup')
if os.path.isdir(backup_dir) is False:
io_function.mkdir(backup_dir)
new_iou_name = os.path.join(backup_dir,
test_id + '_' + os.path.basename(iou_path))
io_function.copy_file_to_dst(iou_path, new_iou_name, overwrite=True)
loss_new_name = os.path.join(backup_dir,
test_id + '_' + os.path.basename(loss_path))
io_function.copy_file_to_dst(loss_path, loss_new_name, overwrite=True)
new_patch_info = os.path.join(backup_dir,
test_id + '_' + os.path.basename(patch_info))
io_function.copy_file_to_dst(patch_info, new_patch_info, overwrite=True)
# plot mIOU, loss, and learnint rate curves, and backup
miou_curve_path = plot_miou_loss_curve.plot_miou_loss_main(
iou_path,
train_count=train_count,
val_count=val_count,
batch_size=batch_size)
loss_curve_path = plot_miou_loss_curve.plot_miou_loss_main(
loss_path,
train_count=train_count,
val_count=val_count,
batch_size=batch_size)
miou_curve_bakname = os.path.join(
backup_dir, test_id + '_' + os.path.basename(miou_curve_path))
io_function.copy_file_to_dst(miou_curve_path,
miou_curve_bakname,
overwrite=True)
loss_curve_bakname = os.path.join(
backup_dir, test_id + '_' + os.path.basename(loss_curve_path))
io_function.copy_file_to_dst(loss_curve_path,
loss_curve_bakname,
overwrite=True)
def inf_remoteSensing_image(model,image_path=None):
'''
input a remote sensing image, then split to many small patches, inference each patch and merge than at last
:param model: trained model
:param image_path:
:return: False if unsuccessful.
'''
# split images
inf_image_dir=parameters.get_string_parameters(FLAGS.inf_para_file,'inf_images_dir')
patch_w = parameters.get_digit_parameters(FLAGS.inf_para_file, "inf_patch_width", None, 'int')
patch_h = parameters.get_digit_parameters(FLAGS.inf_para_file, "inf_patch_height", None, 'int')
overlay_x = parameters.get_digit_parameters(FLAGS.inf_para_file, "inf_pixel_overlay_x", None, 'int')
overlay_y = parameters.get_digit_parameters(FLAGS.inf_para_file, "inf_pixel_overlay_y", None, 'int')
if image_path is not None:
with open('saved_inf_list.txt','w') as f_obj:
f_obj.writelines(image_path)
FLAGS.inf_list_file = 'saved_inf_list.txt'
data_patches_2d = build_RS_data.make_dataset(inf_image_dir,FLAGS.inf_list_file,
patch_w,patch_h,overlay_x,overlay_y,train=False)
if len(data_patches_2d)< 1:
return False
total_patch_count = 0
for img_idx, aImage_patches in enumerate(data_patches_2d):
patch_num = len(aImage_patches)
total_patch_count += patch_num
print('number of patches on Image %d: %d' % (img_idx,patch_num))
print('total number of patches: %d'%total_patch_count)
for img_idx, aImage_patches in enumerate(data_patches_2d):
print('start inference on Image %d' % img_idx)
# ## parallel inference patches
## but it turns out not work due to the Pickle.PicklingError
# # use multiple thread
# num_cores = multiprocessing.cpu_count()
# print('number of thread %d' % num_cores)
# # theadPool = mp.Pool(num_cores) # multi threads, can not utilize all the CPUs? not sure hlc 2018-4-19
# theadPool = Pool(num_cores) # multi processes
#
# parameters_list = [(img_idx,idx,img_patch.org_img,img_patch.boundary,model) for (idx,img_patch) in enumerate(aImage_patches)]
# results = theadPool.map(inference_one_patch, parameters_list)
# print('result_list',results )
# inference patches batch by batch, but it turns out that the frozen graph only accept one patch each time
# Oct 30,2018
# split to many batches (groups)
idx = 0 #index of all patches on this image
patch_batches = build_RS_data.split_patches_into_batches(aImage_patches,FLAGS.inf_batch_size)
for a_batch_of_patches in patch_batches:
# Since it required a constant of batch size for the frozen graph, we copy (duplicate) the first patch
org_patch_num = len(a_batch_of_patches)
while len(a_batch_of_patches) < FLAGS.inf_batch_size:
a_batch_of_patches.append(a_batch_of_patches[0])
# read image data and stack at 0 dimension
multi_image_data = []
for img_patch in a_batch_of_patches:
img_data = build_RS_data.read_patch(img_patch)
## ignore image patch are all black or white
if np.std(img_data) < 0.0001:
print('Image:%d patch:%4d is black or white, ignore' %(img_idx, idx))
idx += 1
continue
multi_image_data.append(img_data)
# ignore image patch are all black or white
if len(multi_image_data) < 1:
continue
multi_images = np.stack(multi_image_data, axis=0)
# inference them
a_batch_seg_map = model.run_rsImg_multi_patches(multi_images)
#save
for num,(seg_map,img_patch) in enumerate(zip(a_batch_seg_map,a_batch_of_patches)):
# ignore the duplicated ones
if num >= org_patch_num:
break
print('Save segmentation result of Image:%d patch:%4d, shape:(%d,%d)' %
(img_idx, idx, seg_map.shape[0], seg_map.shape[1]))
# short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29
file_name = "I%d_%d" % (img_idx, idx)
save_path = os.path.join(FLAGS.inf_output_dir, file_name + '.tif')
# if os.path.isfile(save_path):
# print('already exist, skip')
# idx += 1
# continue
if build_RS_data.save_patch_oneband_8bit(img_patch,seg_map.astype(np.uint8),save_path) is False:
return False
idx += 1
parser.add_argument('--cuda', action='store_true', help='enables cuda')
parser.add_argument('--resume',
default='',
type=str,
metavar='PATH',
help='path to latest checkpoint (default: none)')
parser.add_argument('--useBN',
action='store_true',
help='enalbes batch normalization')
args = parser.parse_args()
print(args)
parameters.set_saved_parafile_path(args.para)
patch_w = parameters.get_digit_parameters("", "inf_patch_width", None, 'int')
patch_h = parameters.get_digit_parameters("", "inf_patch_height", None, 'int')
overlay = parameters.get_digit_parameters("", "inf_pixel_overlay", None, 'int')
dataset = RemoteSensingImg(args.dataroot,
args.list,
patch_w,
patch_h,
overlay,
train=False)
train_loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batchSize,
num_workers=args.workers,
shuffle=False)
model = Net(args.useBN)
def main(options, args):
if options.out_dir is None:
out_dir = "extract_dir"
else:
out_dir = options.out_dir
if os.path.isdir(out_dir) is False:
os.makedirs(out_dir)
img_dir = options.img_dir
extension = options.extension
is_groud_true = options.ground_truth
# print(options.para_file)
augmentation = parameters.get_string_parameters(options.para_file,
'data_augmentation')
augmentation = [item.lower().strip() for item in augmentation.split(',')]
augmentation = list(filter(
None,
augmentation)) # remove empty str (in case only have empty string)
if len(augmentation) < 1:
print('No input augmentation requirement (e.g. flip)')
return True
# print(augmentation)
# sys.exit(1)
# number of classes
num_classes_noBG = parameters.get_digit_parameters(options.para_file,
'NUM_CLASSES_noBG',
None, 'int')
global num_classes
num_classes = num_classes_noBG + 1
#
ignore_classes = parameters.get_string_parameters(
options.para_file, 'data_aug_ignore_classes')
ignore_classes = [item.strip() for item in ignore_classes.split(',')]
ignore_classes = list(filter(
None,
ignore_classes)) # remove empty str (in case only have empty string)
img_list_txt = args[0]
if os.path.isfile(img_list_txt) is False:
print("Error, File %s not exist" % img_list_txt)
return False
f_obj = open(img_list_txt)
index = 1
files_list = f_obj.readlines()
for line in files_list:
# ignore_classes
if len(ignore_classes) > 0:
found_class = [
line.find(ignore_class) >= 0 for ignore_class in ignore_classes
]
if True in found_class:
continue
file_path = line.strip()
file_path = os.path.join(img_dir, file_path + extension)
print("Augmentation of image (%d / %d)" % (index, len(files_list)))
if image_augment(
file_path, out_dir, is_groud_true,
augment=augmentation) is False:
print('Error, Failed in image augmentation')
return False
index += 1
f_obj.close()
if index < len(files_list):
basic.outputlogMessage('Some of the images belong to %s are ingnore' %
','.join(ignore_classes))
# update img_list_txt
new_files = io_function.get_file_list_by_ext(extension,
'.',
bsub_folder=False)
new_files_noext = [
os.path.splitext(os.path.basename(item))[0] + '\n'
for item in new_files
]
basic.outputlogMessage('save new file list to %s' % options.save_list)
with open(options.save_list, 'w') as f_obj:
f_obj.writelines(new_files_noext)
def image_label_to_yolo_format(para_file):
print("Image labels (semantic segmentation) to YOLO object detection")
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
img_ext = parameters.get_string_parameters_None_if_absence(
para_file, 'split_image_format')
proc_num = parameters.get_digit_parameters(para_file, 'process_num', 'int')
SECONDS = time.time()
# get image and label path
image_list = []
label_list = []
with open(os.path.join('list', 'trainval.txt'), 'r') as f_obj:
lines = [item.strip() for item in f_obj.readlines()]
for line in lines:
image_list.append(os.path.join('split_images', line + img_ext))
label_list.append(os.path.join('split_labels', line + img_ext))
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
b_ignore_edge_objects = parameters.get_bool_parameters_None_if_absence(
para_file, 'b_ignore_edge_objects')
if b_ignore_edge_objects is None:
b_ignore_edge_objects = False
# get boxes
total_count = len(image_list)
for idx, (img, label) in enumerate(zip(image_list, label_list)):
get_yolo_boxes_one_img(idx,
total_count,
img,
label,
num_classes_noBG,
rm_edge_obj=b_ignore_edge_objects)
# write obj.data file
train_sample_txt = parameters.get_string_parameters(
para_file, 'training_sample_list_txt')
val_sample_txt = parameters.get_string_parameters(
para_file, 'validation_sample_list_txt')
train_img_list = get_image_list('list', train_sample_txt, 'split_images',
img_ext)
val_img_list = get_image_list('list', val_sample_txt, 'split_images',
img_ext)
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
object_names = parameters.get_string_list_parameters(
para_file, 'object_names')
io_function.mkdir('data')
io_function.mkdir(expr_name)
with open(os.path.join('data', 'obj.data'), 'w') as f_obj:
f_obj.writelines('classes = %d' % num_classes_noBG + '\n')
train_txt = os.path.join('data', 'train.txt')
io_function.save_list_to_txt(train_txt, train_img_list)
f_obj.writelines('train = %s' % train_txt + '\n')
val_txt = os.path.join('data', 'val.txt')
io_function.save_list_to_txt(val_txt, val_img_list)
f_obj.writelines('valid = %s' % val_txt + '\n')
obj_name_txt = os.path.join('data', 'obj.names')
io_function.save_list_to_txt(obj_name_txt, object_names)
f_obj.writelines('names = %s' % obj_name_txt + '\n')
f_obj.writelines('backup = %s' % expr_name + '\n')
duration = time.time() - SECONDS
os.system(
'echo "$(date): time cost of converting to yolo format: %.2f seconds">>time_cost.txt'
% duration)
pass
print("Model: ", args.model)
# print("Dataset: ", args.dataset)
print("the parameter file: ", args.para_file)
# print("Logs: ", args.logs)
# print("Auto Download: ", args.download)
para_file = args.para_file
inf_list_file = args.inf_list_file
inf_output_dir = args.inf_output_dir
expr_name = parameters.get_string_parameters(args.para_file, 'expr_name')
if os.path.isdir(expr_name) is False:
os.mkdir(expr_name)
NO_DATA = parameters.get_digit_parameters(args.para_file, 'dst_nodata',
None, 'int')
num_class_noBG = parameters.get_digit_parameters(args.para_file,
'NUM_CLASSES_noBG', None,
'int')
# modify default setting according to different machine
gpu_count = parameters.get_digit_parameters(args.para_file, 'gpu_count',
None, 'int')
images_per_gpu = parameters.get_digit_parameters(args.para_file,
'images_per_gpu', None,
'int')
PlanetConfig.GPU_COUNT = gpu_count
PlanetConfig.IMAGES_PER_GPU = images_per_gpu
# Number of training and validation steps per epoch, same as nucleus sample
# https://github.com/matterport/Mask_RCNN/issues/1092
def __init__(self, args):
self.args = args
# Define Saver
self.saver = Saver(args)
self.saver.save_experiment_config()
# Define Tensorboard Summary
self.summary = TensorboardSummary(self.saver.experiment_dir)
self.writer = self.summary.create_summary()
# Define Dataloader
kwargs = {'num_workers': args.workers, 'pin_memory': True}
parameters.set_saved_parafile_path(args.para)
patch_w = parameters.get_digit_parameters("", "train_patch_width", None, 'int')
patch_h = parameters.get_digit_parameters("", "train_patch_height", None, 'int')
overlay_x = parameters.get_digit_parameters("", "train_pixel_overlay_x", None, 'int')
overlay_y = parameters.get_digit_parameters("", "train_pixel_overlay_y", None, 'int')
crop_height = parameters.get_digit_parameters("", "crop_height", None, 'int')
crop_width = parameters.get_digit_parameters("", "crop_width", None, 'int')
dataset = RemoteSensingImg(args.dataroot, args.list, patch_w, patch_h, overlay_x, overlay_y)
#train_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size,
# num_workers=args.workers, shuffle=True)
train_length = int(len(dataset) * 0.9)
validation_length = len(dataset) - train_length
#print ("totol data len is %d , train_length is %d"%(len(train_loader),train_length))
[self.train_dataset, self.val_dataset] = torch.utils.data.random_split(dataset, (train_length, validation_length))
print("len of train dataset is %d and val dataset is %d and total datalen is %d"%(len(self.train_dataset),len(self.val_dataset),len(dataset)))
self.train_loader=torch.utils.data.DataLoader(self.train_dataset, batch_size=args.batch_size,num_workers=args.workers, shuffle=True,drop_last=True)
self.val_loader=torch.utils.data.DataLoader(self.val_dataset, batch_size=args.batch_size,num_workers=args.workers, shuffle=True,drop_last=True)
print("len of train loader is %d and val loader is %d"%(len(self.train_loader),len(self.val_loader)))
#self.train_loader, self.val_loader, self.test_loader, self.nclass = make_data_loader(args, **kwargs)
# Define network
model = DeepLab(num_classes=1,
backbone=args.backbone,
output_stride=args.out_stride,
sync_bn=args.sync_bn,
freeze_bn=args.freeze_bn)
train_params = [{'params': model.get_1x_lr_params(), 'lr': args.lr},
{'params': model.get_10x_lr_params(), 'lr': args.lr * 10}]
# Define Optimizer
optimizer = torch.optim.SGD(train_params, momentum=args.momentum,
weight_decay=args.weight_decay, nesterov=args.nesterov)
# whether to use class balanced weights
# if args.use_balanced_weights:
# classes_weights_path = os.path.join(Path.db_root_dir(args.dataset), args.dataset+'_classes_weights.npy')
# if os.path.isfile(classes_weights_path):
# weight = np.load(classes_weights_path)
# else:
# weight = calculate_weigths_labels(args.dataset, self.train_loader, self.nclass)
# weight = torch.from_numpy(weight.astype(np.float32))
# else:
# weight = None
# Define Criterion
#self.criterion = SegmentationLosses(weight=weight, cuda=args.cuda).build_loss(mode=args.loss_type)
self.criterion=nn.BCELoss()
if args.cuda:
self.criterion=self.criterion.cuda()
self.model, self.optimizer = model, optimizer
# Define Evaluator
self.evaluator = Evaluator(2)
# Define lr scheduler
print("lenght of train_loader is %d"%(len(self.train_loader)))
self.scheduler = LR_Scheduler(args.lr_scheduler, args.lr,
args.epochs, len(self.train_loader))
# Using cuda
if args.cuda:
#self.model = torch.nn.DataParallel(self.model, device_ids=self.args.gpu_ids)
self.model = torch.nn.DataParallel(self.model)
patch_replication_callback(self.model)
self.model = self.model.cuda()
# Resuming checkpoint
self.best_pred = 0.0
if args.resume is not None:
if not os.path.isfile(args.resume):
raise RuntimeError("=> no checkpoint found at '{}'" .format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
if args.cuda:
self.model.module.load_state_dict(checkpoint['state_dict'])
else:
self.model.load_state_dict(checkpoint['state_dict'])
if not args.ft:
self.optimizer.load_state_dict(checkpoint['optimizer'])
self.best_pred = checkpoint['best_pred']
print("=> loaded checkpoint '{}' (epoch {}) with best mIoU {}"
.format(args.resume, checkpoint['epoch'], checkpoint['best_pred']))
# Clear start epoch if fine-tuning
if args.ft:
args.start_epoch = 0
self.best_pred=0
def evaluation_result(para_file, result_shp, val_shp, evaluation_txt=None):
"""
evaluate the result based on IoU
:param result_shp: result shape file contains detected polygons
:param val_shp: shape file contains validation polygons
:return: True is successful, False otherwise
"""
basic.outputlogMessage("evaluation result")
IoUs = vector_features.calculate_IoU_scores(result_shp, val_shp)
if IoUs is False:
return False
#save IoU to result shapefile
operation_obj = shape_opeation()
operation_obj.add_one_field_records_to_shapefile(result_shp, IoUs, 'IoU')
iou_threshold = parameters.get_digit_parameters(para_file, 'IOU_threshold',
'float')
true_pos_count = 0
false_pos_count = 0
val_polygon_count = operation_obj.get_shapes_count(val_shp)
# calculate precision, recall, F1 score
for iou in IoUs:
if iou > iou_threshold:
true_pos_count += 1
else:
false_pos_count += 1
false_neg_count = val_polygon_count - true_pos_count
if false_neg_count < 0:
basic.outputlogMessage(
'warning, false negative count is smaller than 0, recall can not be trusted'
)
precision = float(true_pos_count) / (float(true_pos_count) +
float(false_pos_count))
recall = float(true_pos_count) / (float(true_pos_count) +
float(false_neg_count))
if (true_pos_count > 0):
F1score = 2.0 * precision * recall / (precision + recall)
else:
F1score = 0
#output evaluation reslult
if evaluation_txt is None:
evaluation_txt = "evaluation_report.txt"
f_obj = open(evaluation_txt, 'w')
f_obj.writelines('true_pos_count: %d\n' % true_pos_count)
f_obj.writelines('false_pos_count: %d\n' % false_pos_count)
f_obj.writelines('false_neg_count: %d\n' % false_neg_count)
f_obj.writelines('precision: %.6f\n' % precision)
f_obj.writelines('recall: %.6f\n' % recall)
f_obj.writelines('F1score: %.6f\n' % F1score)
f_obj.close()
##########################################################################################
## another method for calculating false_neg_count base on IoU value
# calculate the IoU for validation polygons (ground truths)
IoUs = vector_features.calculate_IoU_scores(val_shp, result_shp)
if IoUs is False:
return False
# if the IoU of a validation polygon smaller than threshold, then it's false negative
false_neg_count = 0
idx_of_false_neg = []
for idx, iou in enumerate(IoUs):
if iou < iou_threshold:
false_neg_count += 1
idx_of_false_neg.append(idx + 1) # index start from 1
precision = float(true_pos_count) / (float(true_pos_count) +
float(false_pos_count))
recall = float(true_pos_count) / (float(true_pos_count) +
float(false_neg_count))
if (true_pos_count > 0):
F1score = 2.0 * precision * recall / (precision + recall)
else:
F1score = 0
# output evaluation reslult
# evaluation_txt = "evaluation_report.txt"
f_obj = open(evaluation_txt, 'a') # add to "evaluation_report.txt"
f_obj.writelines('\n\n** Count false negative by IoU**\n')
f_obj.writelines('true_pos_count: %d\n' % true_pos_count)
f_obj.writelines('false_pos_count: %d\n' % false_pos_count)
f_obj.writelines('false_neg_count_byIoU: %d\n' % false_neg_count)
f_obj.writelines('precision: %.6f\n' % precision)
f_obj.writelines('recall: %.6f\n' % recall)
f_obj.writelines('F1score: %.6f\n' % F1score)
# output the index of false negative
f_obj.writelines('\nindex (start from 1) of false negatives: %s\n' %
','.join([str(item) for item in idx_of_false_neg]))
f_obj.close()
pass
def train_evaluation_deeplab(WORK_DIR, deeplab_dir, expr_name, para_file,
network_setting_ini, gpu_num):
# prepare training folder
EXP_FOLDER = expr_name
INIT_FOLDER = os.path.join(WORK_DIR, EXP_FOLDER, 'init_models')
TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train')
EVAL_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'eval')
VIS_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'vis')
EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export')
io_function.mkdir(INIT_FOLDER)
io_function.mkdir(TRAIN_LOGDIR)
io_function.mkdir(EVAL_LOGDIR)
io_function.mkdir(VIS_LOGDIR)
io_function.mkdir(EXPORT_DIR)
# prepare the tensorflow check point (pretrained model) for training
pre_trained_dir = parameters.get_directory_None_if_absence(
network_setting_ini, 'pre_trained_model_folder')
pre_trained_tar = parameters.get_string_parameters(network_setting_ini,
'TF_INIT_CKPT')
pre_trained_path = os.path.join(pre_trained_dir, pre_trained_tar)
if os.path.isfile(pre_trained_path) is False:
print('pre-trained model: %s not exist, try to download' %
pre_trained_path)
# try to download the file
pre_trained_url = parameters.get_string_parameters_None_if_absence(
network_setting_ini, 'pre_trained_model_url')
res = os.system('wget %s ' % pre_trained_url)
if res != 0:
sys.exit(1)
io_function.movefiletodir(pre_trained_tar, pre_trained_dir)
# unpack pre-trained model to INIT_FOLDER
os.chdir(INIT_FOLDER)
res = os.system('tar -xf %s' % pre_trained_path)
if res != 0:
raise IOError('failed to unpack %s' % pre_trained_path)
os.chdir(WORK_DIR)
dataset_dir = os.path.join(WORK_DIR, 'tfrecord')
batch_size = parameters.get_digit_parameters(network_setting_ini,
'batch_size', 'int')
# maximum iteration number
iteration_num = parameters.get_digit_parameters(network_setting_ini,
'iteration_num', 'int')
base_learning_rate = parameters.get_digit_parameters(
network_setting_ini, 'base_learning_rate', 'float')
train_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_output_stride', 'int')
train_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates1', 'int')
train_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates2', 'int')
train_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates3', 'int')
inf_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_output_stride', 'int')
inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates1', 'int')
inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates2', 'int')
inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates3', 'int')
# depth_multiplier default is 1.0.
depth_multiplier = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'depth_multiplier', 'float')
decoder_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'decoder_output_stride', 'int')
aspp_convs_filters = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'aspp_convs_filters', 'int')
train_script = os.path.join(deeplab_dir, 'train.py')
train_split = os.path.splitext(
parameters.get_string_parameters(para_file,
'training_sample_list_txt'))[0]
model_variant = parameters.get_string_parameters(network_setting_ini,
'model_variant')
checkpoint = parameters.get_string_parameters(network_setting_ini,
'tf_initial_checkpoint')
init_checkpoint_files = io_function.get_file_list_by_pattern(
INIT_FOLDER, checkpoint + '*')
if len(init_checkpoint_files) < 1:
raise IOError('No initial checkpoint in %s with pattern: %s' %
(INIT_FOLDER, checkpoint))
init_checkpoint = os.path.join(INIT_FOLDER, checkpoint)
b_early_stopping = parameters.get_bool_parameters(para_file,
'b_early_stopping')
b_initialize_last_layer = parameters.get_bool_parameters(
para_file, 'b_initialize_last_layer')
dataset = parameters.get_string_parameters(para_file, 'dataset_name')
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
assert num_classes_noBG != None
if b_initialize_last_layer is True:
if pre_trained_tar in pre_trained_tar_21_classes:
print(
'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21'
% pre_trained_tar)
num_classes_noBG = 20
if pre_trained_tar in pre_trained_tar_19_classes:
print(
'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19'
% pre_trained_tar)
num_classes_noBG = 18
num_of_classes = num_classes_noBG + 1
image_crop_size = parameters.get_string_list_parameters(
para_file, 'image_crop_size')
if len(image_crop_size) != 2 and image_crop_size[0].isdigit(
) and image_crop_size[1].isdigit():
raise ValueError('image_crop_size should be height,width')
crop_size_str = ','.join(image_crop_size)
evl_script = os.path.join(deeplab_dir, 'eval.py')
evl_split = os.path.splitext(
parameters.get_string_parameters(para_file,
'validation_sample_list_txt'))[0]
max_eva_number = 1
# validation interval (epoch)
validation_interval = parameters.get_digit_parameters_None_if_absence(
para_file, 'validation_interval', 'int')
train_count, val_count = get_train_val_sample_count(WORK_DIR, para_file)
iter_per_epoch = math.ceil(train_count / batch_size)
total_epoches = math.ceil(iteration_num / iter_per_epoch)
already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR)
if already_trained_iteration >= iteration_num:
basic.outputlogMessage('Training already run %d iterations, skip' %
already_trained_iteration)
return True
if validation_interval is None:
basic.outputlogMessage(
'No input validation_interval, so training to %d, then evaluating in the end'
% iteration_num)
# run training
train_deeplab(train_script, dataset, train_split, num_of_classes,
base_learning_rate, model_variant, init_checkpoint,
TRAIN_LOGDIR, dataset_dir, gpu_num, train_atrous_rates1,
train_atrous_rates2, train_atrous_rates3,
train_output_stride, crop_size_str, batch_size,
iteration_num, depth_multiplier, decoder_output_stride,
aspp_convs_filters, b_initialize_last_layer)
# run evaluation
evaluation_deeplab(evl_script, dataset, evl_split, num_of_classes,
model_variant, inf_atrous_rates1, inf_atrous_rates2,
inf_atrous_rates3, inf_output_stride, TRAIN_LOGDIR,
EVAL_LOGDIR, dataset_dir, crop_size_str,
max_eva_number, depth_multiplier,
decoder_output_stride, aspp_convs_filters)
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
get_loss_learning_rate_list(TRAIN_LOGDIR)
else:
basic.outputlogMessage(
'training to the maximum iteration of %d, and evaluating very %d epoch(es)'
% (iteration_num, validation_interval))
for epoch in range(validation_interval,
total_epoches + validation_interval,
validation_interval):
to_iter_num = min(epoch * iter_per_epoch, iteration_num)
if to_iter_num <= already_trained_iteration:
continue
basic.outputlogMessage(
'training and evaluating to %d epoches (to iteration: %d)' %
(epoch, to_iter_num))
# run training
train_deeplab(train_script, dataset, train_split, num_of_classes,
base_learning_rate, model_variant, init_checkpoint,
TRAIN_LOGDIR, dataset_dir, gpu_num,
train_atrous_rates1, train_atrous_rates2,
train_atrous_rates3, train_output_stride,
crop_size_str, batch_size, to_iter_num,
depth_multiplier, decoder_output_stride,
aspp_convs_filters, b_initialize_last_layer)
# run evaluation
evaluation_deeplab(evl_script, dataset, evl_split, num_of_classes,
model_variant, inf_atrous_rates1,
inf_atrous_rates2, inf_atrous_rates3,
inf_output_stride, TRAIN_LOGDIR, EVAL_LOGDIR,
dataset_dir, crop_size_str, max_eva_number,
depth_multiplier, decoder_output_stride,
aspp_convs_filters)
# get miou
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
# save loss value to disk
get_loss_learning_rate_list(TRAIN_LOGDIR)
# check if need to early stopping
if b_early_stopping:
if len(miou_dict['overall']) >= 5:
# if the last five miou did not improve, then stop training
if np.all(np.diff(miou_dict['overall'][-5:]) < 0.005
): # 0.0001 (%0.01) # 0.5 %
basic.outputlogMessage(
'early stopping: stop training because overall miou did not improved in the last five evaluation'
)
output_early_stopping_message(TRAIN_LOGDIR)
break
# plot mIOU, loss, and learnint rate curves
iou_path = os.path.join(EVAL_LOGDIR, 'miou.txt')
loss_path = os.path.join(TRAIN_LOGDIR, 'loss_learning_rate.txt')
miou_curve_path = plot_miou_loss_curve.plot_miou_loss_main(
iou_path,
train_count=train_count,
val_count=val_count,
batch_size=batch_size)
loss_curve_path = plot_miou_loss_curve.plot_miou_loss_main(
loss_path,
train_count=train_count,
val_count=val_count,
batch_size=batch_size)
# backup miou and training_loss & learning rate
test_id = os.path.basename(WORK_DIR) + '_' + expr_name
backup_dir = os.path.join(WORK_DIR, 'result_backup')
if os.path.isdir(backup_dir) is False:
io_function.mkdir(backup_dir)
new_iou_name = os.path.join(backup_dir,
test_id + '_' + os.path.basename(iou_path))
io_function.copy_file_to_dst(iou_path, new_iou_name, overwrite=True)
miou_curve_bakname = os.path.join(
backup_dir, test_id + '_' + os.path.basename(miou_curve_path))
io_function.copy_file_to_dst(miou_curve_path,
miou_curve_bakname,
overwrite=True)
loss_new_name = os.path.join(backup_dir,
test_id + '_' + os.path.basename(loss_path))
io_function.copy_file_to_dst(loss_path, loss_new_name, overwrite=True)
loss_curve_bakname = os.path.join(
backup_dir, test_id + '_' + os.path.basename(loss_curve_path))
io_function.copy_file_to_dst(loss_curve_path,
loss_curve_bakname,
overwrite=True)
def yolo_postProcess(para_file,
inf_post_note,
b_skip_getshp=False,
test_id=None):
# test_id is the related to training
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
# the test string in 'exe.sh'
test_note = inf_post_note
WORK_DIR = os.getcwd()
SECONDS = time.time()
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
network_setting_ini = parameters.get_string_parameters(
para_file, 'network_setting_ini')
inf_dir = parameters.get_directory(para_file, 'inf_output_dir')
nms_overlap_thr = parameters.get_digit_parameters(para_file,
'nms_overlapThresh',
'float')
if test_id is None:
test_id = os.path.basename(WORK_DIR) + '_' + expr_name
# get name of inference areas
multi_inf_regions = parameters.get_string_list_parameters(
para_file, 'inference_regions')
# run post-processing parallel
# max_parallel_postProc_task = 8
backup_dir = os.path.join(WORK_DIR, 'result_backup')
io_function.mkdir(backup_dir)
# loop each inference regions
sub_tasks = []
same_area_time_inis = group_same_area_time_observations(multi_inf_regions)
region_eva_reports = {}
for key in same_area_time_inis.keys():
multi_observations = same_area_time_inis[key]
area_name = parameters.get_string_parameters(
multi_observations[0],
'area_name') # they have the same name and time
area_time = parameters.get_string_parameters(multi_observations[0],
'area_time')
merged_shp_list = []
for area_idx, area_ini in enumerate(multi_observations):
area_remark = parameters.get_string_parameters(
area_ini, 'area_remark')
area_save_dir, shp_pre, _ = get_observation_save_dir_shp_pre(
inf_dir, area_name, area_time, area_remark, test_id)
# get image list
inf_image_dir = parameters.get_directory(area_ini, 'inf_image_dir')
# it is ok consider a file name as pattern and pass it the following functions to get file list
inf_image_or_pattern = parameters.get_string_parameters(
area_ini, 'inf_image_or_pattern')
inf_img_list = io_function.get_file_list_by_pattern(
inf_image_dir, inf_image_or_pattern)
img_count = len(inf_img_list)
if img_count < 1:
raise ValueError(
'No image for inference, please check inf_image_dir and inf_image_or_pattern in %s'
% area_ini)
merged_shp = os.path.join(WORK_DIR, area_save_dir,
shp_pre + '.shp')
if b_skip_getshp:
pass
else:
# post image one by one
result_shp_list = []
for img_idx, img_path in enumerate(inf_img_list):
out_shp = yolo_results_to_shapefile(
WORK_DIR, img_idx, area_save_dir, nms_overlap_thr,
test_id)
if out_shp is not None:
#TODO: remove some boxes based on DEM difference?
result_shp_list.append(os.path.join(WORK_DIR, out_shp))
# merge shapefiles
merge_shape_files(result_shp_list,
merged_shp,
b_create_id=True)
merged_shp_list.append(merged_shp)
if b_skip_getshp is False:
# add occurrence to each polygons
get_occurence_for_multi_observation(merged_shp_list)
for area_idx, area_ini in enumerate(multi_observations):
area_remark = parameters.get_string_parameters(
area_ini, 'area_remark')
area_save_dir, shp_pre, area_remark_time = get_observation_save_dir_shp_pre(
inf_dir, area_name, area_time, area_remark, test_id)
merged_shp = os.path.join(WORK_DIR, area_save_dir,
shp_pre + '.shp')
# add attributes to shapefile (no other attribute to add)
shp_attributes = os.path.join(WORK_DIR, area_save_dir,
shp_pre + '_post_NOrm.shp')
# shp_attributes = merged_shp
add_boxes_attributes(merged_shp,
shp_attributes) # para_file, area_ini
# remove polygons
shp_post = os.path.join(WORK_DIR, area_save_dir,
shp_pre + '_post.shp')
remove_polygons_main(shp_attributes, shp_post, para_file)
# evaluate the mapping results
out_report = os.path.join(WORK_DIR, area_save_dir,
shp_pre + '_evaluation_report.txt')
evaluation_polygons(shp_post, para_file, area_ini, out_report)
##### copy and backup files ######
# copy files to result_backup
if len(test_note) > 0:
backup_dir_area = os.path.join(
backup_dir, area_name + '_' + area_remark_time + '_' +
test_id + '_' + test_note)
else:
backup_dir_area = os.path.join(
backup_dir,
area_name + '_' + area_remark_time + '_' + test_id)
io_function.mkdir(backup_dir_area)
if len(test_note) > 0:
bak_merged_shp = os.path.join(
backup_dir_area, '_'.join([shp_pre, test_note]) + '.shp')
bak_post_shp = os.path.join(
backup_dir_area,
'_'.join([shp_pre, 'post', test_note]) + '.shp')
bak_eva_report = os.path.join(
backup_dir_area,
'_'.join([shp_pre, 'eva_report', test_note]) + '.txt')
bak_area_ini = os.path.join(
backup_dir_area,
'_'.join([shp_pre, 'region', test_note]) + '.ini')
else:
bak_merged_shp = os.path.join(backup_dir_area,
'_'.join([shp_pre]) + '.shp')
bak_post_shp = os.path.join(
backup_dir_area, '_'.join([shp_pre, 'post']) + '.shp')
bak_eva_report = os.path.join(
backup_dir_area,
'_'.join([shp_pre, 'eva_report']) + '.txt')
bak_area_ini = os.path.join(
backup_dir_area, '_'.join([shp_pre, 'region']) + '.ini')
io_function.copy_shape_file(merged_shp, bak_merged_shp)
io_function.copy_shape_file(shp_post, bak_post_shp)
io_function.copy_file_to_dst(area_ini,
bak_area_ini,
overwrite=True)
if os.path.isfile(out_report):
io_function.copy_file_to_dst(out_report,
bak_eva_report,
overwrite=True)
region_eva_reports[shp_pre] = bak_eva_report
if len(test_note) > 0:
bak_para_ini = os.path.join(
backup_dir, '_'.join([test_id, 'para', test_note]) + '.ini')
# bak_network_ini = os.path.join(backup_dir, '_'.join([test_id,'network',test_note]) + '.ini' )
bak_time_cost = os.path.join(
backup_dir, '_'.join([test_id, 'time_cost', test_note]) + '.txt')
else:
bak_para_ini = os.path.join(backup_dir,
'_'.join([test_id, 'para']) + '.ini')
# bak_network_ini = os.path.join(backup_dir, '_'.join([test_id, 'network']) + '.ini')
bak_time_cost = os.path.join(backup_dir,
'_'.join([test_id, 'time_cost']) + '.txt')
io_function.copy_file_to_dst(para_file, bak_para_ini)
# io_function.copy_file_to_dst(network_setting_ini, bak_network_ini)
if os.path.isfile('time_cost.txt'):
io_function.copy_file_to_dst('time_cost.txt', bak_time_cost)
# output the evaluation report to screen
for key in region_eva_reports.keys():
report = region_eva_reports[key]
print('evaluation report for %s:' % key)
os.system('head -n 7 %s' % report)
# output evaluation report to table
if len(test_note) > 0:
out_table = os.path.join(
backup_dir,
'_'.join([test_id, 'accuracy_table', test_note]) + '.xlsx')
else:
out_table = os.path.join(
backup_dir, '_'.join([test_id, 'accuracy_table']) + '.xlsx')
eva_reports = [region_eva_reports[key] for key in region_eva_reports]
if len(eva_reports) > 0:
eva_report_to_tables.eva_reports_to_table(eva_reports, out_table)
duration = time.time() - SECONDS
os.system(
'echo "$(date): time cost of post-procesing: %.2f seconds">>time_cost.txt'
% duration)
pass
default=0,
type=int,
help=
'switch to train edge, if this is on batchSize should be 1, 0 is off 1 is on'
)
parser.add_argument('--test',
action='store_true',
help='switch to test model, without train')
args = parser.parse_args()
print(args)
############## dataset processing
parameters.set_saved_parafile_path(args.para)
patch_w = parameters.get_digit_parameters("", "train_patch_width", None, 'int')
patch_h = parameters.get_digit_parameters("", "train_patch_height", None,
'int')
overlay_x = parameters.get_digit_parameters("", "train_pixel_overlay_x", None,
'int')
overlay_y = parameters.get_digit_parameters("", "train_pixel_overlay_y", None,
'int')
crop_height = parameters.get_digit_parameters("", "crop_height", None, 'int')
crop_width = parameters.get_digit_parameters("", "crop_width", None, 'int')
dataset = RemoteSensingImg(args.dataroot, args.list, patch_w, patch_h,
overlay_x, overlay_y, args.edge)
train_loader = torch.utils.data.DataLoader(dataset,
batch_size=args.batchSize,
num_workers=args.workers,
shuffle=True)
# torch.backends.cudnn.enabled = True
def calculate_polygon_topography(polygons_shp,
para_file,
dem_files,
slope_files,
aspect_files=None,
dem_diffs=None):
"""
calculate the topography information such elevation and slope of each polygon
Args:
polygons_shp: input shapfe file
dem_files: DEM raster file or tiles, should have the same projection of shapefile
slope_files: slope raster file or tiles (can be drived from dem file by using QGIS or ArcGIS)
aspect_files: aspect raster file or tiles (can be drived from dem file by using QGIS or ArcGIS)
Returns: True if successful, False Otherwise
"""
if io_function.is_file_exist(polygons_shp) is False:
return False
operation_obj = shape_opeation()
## calculate the topography information from the buffer area
# the para file was set in parameters.set_saved_parafile_path(options.para_file)
b_use_buffer_area = parameters.get_bool_parameters(
para_file, 'b_topo_use_buffer_area')
if b_use_buffer_area is True:
b_buffer_size = 5 # meters (the same as the shape file)
basic.outputlogMessage(
"info: calculate the topography information from the buffer area")
buffer_polygon_shp = io_function.get_name_by_adding_tail(
polygons_shp, 'buffer')
# if os.path.isfile(buffer_polygon_shp) is False:
if vector_features.get_buffer_polygons(
polygons_shp, buffer_polygon_shp, b_buffer_size) is False:
basic.outputlogMessage(
"error, failed in producing the buffer_polygon_shp")
return False
# else:
# basic.outputlogMessage("warning, buffer_polygon_shp already exist, skip producing it")
# replace the polygon shape file
polygons_shp_backup = polygons_shp
polygons_shp = buffer_polygon_shp
else:
basic.outputlogMessage(
"info: calculate the topography information from the inside of each polygon"
)
# all_touched: bool, optional
# Whether to include every raster cell touched by a geometry, or only
# those having a center point within the polygon.
# defaults to `False`
# Since the dem usually is coarser, so we set all_touched = True
all_touched = True
process_num = 4
# #DEM
if dem_files is not None:
stats_list = ['min', 'max', 'mean', 'median',
'std'] #['min', 'max', 'mean', 'count','median','std']
# if operation_obj.add_fields_from_raster(polygons_shp, dem_file, "dem", band=1,stats_list=stats_list,all_touched=all_touched) is False:
# return False
if zonal_stats_multiRasters(polygons_shp,
dem_files,
stats=stats_list,
prefix='dem',
band=1,
all_touched=all_touched,
process_num=process_num) is False:
return False
else:
basic.outputlogMessage(
"warning, DEM file not exist, skip the calculation of DEM information"
)
# #slope
if slope_files is not None:
stats_list = ['min', 'max', 'mean', 'median', 'std']
if zonal_stats_multiRasters(polygons_shp,
slope_files,
stats=stats_list,
prefix='slo',
band=1,
all_touched=all_touched,
process_num=process_num) is False:
return False
else:
basic.outputlogMessage(
"warning, slope file not exist, skip the calculation of slope information"
)
# #aspect
if aspect_files is not None:
stats_list = ['min', 'max', 'mean', 'std']
if zonal_stats_multiRasters(polygons_shp,
aspect_files,
stats=stats_list,
prefix='asp',
band=1,
all_touched=all_touched,
process_num=process_num) is False:
return False
else:
basic.outputlogMessage(
'warning, aspect file not exist, ignore adding aspect information')
# elevation difference
if dem_diffs is not None:
stats_list = ['min', 'max', 'mean', 'median', 'std', 'area']
# only count the pixel within this range when do statistics
dem_diff_range_str = parameters.get_string_list_parameters(
para_file, 'dem_difference_range')
range = [
None if item.upper() == 'NONE' else float(item)
for item in dem_diff_range_str
]
# expand the polygon when doing dem difference statistics
buffer_size_dem_diff = parameters.get_digit_parameters(
para_file, 'buffer_size_dem_diff', 'float')
if zonal_stats_multiRasters(polygons_shp,
dem_diffs,
stats=stats_list,
prefix='demD',
band=1,
all_touched=all_touched,
process_num=process_num,
range=range,
buffer=buffer_size_dem_diff) is False:
return False
else:
basic.outputlogMessage(
'warning, dem difference file not exist, ignore adding dem diff information'
)
# # hillshape
# copy the topography information
if b_use_buffer_area is True:
operation_obj.add_fields_shape(polygons_shp_backup, buffer_polygon_shp,
polygons_shp_backup)
return True
