def main(options, args):
polygons_shp = args[0]
output = options.output
if output is None:
output = io_function.get_name_by_adding_tail(polygons_shp, 'removed')
para_file = options.para_file
assert io_function.is_file_exist(polygons_shp)
# remove polygons based on area
rm_area_save_shp = io_function.get_name_by_adding_tail(
polygons_shp, 'rmArea')
# area_thr = 1000 #10 pixels
area_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_gully_area', 'int')
b_smaller = True
if area_thr is not None:
remove_polygons(polygons_shp, 'INarea', area_thr, b_smaller,
rm_area_save_shp)
# remove polygons based on slope information
rm_slope_save_shp1 = io_function.get_name_by_adding_tail(
polygons_shp, 'rmslope1')
# slope_small_thr = 2
slope_small_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'min_slope', 'float')
b_smaller = True
if slope_small_thr is not None:
remove_polygons(rm_area_save_shp, 'slo_mean', slope_small_thr,
b_smaller, rm_slope_save_shp1)
rm_slope_save_shp2 = io_function.get_name_by_adding_tail(
polygons_shp, 'rmslope2')
# slope_large_thr = 20
slope_large_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'max_slope', 'float')
b_smaller = False
if slope_large_thr is not None:
remove_polygons(rm_slope_save_shp1, 'slo_mean', slope_large_thr,
b_smaller, rm_slope_save_shp2)
# remove polgyons based on dem
rm_dem_save_shp = output # final output
# dem_small_thr = 3000
dem_small_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_elevation', 'int')
b_smaller = True
if dem_small_thr is not None:
remove_polygons(rm_slope_save_shp2, 'dem_mean', dem_small_thr,
b_smaller, rm_dem_save_shp)
pass
def get_iteration_num(WORK_DIR, para_file, network_setting_ini):
'''
get iteration num from setting (iteration_num) or calculate from train_epoch_num, batch_size and training sample count
:param WORK_DIR:
:param para_file:
:param network_setting_ini:
:return:
'''
train_epoch_num = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_epoch_num', 'int')
if train_epoch_num is None:
iteration_num = parameters.get_digit_parameters(
network_setting_ini, 'iteration_num', 'int')
with open('iteration_num.txt', 'w') as f_obj:
f_obj.writelines('iteration num is read from setting: %d\n' %
iteration_num)
else:
train_count, val_count = get_train_val_sample_count(
WORK_DIR, para_file)
batch_size = parameters.get_digit_parameters(network_setting_ini,
'batch_size', 'int')
iteration_num = math.ceil(train_epoch_num * train_count / batch_size)
with open('iteration_num.txt', 'w') as f_obj:
f_obj.writelines(
'iteration num (%d) is calculated from train_epoch_num: %d, training_sample_count: %d, '
'batch_size: %d\n' %
(iteration_num, train_epoch_num, train_count, batch_size))
return iteration_num
def generate_image_CUT(python_path, generate_script, gan_para_file, gpu_ids,
image_list, save_folder):
if os.path.isfile('generate.txt_done'):
basic.outputlogMessage(
'generate of new images using GAN in %s has completed previoulsy, please remove them if necessary'
% os.getcwd())
return True
time0 = time.time()
generate_tile_width = parameters.get_digit_parameters(
gan_para_file, 'generate_tile_width', 'int')
generate_tile_height = parameters.get_digit_parameters(
gan_para_file, 'generate_tile_height', 'int')
generate_overlay_x = parameters.get_digit_parameters(
gan_para_file, 'generate_overlay_x', 'int')
generate_overlay_y = parameters.get_digit_parameters(
gan_para_file, 'generate_overlay_y', 'int')
folder = os.path.basename(os.getcwd())
img_list_txt = 'image_to_generate_list.txt'
io_function.save_list_to_txt(img_list_txt, image_list)
command_string = python_path + ' ' + generate_script \
+ ' --dataset_mode '+'satelliteimage' \
+ ' --model '+ 'generate' \
+ ' --image_A_dir_txt ' + img_list_txt \
+ ' --tile_width ' + str(generate_tile_width) \
+ ' --tile_height ' + str(generate_tile_height) \
+ ' --overlay_x ' + str(generate_overlay_x) \
+ ' --overlay_y ' + str(generate_overlay_y) \
+ ' --name ' + folder \
+ ' --results_dir ' + save_folder \
+ ' --gpu_ids ' + ','.join([str(item) for item in gpu_ids])
train_max_dataset_size = parameters.get_digit_parameters_None_if_absence(
gan_para_file, 'gen_max_dataset_size', 'int')
if train_max_dataset_size is not None:
command_string += ' --max_dataset_size ' + str(train_max_dataset_size)
# if it's cycleGAN, need to assign A generator
gan_model = parameters.get_string_parameters(gan_para_file, 'gan_model')
if gan_model == 'cycle_gan':
command_string += ' --model_suffix _A ' # from A to B
# status, result = basic.exec_command_string(command_string) # this will wait command finished
# os.system(command_string + "&") # don't know when it finished
res = os.system(command_string) # this work
# print('command_string deeplab_inf_script: res',res)
if res != 0:
sys.exit(1)
duration = time.time() - time0
os.system(
'echo "$(date): time cost of generate images using a GAN : %.2f seconds">>"time_cost.txt"'
% (duration))
# write a file to indicate that the process has completed.
os.system('echo done > generate.txt_done')
return True
def add_polygon_attributes(input, output, para_file, data_para_file):
if io_function.is_file_exist(input) is False:
return False
# copy output
if io_function.copy_shape_file(input, output) is False:
raise IOError('copy shape file %s failed' % input)
# remove narrow parts of mapped polygons
polygon_narrow_part_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'mapped_polygon_narrow_threshold', 'float')
# if it is not None, then it will try to remove narrow parts of polygons
if polygon_narrow_part_thr is not None and polygon_narrow_part_thr > 0:
# use the buffer operation to remove narrow parts of polygons
basic.outputlogMessage(
"start removing narrow parts (thr %.2f) in polygons" %
(polygon_narrow_part_thr * 2))
if vector_gpd.remove_narrow_parts_of_polygons_shp_NOmultiPolygon(
input, output, polygon_narrow_part_thr):
message = "Finished removing narrow parts (thr %.2f) in polygons and save to %s" % (
polygon_narrow_part_thr * 2, output)
basic.outputlogMessage(message)
else:
pass
else:
basic.outputlogMessage(
"warning, mapped_polygon_narrow_threshold is not in the parameter file, skip removing narrow parts"
)
# calculate area, perimeter of polygons
if cal_add_area_length_of_polygon(output) is False:
return False
# calculate the polygon information
b_calculate_shape_info = parameters.get_bool_parameters_None_if_absence(
para_file, 'b_calculate_shape_info')
if b_calculate_shape_info:
# remove "_shapeInfo.shp" to make it calculate shape information again
os.system('rm *_shapeInfo.shp')
if calculate_gully_information(output) is False:
return False
# add topography of each polygons
dem_files, slope_files, aspect_files, dem_diff_files = get_topographic_files(
data_para_file)
if calculate_polygon_topography(output,
para_file,
dem_files,
slope_files,
aspect_files=aspect_files,
dem_diffs=dem_diff_files) is False:
basic.outputlogMessage(
'Warning: calculate information of topography failed')
# return False # don't return
return True
def split_train_val(para_file):
print("split data set into training and validation")
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
code_dir = os.path.join(os.path.dirname(sys.argv[0]), '..')
sys.path.insert(0, code_dir)
import parameters
script = os.path.join(code_dir, 'datasets', 'train_test_split.py')
training_data_per = parameters.get_digit_parameters_None_if_absence(
para_file, 'training_data_per', 'float')
train_sample_txt = parameters.get_string_parameters(
para_file, 'training_sample_list_txt')
val_sample_txt = parameters.get_string_parameters(
para_file, 'validation_sample_list_txt')
dir = 'list'
all_img_list = os.path.join(dir, 'trainval.txt')
# command_string = script + ' -p ' + str(training_data_per) + \
# ' -t ' + train_sample_txt + \
# ' -v ' + val_sample_txt + \
# ' --shuffle ' + all_img_list
# res = os.system(command_string)
# if res!=0:
# sys.exit(1)
Do_shuffle = True
from datasets.train_test_split import train_test_split_main
train_test_split_main(all_img_list, training_data_per, Do_shuffle,
train_sample_txt, val_sample_txt)
# save brief information of image patches
img_ext = parameters.get_string_parameters_None_if_absence(
para_file, 'split_image_format')
get_image_with_height_list(os.path.join(dir, train_sample_txt),
img_ext,
info_type='training')
get_image_with_height_list(os.path.join(dir, val_sample_txt),
img_ext,
info_type='validation')
# save the count of each classes in training and validation
get_sample_count_of_each_class(os.path.join(dir, train_sample_txt),
info_type='training')
get_sample_count_of_each_class(os.path.join(dir, val_sample_txt),
info_type='validation')
def main(options, args):
polygons_shp = args[0]
polygons_shp_backup = args[0]
output = options.output
if output is None:
output = io_function.get_name_by_adding_tail(polygons_shp, 'removed')
para_file = options.para_file
assert io_function.is_file_exist(polygons_shp)
# remove polygons based on area
# area_thr = 1000 #10 pixels
area_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_area', 'int')
b_smaller = True
if area_thr is not None:
rm_area_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmArea')
if remove_polygons(polygons_shp, 'INarea', area_thr, b_smaller,
rm_area_save_shp) is False:
basic.outputlogMessage(
"error, removing polygons based on size failed")
else:
polygons_shp = rm_area_save_shp
else:
basic.outputlogMessage(
'warning, minimum_area is absent in the para file, skip removing polygons based on areas'
)
# remove polygons based on slope information
# slope_small_thr = 2
slope_small_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_slope', 'float')
b_smaller = True
if slope_small_thr is not None:
rm_slope_save_shp1 = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmslope1')
if remove_polygons(polygons_shp, 'slo_mean', slope_small_thr,
b_smaller, rm_slope_save_shp1) is False:
basic.outputlogMessage(
"error, removing polygons based on slo_mean failed")
else:
polygons_shp = rm_slope_save_shp1
else:
basic.outputlogMessage(
'warning, minimum_slope is absent in the para file, skip removing polygons based on minimum slope'
)
# slope_large_thr = 20
slope_large_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'maximum_slope', 'float')
b_smaller = False
if slope_large_thr is not None:
rm_slope_save_shp2 = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmslope2')
if remove_polygons(polygons_shp, 'slo_mean', slope_large_thr,
b_smaller, rm_slope_save_shp2) is False:
basic.outputlogMessage(
"error, removing polygons based on slo_mean (2) failed")
else:
polygons_shp = rm_slope_save_shp2
else:
basic.outputlogMessage(
'warning, maximum_slope is absent in the para file, skip removing polygons based on maximum slope'
)
# remove polygons based on dem
# dem_small_thr = 3000
dem_small_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_elevation', 'int')
b_smaller = True
if dem_small_thr is not None:
rm_dem_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmDEM')
if remove_polygons(polygons_shp, 'dem_mean', dem_small_thr, b_smaller,
rm_dem_save_shp) is False:
basic.outputlogMessage(
"error, removing polygons based on dem_mean failed")
else:
polygons_shp = rm_dem_save_shp
else:
basic.outputlogMessage(
'warning, minimum_elevation is absent in the para file, skip removing polygons based on minimum elevation'
)
# remove polygons not in the extent
outline_shp = parameters.get_string_parameters_None_if_absence(
para_file, 'target_outline_shp')
if outline_shp is not None:
rm_outline_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmOutline')
remove_polygons_outside_extent(polygons_shp, outline_shp,
rm_outline_save_shp)
polygons_shp = rm_outline_save_shp
else:
basic.outputlogMessage(
'warning, target_outline_shp is absent in the para file, skip removing polygons based on outlines'
)
# copy to final output
copy_shape_file(polygons_shp, output)
pass
def predict_one_image_deeplab(deeplab_inf_script,
para_file,
network_ini,
save_dir,
inf_list_file,
gpuid=None,
trained_model=None):
done_indicator = '%s_done' % inf_list_file
if os.path.isfile(done_indicator):
basic.outputlogMessage('warning, %s exist, skip prediction' %
done_indicator)
return
# use a specific GPU for prediction, only inference one image
time0 = time.time()
if gpuid is not None:
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpuid)
# command_string = deeplab_predict_script + ' '+ para_file + ' ' + save_dir + ' ' + inf_list_file + ' ' + str(gpuid)
# # status, result = basic.exec_command_string(command_string) # this will wait command finished
# # os.system(command_string + "&") # don't know when it finished
# os.system(command_string ) # this work
if trained_model is None:
WORK_DIR = os.getcwd()
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
EXP_FOLDER = expr_name
EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export')
TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train')
iteration_num = get_trained_iteration(TRAIN_LOGDIR)
EXPORT_PATH = os.path.join(
EXPORT_DIR, 'frozen_inference_graph_%s.pb' % iteration_num)
frozen_graph_path = EXPORT_PATH
else:
frozen_graph_path = trained_model
if os.path.isfile(frozen_graph_path) is False:
raise IOError('cannot find trained model: %s' % frozen_graph_path)
inf_batch_size = parameters.get_digit_parameters_None_if_absence(
network_ini, 'inf_batch_size', 'int')
if inf_batch_size is None:
raise ValueError('inf_batch_size not set in %s' % network_ini)
command_string = tf1x_python + ' ' + deeplab_inf_script \
+ ' --inf_para_file='+para_file \
+ ' --inf_list_file='+inf_list_file \
+ ' --inf_batch_size='+str(inf_batch_size) \
+ ' --inf_output_dir='+save_dir \
+ ' --frozen_graph_path='+frozen_graph_path
# status, result = basic.exec_command_string(command_string) # this will wait command finished
# os.system(command_string + "&") # don't know when it finished
res = os.system(command_string) # this work
# print('command_string deeplab_inf_script: res',res)
if res != 0:
sys.exit(1)
duration = time.time() - time0
os.system(
'echo "$(date): time cost of inference for image in %s: %.2f seconds">>"time_cost.txt"'
% (inf_list_file, duration))
# write a file to indicate that the prediction has done.
os.system('echo %s > %s_done' % (inf_list_file, inf_list_file))
def image_augment_main(para_file, img_list_txt, save_list, img_dir, out_dir,
extension, is_ground_truth, proc_num):
basic.setlogfile('log_data_augmentation.txt')
if os.path.isfile(img_list_txt) is False:
raise IOError("File %s not exist" % img_list_txt)
if os.path.isdir(out_dir) is False:
os.makedirs(out_dir)
if img_dir != out_dir:
raise ValueError(
'set image dir and output dir be the same, making it easy to update image list'
)
# print(options.para_file)
augmentation = parameters.get_string_list_parameters_None_if_absence(
para_file, 'data_augmentation')
if augmentation is None or len(augmentation) < 1:
basic.outputlogMessage(
'No input augmentation requirement (e.g. flip), skip data augmentation'
)
return True
# number of classes
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
global num_classes
num_classes = num_classes_noBG + 1
# ignored classes
ignore_classes = parameters.get_string_list_parameters_None_if_absence(
para_file, 'data_aug_ignore_classes')
with open(img_list_txt, 'r') as f_obj:
files_list = f_obj.readlines()
file_count = len(files_list)
index = 1
# for line in files_list:
#
# # ignore_classes
# if ignore_classes is not None and len(ignore_classes)>0:
# found_class = [ line.find(ignore_class) >= 0 for ignore_class in ignore_classes ]
# if True in found_class:
# continue
#
# file_path = line.strip()
# file_path = os.path.join(img_dir,file_path+extension)
# print ("Augmentation of image (%d / %d)"%(index,file_count))
# if image_augment(file_path,out_dir,is_ground_truth,augment=augmentation) is False:
# print ('Error, Failed in image augmentation')
# return False
# index += 1
parameters_list = [(index + 1, line, ignore_classes, file_count, img_dir,
extension, out_dir, is_ground_truth, augmentation)
for index, line in enumerate(files_list)]
theadPool = Pool(proc_num) # multi processes
results = theadPool.starmap(augment_one_line,
parameters_list) # need python3
augmented = [1 for item in results if item is True]
# print(sum(augmented))
if sum(augmented) < file_count:
basic.outputlogMessage('Some of the images belong to %s are ignored' %
','.join(ignore_classes))
# update img_list_txt (img_dir is the same as out_dir)
new_files = io_function.get_file_list_by_ext(extension,
out_dir,
bsub_folder=False)
new_files_noext = [
os.path.splitext(os.path.basename(item))[0] + '\n'
for item in new_files
]
basic.outputlogMessage('save new file list to %s' % save_list)
with open(save_list, 'w') as f_obj:
f_obj.writelines(new_files_noext)
def split_train_val(para_file):
print("split data set into training and validation")
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd()))
code_dir = os.path.join(os.path.dirname(sys.argv[0]), '..')
sys.path.insert(0, code_dir)
import parameters
script = os.path.join(code_dir, 'datasets', 'train_test_split.py')
training_data_per = parameters.get_digit_parameters_None_if_absence(para_file, 'training_data_per','float')
train_sample_txt = parameters.get_string_parameters(para_file, 'training_sample_list_txt')
val_sample_txt = parameters.get_string_parameters(para_file, 'validation_sample_list_txt')
dir = 'list'
all_img_list = os.path.join(dir,'trainval.txt')
# command_string = script + ' -p ' + str(training_data_per) + \
# ' -t ' + train_sample_txt + \
# ' -v ' + val_sample_txt + \
# ' --shuffle ' + all_img_list
# res = os.system(command_string)
# if res!=0:
# sys.exit(1)
if training_data_per is None:
# similar to VOC dataset, we only used 1449 images for validation (because the data also used for training,
# so it is training accuracy, not validation accuracy)
with open(all_img_list, 'r') as f_obj:
file_names = f_obj.readlines()
if len(file_names) < 1449:
# val.txt is identical to trainval.txt
io_function.copy_file_to_dst(all_img_list,os.path.join(dir,train_sample_txt))
io_function.copy_file_to_dst(all_img_list,os.path.join(dir,val_sample_txt))
else:
io_function.copy_file_to_dst(all_img_list, os.path.join(dir, train_sample_txt))
# randomly get 1449 image from trainval.txt
import random
sel_file_index = random.sample(range(len(file_names)), 1449) # get a list of number without duplicates
with open(os.path.join(dir, val_sample_txt), 'w') as w_obj:
sel_file_names = [file_names[item] for item in sel_file_index]
w_obj.writelines(sel_file_names)
else:
# split training and validation datasets
Do_shuffle = True
from datasets.train_test_split import train_test_split_main
train_test_split_main(all_img_list,training_data_per,Do_shuffle,train_sample_txt,val_sample_txt)
# save brief information of image patches
img_ext = parameters.get_string_parameters_None_if_absence(para_file, 'split_image_format')
get_image_with_height_list(os.path.join(dir,train_sample_txt), img_ext, info_type='training')
get_image_with_height_list(os.path.join(dir,val_sample_txt), img_ext, info_type='validation')
# save the count of each classes in training and validation
get_sample_count_of_each_class(os.path.join(dir, train_sample_txt), info_type='training')
get_sample_count_of_each_class(os.path.join(dir, val_sample_txt), info_type='validation')
def train_CUT_gan(python_path, train_script, gan_para_file, gpu_ids):
if os.path.isfile('train.txt_done'):
basic.outputlogMessage(
'training of GAN in %s has completed previoulsy, please remove it if necessary'
% os.getcwd())
return True
time0 = time.time()
train_tile_width = parameters.get_digit_parameters(gan_para_file,
'train_tile_width',
'int')
train_tile_height = parameters.get_digit_parameters(
gan_para_file, 'train_tile_height', 'int')
train_overlay_x = parameters.get_digit_parameters(gan_para_file,
'train_overlay_x', 'int')
train_overlay_y = parameters.get_digit_parameters(gan_para_file,
'train_overlay_y', 'int')
gan_model = parameters.get_string_parameters(gan_para_file, 'gan_model')
folder = os.path.basename(os.getcwd())
train_models = io_function.get_file_list_by_pattern(
os.path.join('checkpoints', folder), 'latest*.pth')
command_string = python_path + ' ' + train_script \
+ ' --model ' + str(gan_model) \
+ ' --dataset_mode '+'satelliteimage' \
+ ' --image_A_dir_txt ' + 'image_A_list.txt' \
+ ' --image_B_dir_txt ' + 'image_B_list.txt' \
+ ' --tile_width ' + str(train_tile_width) \
+ ' --tile_height ' + str(train_tile_height) \
+ ' --overlay_x ' + str(train_overlay_x) \
+ ' --overlay_y ' + str(train_overlay_y) \
+ ' --display_env ' + folder \
+ ' --name ' + folder \
+ ' --gpu_ids ' + ','.join([str(item) for item in gpu_ids])
# if trained model exist, continue train
if len(train_models) > 0:
command_string += ' --continue_train '
n_epochs = parameters.get_digit_parameters(gan_para_file, 'n_epochs',
'int')
n_epochs_decay = parameters.get_digit_parameters(gan_para_file,
'n_epochs_decay', 'int')
command_string += ' --n_epochs ' + str(
n_epochs) + ' --n_epochs_decay ' + str(n_epochs_decay)
train_max_dataset_size = parameters.get_digit_parameters_None_if_absence(
gan_para_file, 'train_max_dataset_size', 'int')
if train_max_dataset_size is not None:
command_string += ' --max_dataset_size ' + str(train_max_dataset_size)
# status, result = basic.exec_command_string(command_string) # this will wait command finished
# os.system(command_string + "&") # don't know when it finished
res = os.system(command_string) # this work
# print('command_string deeplab_inf_script: res',res)
if res != 0:
sys.exit(1)
duration = time.time() - time0
os.system(
'echo "$(date): time cost of training a GAN : %.2f seconds">>"time_cost.txt"'
% (duration))
# write a file to indicate that the training has completed.
os.system('echo done > train.txt_done')
return True
def main(options, args):
print("%s : export the frozen inference graph" %
os.path.basename(sys.argv[0]))
para_file = args[0]
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
network_setting_ini = parameters.get_string_parameters(
para_file, 'network_setting_ini')
tf_research_dir = parameters.get_directory_None_if_absence(
network_setting_ini, 'tf_research_dir')
print(tf_research_dir)
if tf_research_dir is None:
raise ValueError('tf_research_dir is not in %s' % para_file)
if os.path.isdir(tf_research_dir) is False:
raise ValueError('%s does not exist' % tf_research_dir)
if os.getenv('PYTHONPATH'):
os.environ['PYTHONPATH'] = os.getenv(
'PYTHONPATH') + ':' + tf_research_dir + ':' + os.path.join(
tf_research_dir, 'slim')
else:
os.environ['PYTHONPATH'] = tf_research_dir + ':' + os.path.join(
tf_research_dir, 'slim')
global tf1x_python
tf1x_python = parameters.get_file_path_parameters(network_setting_ini,
'tf1x_python')
deeplab_dir = os.path.join(tf_research_dir, 'deeplab')
WORK_DIR = os.getcwd()
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
EXP_FOLDER = expr_name
TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train')
EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export')
inf_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_output_stride', 'int')
inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates1', 'int')
inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates2', 'int')
inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates3', 'int')
# depth_multiplier default is 1.0.
depth_multiplier = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'depth_multiplier', 'float')
decoder_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'decoder_output_stride', 'int')
aspp_convs_filters = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'aspp_convs_filters', 'int')
model_variant = parameters.get_string_parameters(network_setting_ini,
'model_variant')
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
assert num_classes_noBG != None
b_initialize_last_layer = parameters.get_bool_parameters(
para_file, 'b_initialize_last_layer')
if b_initialize_last_layer is False:
pre_trained_tar = parameters.get_string_parameters(
network_setting_ini, 'TF_INIT_CKPT')
if pre_trained_tar in pre_trained_tar_21_classes:
print(
'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21'
% pre_trained_tar)
num_classes_noBG = 20
if pre_trained_tar in pre_trained_tar_19_classes:
print(
'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19'
% pre_trained_tar)
num_classes_noBG = 18
num_of_classes = num_classes_noBG + 1
image_crop_size = parameters.get_string_list_parameters(
para_file, 'image_crop_size')
if len(image_crop_size) != 2 and image_crop_size[0].isdigit(
) and image_crop_size[1].isdigit():
raise ValueError('image_crop_size should be height,width')
iteration_num = get_trained_iteration(TRAIN_LOGDIR)
multi_scale = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'export_multi_scale', 'int')
export_script = os.path.join(deeplab_dir, 'export_model.py')
CKPT_PATH = os.path.join(TRAIN_LOGDIR, 'model.ckpt-%s' % iteration_num)
EXPORT_PATH = os.path.join(EXPORT_DIR,
'frozen_inference_graph_%s.pb' % iteration_num)
if os.path.isfile(EXPORT_PATH):
basic.outputlogMessage('%s exists, skipping exporting models' %
EXPORT_PATH)
return
export_graph(export_script, CKPT_PATH, EXPORT_PATH, model_variant,
num_of_classes, inf_atrous_rates1, inf_atrous_rates2,
inf_atrous_rates3, inf_output_stride, image_crop_size[0],
image_crop_size[1], multi_scale, depth_multiplier,
decoder_output_stride, aspp_convs_filters)
def main(options, args):
input = args[0]
output = args[1]
if io_function.is_file_exist(input) is False:
return False
data_para_file = options.data_para
if data_para_file is None:
data_para_file = options.para_file
## remove non-gully polygons
# output_rm_nonclass = io_function.get_name_by_adding_tail(input, 'rm_nonclass')
# if remove_nonclass_polygon(input,output_rm_nonclass,field_name='svmclass') is False:
# return False
# merge the touched polygons
# ouput_merged = io_function.get_name_by_adding_tail(input,'merged')
# if merge_polygons_in_gully(input,ouput_merged) is False:
# return False
# ouput_merged = input
# copy output
if io_function.copy_shape_file(input, output) is False:
raise IOError('copy shape file %s failed' % input)
else:
pass
# remove narrow parts of mapped polygons
polygon_narrow_part_thr = parameters.get_digit_parameters_None_if_absence(
'', 'mapped_polygon_narrow_threshold', 'float')
# if it is not None, then it will try to remove narrow parts of polygons
if polygon_narrow_part_thr is not None and polygon_narrow_part_thr > 0:
# use the buffer operation to remove narrow parts of polygons
basic.outputlogMessage(
"start removing narrow parts (thr %.2f) in polygons" %
(polygon_narrow_part_thr * 2))
if vector_gpd.remove_narrow_parts_of_polygons_shp_NOmultiPolygon(
input, output, polygon_narrow_part_thr):
message = "Finished removing narrow parts (thr %.2f) in polygons and save to %s" % (
polygon_narrow_part_thr * 2, output)
basic.outputlogMessage(message)
else:
pass
else:
basic.outputlogMessage(
"warning, mapped_polygon_narrow_threshold is not in the parameter file, skip removing narrow parts"
)
# calculate area, perimeter of polygons
if cal_add_area_length_of_polygon(output) is False:
return False
# calculate the polygon information
b_calculate_shape_info = parameters.get_bool_parameters_None_if_absence(
'', 'b_calculate_shape_info')
if b_calculate_shape_info:
# remove "_shapeInfo.shp" to make it calculate shape information again
os.system('rm *_shapeInfo.shp')
if calculate_gully_information(output) is False:
return False
# # remove small and not narrow polygons
# if options.min_area is None:
# basic.outputlogMessage('minimum area is required for remove polygons')
# return False
# area_thr = options.min_area
#
# if options.min_ratio is None:
# basic.outputlogMessage('minimum ratio of perimeter/area is required for remove polygons')
# return False
# ratio_thr = options.min_ratio
# if remove_small_round_polygons(ouput_merged,output,area_thr,ratio_thr) is False:
# return False
# add topography of each polygons
dem_file, slope_file, aspect_file, dem_diff_file = get_topographic_files(
data_para_file)
if calculate_polygon_topography(output,
dem_file,
slope_file,
aspect_file=aspect_file,
dem_diff=dem_diff_file) is False:
basic.outputlogMessage(
'Warning: calculate information of topography failed')
# return False # don't return
# add hydrology information
flow_accum = parameters.get_flow_accumulation()
if os.path.isfile(flow_accum):
if calculate_hydrology(output, flow_accum) is False:
basic.outputlogMessage(
'Warning: calculate information of hydrology failed')
# return False # don't return
else:
basic.outputlogMessage(
"warning, flow accumulation file not exist, skip the calculation of flow accumulation"
)
# # evaluation result
# val_path = parameters.get_validation_shape()
# if os.path.isfile(val_path):
# evaluation_result(output,val_path)
# else:
# basic.outputlogMessage("warning, validation polygon not exist, skip evaluation")
pass
def run_evaluation(WORK_DIR,
deeplab_dir,
expr_name,
para_file,
network_setting_ini,
gpu_num,
train_dir=None):
EXP_FOLDER = expr_name
if train_dir is None:
TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train')
else:
TRAIN_LOGDIR = train_dir
EVAL_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'eval')
dataset_dir = os.path.join(WORK_DIR, 'tfrecord')
inf_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_output_stride', 'int')
inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates1', 'int')
inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates2', 'int')
inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates3', 'int')
b_initialize_last_layer = parameters.get_bool_parameters(
para_file, 'b_initialize_last_layer')
pre_trained_tar = parameters.get_string_parameters(network_setting_ini,
'TF_INIT_CKPT')
# depth_multiplier default is 1.0.
depth_multiplier = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'depth_multiplier', 'float')
decoder_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'decoder_output_stride', 'int')
aspp_convs_filters = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'aspp_convs_filters', 'int')
model_variant = parameters.get_string_parameters(network_setting_ini,
'model_variant')
dataset = parameters.get_string_parameters(para_file, 'dataset_name')
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
assert num_classes_noBG != None
if b_initialize_last_layer is True:
if pre_trained_tar in pre_trained_tar_21_classes:
print(
'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21'
% pre_trained_tar)
num_classes_noBG = 20
if pre_trained_tar in pre_trained_tar_19_classes:
print(
'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19'
% pre_trained_tar)
num_classes_noBG = 18
num_of_classes = num_classes_noBG + 1
image_crop_size = parameters.get_string_list_parameters(
para_file, 'image_crop_size')
if len(image_crop_size) != 2 and image_crop_size[0].isdigit(
) and image_crop_size[1].isdigit():
raise ValueError('image_crop_size should be height,width')
crop_size_str = ','.join(image_crop_size)
evl_script = os.path.join(deeplab_dir, 'eval.py')
evl_split = os.path.splitext(
parameters.get_string_parameters(para_file,
'validation_sample_list_txt'))[0]
max_eva_number = 1
eval_interval_secs = 300
# gpuid = '' # do not use GPUs
evaluation_deeplab(evl_script,
dataset,
evl_split,
num_of_classes,
model_variant,
inf_atrous_rates1,
inf_atrous_rates2,
inf_atrous_rates3,
inf_output_stride,
TRAIN_LOGDIR,
EVAL_LOGDIR,
dataset_dir,
crop_size_str,
max_eva_number,
depth_multiplier,
decoder_output_stride,
aspp_convs_filters,
eval_interval_secs=eval_interval_secs)
# get miou again
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
def image_label_to_yolo_format(para_file):
print("Image labels (semantic segmentation) to YOLO object detection")
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
img_ext = parameters.get_string_parameters_None_if_absence(
para_file, 'split_image_format')
proc_num = parameters.get_digit_parameters(para_file, 'process_num', 'int')
SECONDS = time.time()
# get image and label path
image_list = []
label_list = []
with open(os.path.join('list', 'trainval.txt'), 'r') as f_obj:
lines = [item.strip() for item in f_obj.readlines()]
for line in lines:
image_list.append(os.path.join('split_images', line + img_ext))
label_list.append(os.path.join('split_labels', line + img_ext))
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
b_ignore_edge_objects = parameters.get_bool_parameters_None_if_absence(
para_file, 'b_ignore_edge_objects')
if b_ignore_edge_objects is None:
b_ignore_edge_objects = False
# get boxes
total_count = len(image_list)
for idx, (img, label) in enumerate(zip(image_list, label_list)):
get_yolo_boxes_one_img(idx,
total_count,
img,
label,
num_classes_noBG,
rm_edge_obj=b_ignore_edge_objects)
# write obj.data file
train_sample_txt = parameters.get_string_parameters(
para_file, 'training_sample_list_txt')
val_sample_txt = parameters.get_string_parameters(
para_file, 'validation_sample_list_txt')
train_img_list = get_image_list('list', train_sample_txt, 'split_images',
img_ext)
val_img_list = get_image_list('list', val_sample_txt, 'split_images',
img_ext)
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
object_names = parameters.get_string_list_parameters(
para_file, 'object_names')
io_function.mkdir('data')
io_function.mkdir(expr_name)
with open(os.path.join('data', 'obj.data'), 'w') as f_obj:
f_obj.writelines('classes = %d' % num_classes_noBG + '\n')
train_txt = os.path.join('data', 'train.txt')
io_function.save_list_to_txt(train_txt, train_img_list)
f_obj.writelines('train = %s' % train_txt + '\n')
val_txt = os.path.join('data', 'val.txt')
io_function.save_list_to_txt(val_txt, val_img_list)
f_obj.writelines('valid = %s' % val_txt + '\n')
obj_name_txt = os.path.join('data', 'obj.names')
io_function.save_list_to_txt(obj_name_txt, object_names)
f_obj.writelines('names = %s' % obj_name_txt + '\n')
f_obj.writelines('backup = %s' % expr_name + '\n')
duration = time.time() - SECONDS
os.system(
'echo "$(date): time cost of converting to yolo format: %.2f seconds">>time_cost.txt'
% duration)
pass
def train_evaluation_deeplab(WORK_DIR, deeplab_dir, expr_name, para_file,
network_setting_ini, gpu_num):
# prepare training folder
EXP_FOLDER = expr_name
INIT_FOLDER = os.path.join(WORK_DIR, EXP_FOLDER, 'init_models')
TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train')
EVAL_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'eval')
VIS_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'vis')
EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export')
io_function.mkdir(INIT_FOLDER)
io_function.mkdir(TRAIN_LOGDIR)
io_function.mkdir(EVAL_LOGDIR)
io_function.mkdir(VIS_LOGDIR)
io_function.mkdir(EXPORT_DIR)
# prepare the tensorflow check point (pretrained model) for training
pre_trained_dir = parameters.get_directory_None_if_absence(
network_setting_ini, 'pre_trained_model_folder')
pre_trained_tar = parameters.get_string_parameters(network_setting_ini,
'TF_INIT_CKPT')
pre_trained_path = os.path.join(pre_trained_dir, pre_trained_tar)
if os.path.isfile(pre_trained_path) is False:
print('pre-trained model: %s not exist, try to download' %
pre_trained_path)
# try to download the file
pre_trained_url = parameters.get_string_parameters_None_if_absence(
network_setting_ini, 'pre_trained_model_url')
res = os.system('wget %s ' % pre_trained_url)
if res != 0:
sys.exit(1)
io_function.movefiletodir(pre_trained_tar, pre_trained_dir)
# unpack pre-trained model to INIT_FOLDER
os.chdir(INIT_FOLDER)
res = os.system('tar -xf %s' % pre_trained_path)
if res != 0:
raise IOError('failed to unpack %s' % pre_trained_path)
os.chdir(WORK_DIR)
dataset_dir = os.path.join(WORK_DIR, 'tfrecord')
batch_size = parameters.get_digit_parameters(network_setting_ini,
'batch_size', 'int')
# maximum iteration number
iteration_num = parameters.get_digit_parameters(network_setting_ini,
'iteration_num', 'int')
base_learning_rate = parameters.get_digit_parameters(
network_setting_ini, 'base_learning_rate', 'float')
train_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_output_stride', 'int')
train_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates1', 'int')
train_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates2', 'int')
train_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates3', 'int')
inf_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_output_stride', 'int')
inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates1', 'int')
inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates2', 'int')
inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates3', 'int')
# depth_multiplier default is 1.0.
depth_multiplier = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'depth_multiplier', 'float')
decoder_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'decoder_output_stride', 'int')
aspp_convs_filters = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'aspp_convs_filters', 'int')
train_script = os.path.join(deeplab_dir, 'train.py')
train_split = os.path.splitext(
parameters.get_string_parameters(para_file,
'training_sample_list_txt'))[0]
model_variant = parameters.get_string_parameters(network_setting_ini,
'model_variant')
checkpoint = parameters.get_string_parameters(network_setting_ini,
'tf_initial_checkpoint')
init_checkpoint_files = io_function.get_file_list_by_pattern(
INIT_FOLDER, checkpoint + '*')
if len(init_checkpoint_files) < 1:
raise IOError('No initial checkpoint in %s with pattern: %s' %
(INIT_FOLDER, checkpoint))
init_checkpoint = os.path.join(INIT_FOLDER, checkpoint)
b_early_stopping = parameters.get_bool_parameters(para_file,
'b_early_stopping')
b_initialize_last_layer = parameters.get_bool_parameters(
para_file, 'b_initialize_last_layer')
dataset = parameters.get_string_parameters(para_file, 'dataset_name')
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
assert num_classes_noBG != None
if b_initialize_last_layer is True:
if pre_trained_tar in pre_trained_tar_21_classes:
print(
'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21'
% pre_trained_tar)
num_classes_noBG = 20
if pre_trained_tar in pre_trained_tar_19_classes:
print(
'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19'
% pre_trained_tar)
num_classes_noBG = 18
num_of_classes = num_classes_noBG + 1
image_crop_size = parameters.get_string_list_parameters(
para_file, 'image_crop_size')
if len(image_crop_size) != 2 and image_crop_size[0].isdigit(
) and image_crop_size[1].isdigit():
raise ValueError('image_crop_size should be height,width')
crop_size_str = ','.join(image_crop_size)
evl_script = os.path.join(deeplab_dir, 'eval.py')
evl_split = os.path.splitext(
parameters.get_string_parameters(para_file,
'validation_sample_list_txt'))[0]
max_eva_number = 1
# validation interval (epoch)
validation_interval = parameters.get_digit_parameters_None_if_absence(
para_file, 'validation_interval', 'int')
train_count, val_count = get_train_val_sample_count(WORK_DIR, para_file)
iter_per_epoch = math.ceil(train_count / batch_size)
total_epoches = math.ceil(iteration_num / iter_per_epoch)
already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR)
if already_trained_iteration >= iteration_num:
basic.outputlogMessage('Training already run %d iterations, skip' %
already_trained_iteration)
return True
if validation_interval is None:
basic.outputlogMessage(
'No input validation_interval, so training to %d, then evaluating in the end'
% iteration_num)
# run training
train_deeplab(train_script, dataset, train_split, num_of_classes,
base_learning_rate, model_variant, init_checkpoint,
TRAIN_LOGDIR, dataset_dir, gpu_num, train_atrous_rates1,
train_atrous_rates2, train_atrous_rates3,
train_output_stride, crop_size_str, batch_size,
iteration_num, depth_multiplier, decoder_output_stride,
aspp_convs_filters, b_initialize_last_layer)
# run evaluation
evaluation_deeplab(evl_script, dataset, evl_split, num_of_classes,
model_variant, inf_atrous_rates1, inf_atrous_rates2,
inf_atrous_rates3, inf_output_stride, TRAIN_LOGDIR,
EVAL_LOGDIR, dataset_dir, crop_size_str,
max_eva_number, depth_multiplier,
decoder_output_stride, aspp_convs_filters)
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
get_loss_learning_rate_list(TRAIN_LOGDIR)
else:
basic.outputlogMessage(
'training to the maximum iteration of %d, and evaluating very %d epoch(es)'
% (iteration_num, validation_interval))
for epoch in range(validation_interval,
total_epoches + validation_interval,
validation_interval):
to_iter_num = min(epoch * iter_per_epoch, iteration_num)
if to_iter_num <= already_trained_iteration:
continue
basic.outputlogMessage(
'training and evaluating to %d epoches (to iteration: %d)' %
(epoch, to_iter_num))
# run training
train_deeplab(train_script, dataset, train_split, num_of_classes,
base_learning_rate, model_variant, init_checkpoint,
TRAIN_LOGDIR, dataset_dir, gpu_num,
train_atrous_rates1, train_atrous_rates2,
train_atrous_rates3, train_output_stride,
crop_size_str, batch_size, to_iter_num,
depth_multiplier, decoder_output_stride,
aspp_convs_filters, b_initialize_last_layer)
# run evaluation
evaluation_deeplab(evl_script, dataset, evl_split, num_of_classes,
model_variant, inf_atrous_rates1,
inf_atrous_rates2, inf_atrous_rates3,
inf_output_stride, TRAIN_LOGDIR, EVAL_LOGDIR,
dataset_dir, crop_size_str, max_eva_number,
depth_multiplier, decoder_output_stride,
aspp_convs_filters)
# get miou
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
# save loss value to disk
get_loss_learning_rate_list(TRAIN_LOGDIR)
# check if need to early stopping
if b_early_stopping:
if len(miou_dict['overall']) >= 5:
# if the last five miou did not improve, then stop training
if np.all(np.diff(miou_dict['overall'][-5:]) < 0.005
): # 0.0001 (%0.01) # 0.5 %
basic.outputlogMessage(
'early stopping: stop training because overall miou did not improved in the last five evaluation'
)
output_early_stopping_message(TRAIN_LOGDIR)
break
# plot mIOU, loss, and learnint rate curves
iou_path = os.path.join(EVAL_LOGDIR, 'miou.txt')
loss_path = os.path.join(TRAIN_LOGDIR, 'loss_learning_rate.txt')
miou_curve_path = plot_miou_loss_curve.plot_miou_loss_main(
iou_path,
train_count=train_count,
val_count=val_count,
batch_size=batch_size)
loss_curve_path = plot_miou_loss_curve.plot_miou_loss_main(
loss_path,
train_count=train_count,
val_count=val_count,
batch_size=batch_size)
# backup miou and training_loss & learning rate
test_id = os.path.basename(WORK_DIR) + '_' + expr_name
backup_dir = os.path.join(WORK_DIR, 'result_backup')
if os.path.isdir(backup_dir) is False:
io_function.mkdir(backup_dir)
new_iou_name = os.path.join(backup_dir,
test_id + '_' + os.path.basename(iou_path))
io_function.copy_file_to_dst(iou_path, new_iou_name, overwrite=True)
miou_curve_bakname = os.path.join(
backup_dir, test_id + '_' + os.path.basename(miou_curve_path))
io_function.copy_file_to_dst(miou_curve_path,
miou_curve_bakname,
overwrite=True)
loss_new_name = os.path.join(backup_dir,
test_id + '_' + os.path.basename(loss_path))
io_function.copy_file_to_dst(loss_path, loss_new_name, overwrite=True)
loss_curve_bakname = os.path.join(
backup_dir, test_id + '_' + os.path.basename(loss_curve_path))
io_function.copy_file_to_dst(loss_curve_path,
loss_curve_bakname,
overwrite=True)
def train_evaluation_deeplab_separate(WORK_DIR, deeplab_dir, expr_name,
para_file, network_setting_ini, gpu_num):
'''
in "train_evaluation_deeplab", run training, stop, then evaluation, then traininng, make learning rate strange, and the results worse.
so in this function, we start two process, one for training, another for evaluation (run on CPU)
'''
# prepare training folder
EXP_FOLDER = expr_name
INIT_FOLDER = os.path.join(WORK_DIR, EXP_FOLDER, 'init_models')
TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train')
EVAL_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'eval')
VIS_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'vis')
EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export')
io_function.mkdir(INIT_FOLDER)
io_function.mkdir(TRAIN_LOGDIR)
io_function.mkdir(EVAL_LOGDIR)
io_function.mkdir(VIS_LOGDIR)
io_function.mkdir(EXPORT_DIR)
# prepare the tensorflow check point (pretrained model) for training
pre_trained_dir = parameters.get_directory_None_if_absence(
network_setting_ini, 'pre_trained_model_folder')
pre_trained_tar = parameters.get_string_parameters(network_setting_ini,
'TF_INIT_CKPT')
pre_trained_path = os.path.join(pre_trained_dir, pre_trained_tar)
if os.path.isfile(pre_trained_path) is False:
print('pre-trained model: %s not exist, try to download' %
pre_trained_path)
# try to download the file
pre_trained_url = parameters.get_string_parameters_None_if_absence(
network_setting_ini, 'pre_trained_model_url')
res = os.system('wget %s ' % pre_trained_url)
if res != 0:
sys.exit(1)
io_function.movefiletodir(pre_trained_tar, pre_trained_dir)
# unpack pre-trained model to INIT_FOLDER
os.chdir(INIT_FOLDER)
res = os.system('tar -xf %s' % pre_trained_path)
if res != 0:
raise IOError('failed to unpack %s' % pre_trained_path)
os.chdir(WORK_DIR)
dataset_dir = os.path.join(WORK_DIR, 'tfrecord')
batch_size = parameters.get_digit_parameters(network_setting_ini,
'batch_size', 'int')
# maximum iteration number
iteration_num = parameters.get_digit_parameters(network_setting_ini,
'iteration_num', 'int')
base_learning_rate = parameters.get_digit_parameters(
network_setting_ini, 'base_learning_rate', 'float')
train_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_output_stride', 'int')
train_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates1', 'int')
train_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates2', 'int')
train_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'train_atrous_rates3', 'int')
inf_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_output_stride', 'int')
inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates1', 'int')
inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates2', 'int')
inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates3', 'int')
# depth_multiplier default is 1.0.
depth_multiplier = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'depth_multiplier', 'float')
decoder_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'decoder_output_stride', 'int')
aspp_convs_filters = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'aspp_convs_filters', 'int')
train_script = os.path.join(deeplab_dir, 'train.py')
train_split = os.path.splitext(
parameters.get_string_parameters(para_file,
'training_sample_list_txt'))[0]
model_variant = parameters.get_string_parameters(network_setting_ini,
'model_variant')
checkpoint = parameters.get_string_parameters(network_setting_ini,
'tf_initial_checkpoint')
init_checkpoint_files = io_function.get_file_list_by_pattern(
INIT_FOLDER, checkpoint + '*')
if len(init_checkpoint_files) < 1:
raise IOError('No initial checkpoint in %s with pattern: %s' %
(INIT_FOLDER, checkpoint))
init_checkpoint = os.path.join(INIT_FOLDER, checkpoint)
b_early_stopping = parameters.get_bool_parameters(para_file,
'b_early_stopping')
b_initialize_last_layer = parameters.get_bool_parameters(
para_file, 'b_initialize_last_layer')
dataset = parameters.get_string_parameters(para_file, 'dataset_name')
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
assert num_classes_noBG != None
if b_initialize_last_layer is True:
if pre_trained_tar in pre_trained_tar_21_classes:
print(
'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21'
% pre_trained_tar)
num_classes_noBG = 20
if pre_trained_tar in pre_trained_tar_19_classes:
print(
'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19'
% pre_trained_tar)
num_classes_noBG = 18
num_of_classes = num_classes_noBG + 1
image_crop_size = parameters.get_string_list_parameters(
para_file, 'image_crop_size')
if len(image_crop_size) != 2 and image_crop_size[0].isdigit(
) and image_crop_size[1].isdigit():
raise ValueError('image_crop_size should be height,width')
crop_size_str = ','.join(image_crop_size)
# validation interval (epoch), do
# validation_interval = parameters.get_digit_parameters_None_if_absence(para_file,'validation_interval','int')
train_count, val_count = get_train_val_sample_count(WORK_DIR, para_file)
iter_per_epoch = math.ceil(train_count / batch_size)
total_epoches = math.ceil(iteration_num / iter_per_epoch)
already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR)
if already_trained_iteration >= iteration_num:
basic.outputlogMessage('Training already run %d iterations, skip' %
already_trained_iteration)
return True
save_interval_secs = 1200 # default is 1200 second for saving model
save_summaries_secs = 600 # default is 600 second for saving summaries
eval_interval_secs = save_interval_secs # default is 300 second for running evaluation, if no new saved model, no need to run evaluation?
train_process = Process(
target=train_deeplab,
args=(train_script, dataset, train_split, num_of_classes,
base_learning_rate, model_variant, init_checkpoint, TRAIN_LOGDIR,
dataset_dir, gpu_num, train_atrous_rates1, train_atrous_rates2,
train_atrous_rates3, train_output_stride, crop_size_str,
batch_size, iteration_num, depth_multiplier,
decoder_output_stride, aspp_convs_filters,
b_initialize_last_layer))
train_process.start()
time.sleep(60) # wait
if train_process.exitcode is not None and train_process.exitcode != 0:
sys.exit(1)
# eval_process.start()
# time.sleep(10) # wait
# if eval_process.exitcode is not None and eval_process.exitcode != 0:
# sys.exit(1)
while True:
# only run evaluation when there is new trained model
already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR)
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
basic.outputlogMessage(
'Already trained iteration: %d, latest evaluation at %d step' %
(already_trained_iteration, miou_dict['step'][-1]))
if already_trained_iteration > miou_dict['step'][-1]:
# run evaluation and wait until it finished
gpuid = "" # set gpuid to empty string, making evaluation run on CPU
evl_script = os.path.join(deeplab_dir, 'eval.py')
evl_split = os.path.splitext(
parameters.get_string_parameters(
para_file, 'validation_sample_list_txt'))[0]
# max_eva_number = -1 # run as many evaluation as possible, --eval_interval_secs (default is 300 seconds)
max_eva_number = 1 # only run once inside the while loop, use while loop to control multiple evaluation
eval_process = Process(
target=evaluation_deeplab,
args=(evl_script, dataset, evl_split, num_of_classes,
model_variant, inf_atrous_rates1, inf_atrous_rates2,
inf_atrous_rates3, inf_output_stride, TRAIN_LOGDIR,
EVAL_LOGDIR, dataset_dir, crop_size_str, max_eva_number,
depth_multiplier, decoder_output_stride,
aspp_convs_filters, gpuid, eval_interval_secs))
eval_process.start(
) # put Process inside while loop to avoid error: AssertionError: cannot start a process twice
while eval_process.is_alive():
time.sleep(5)
# check if need early stopping
if b_early_stopping:
print(datetime.now(), 'check early stopping')
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
if 'overall' in miou_dict.keys() and len(
miou_dict['overall']) >= 5:
# if the last five miou did not improve, then stop training
if np.all(np.diff(miou_dict['overall'][-5:]) < 0.005
): # 0.0001 (%0.01) # 0.5 %
basic.outputlogMessage(
'early stopping: stop training because overall miou did not improved in the last five evaluation'
)
output_early_stopping_message(TRAIN_LOGDIR)
# train_process.kill() # this one seems not working
# subprocess pid different from ps output
# https://stackoverflow.com/questions/4444141/subprocess-pid-different-from-ps-output
# os.system('kill ' + str(train_process.pid)) # still not working. train_process.pid is not the one output by ps -aux
# train_process.terminate() # Note that descendant processes of the process will not be terminated
# train_process.join() # Wait until child process terminates
with open('train_py_pid.txt', 'r') as f_obj:
lines = f_obj.readlines()
train_pid = int(lines[0].strip())
os.system('kill ' + str(train_pid))
basic.outputlogMessage(
'kill training processing with id: %d' % train_pid)
break # this breaks the while loop, making that it may not evaluate on some new saved model.
# if the evaluation step is less than saved model iteration, run another iteration again immediately
already_trained_iteration = get_trained_iteration(TRAIN_LOGDIR)
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
if already_trained_iteration > miou_dict['step'][-1]:
continue
# if finished training
if train_process.is_alive() is False:
break
# # if eval_process exit, then quit training as well
# if eval_process.is_alive() is False and train_process.is_alive():
# train_process.kill()
# break
time.sleep(eval_interval_secs) # wait for next evaluation
# save loss value to disk
get_loss_learning_rate_list(TRAIN_LOGDIR)
# get miou again
miou_dict = get_miou_list_class_all(EVAL_LOGDIR, num_of_classes)
# eval_process did not exit as expected, kill it again.
# os.system('kill ' + str(eval_process.pid))
# get iou and backup
iou_path = os.path.join(EVAL_LOGDIR, 'miou.txt')
loss_path = os.path.join(TRAIN_LOGDIR, 'loss_learning_rate.txt')
patch_info = os.path.join(WORK_DIR, 'sub_images_patches_info.txt')
# backup miou and training_loss & learning rate
test_id = os.path.basename(WORK_DIR) + '_' + expr_name
backup_dir = os.path.join(WORK_DIR, 'result_backup')
if os.path.isdir(backup_dir) is False:
io_function.mkdir(backup_dir)
new_iou_name = os.path.join(backup_dir,
test_id + '_' + os.path.basename(iou_path))
io_function.copy_file_to_dst(iou_path, new_iou_name, overwrite=True)
loss_new_name = os.path.join(backup_dir,
test_id + '_' + os.path.basename(loss_path))
io_function.copy_file_to_dst(loss_path, loss_new_name, overwrite=True)
new_patch_info = os.path.join(backup_dir,
test_id + '_' + os.path.basename(patch_info))
io_function.copy_file_to_dst(patch_info, new_patch_info, overwrite=True)
# plot mIOU, loss, and learnint rate curves, and backup
miou_curve_path = plot_miou_loss_curve.plot_miou_loss_main(
iou_path,
train_count=train_count,
val_count=val_count,
batch_size=batch_size)
loss_curve_path = plot_miou_loss_curve.plot_miou_loss_main(
loss_path,
train_count=train_count,
val_count=val_count,
batch_size=batch_size)
miou_curve_bakname = os.path.join(
backup_dir, test_id + '_' + os.path.basename(miou_curve_path))
io_function.copy_file_to_dst(miou_curve_path,
miou_curve_bakname,
overwrite=True)
loss_curve_bakname = os.path.join(
backup_dir, test_id + '_' + os.path.basename(loss_curve_path))
io_function.copy_file_to_dst(loss_curve_path,
loss_curve_bakname,
overwrite=True)
def remove_polygons_main(polygons_shp, output, para_file):
polygons_shp_backup = polygons_shp
assert io_function.is_file_exist(polygons_shp)
# remove polygons based on area
# area_thr = 1000 #10 pixels
area_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_area', 'int')
b_smaller = True
if area_thr is not None:
rm_area_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmArea')
if remove_polygons(polygons_shp, 'INarea', area_thr, b_smaller,
rm_area_save_shp) is False:
basic.outputlogMessage(
"error, removing polygons based on size failed")
else:
polygons_shp = rm_area_save_shp
else:
basic.outputlogMessage(
'warning, minimum_area is absent in the para file, skip removing polygons based on areas'
)
# remove polygons based on slope information
# slope_small_thr = 2
slope_small_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_slope', 'float')
b_smaller = True
if slope_small_thr is not None:
rm_slope_save_shp1 = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmslope1')
if remove_polygons(polygons_shp, 'slo_mean', slope_small_thr,
b_smaller, rm_slope_save_shp1) is False:
basic.outputlogMessage(
"error, removing polygons based on slo_mean failed")
else:
polygons_shp = rm_slope_save_shp1
else:
basic.outputlogMessage(
'warning, minimum_slope is absent in the para file, skip removing polygons based on minimum slope'
)
# slope_large_thr = 20
slope_large_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'maximum_slope', 'float')
b_smaller = False
if slope_large_thr is not None:
rm_slope_save_shp2 = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmslope2')
if remove_polygons(polygons_shp, 'slo_mean', slope_large_thr,
b_smaller, rm_slope_save_shp2) is False:
basic.outputlogMessage(
"error, removing polygons based on slo_mean (2) failed")
else:
polygons_shp = rm_slope_save_shp2
else:
basic.outputlogMessage(
'warning, maximum_slope is absent in the para file, skip removing polygons based on maximum slope'
)
# remove polygons based on dem
# dem_small_thr = 3000
dem_small_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_elevation', 'int')
b_smaller = True
if dem_small_thr is not None:
rm_dem_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmDEM')
if remove_polygons(polygons_shp, 'dem_mean', dem_small_thr, b_smaller,
rm_dem_save_shp) is False:
basic.outputlogMessage(
"error, removing polygons based on dem_mean failed")
else:
polygons_shp = rm_dem_save_shp
else:
basic.outputlogMessage(
'warning, minimum_elevation is absent in the para file, skip removing polygons based on minimum elevation'
)
ratio_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_ratio_width_height', 'float')
b_smaller = True
if ratio_thr is not None:
rm_ratio_w_h_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmRwh')
if remove_polygons(polygons_shp, 'ratio_w_h', ratio_thr, b_smaller,
rm_ratio_w_h_save_shp) is False:
basic.outputlogMessage(
"error, removing polygons based on ratio_w_h failed")
else:
polygons_shp = rm_ratio_w_h_save_shp
else:
basic.outputlogMessage(
'warning, minimum_ratio_width_height is absent in the para file, skip removing polygons based on minimum ratio of width and height'
)
# remove polygons based elevation reduction
minimum_dem_reduction_area_thr = parameters.get_digit_parameters_None_if_absence(
para_file, 'minimum_dem_reduction_area', 'float')
b_smaller = True
if minimum_dem_reduction_area_thr is not None:
rm_demD_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmdemD')
if remove_polygons(polygons_shp, 'demD_area',
minimum_dem_reduction_area_thr, b_smaller,
rm_demD_save_shp) is False:
basic.outputlogMessage(
"error, removing polygons based on demD_area failed")
else:
polygons_shp = rm_demD_save_shp
else:
basic.outputlogMessage(
'warning, minimum_dem_reduction_area is absent in the para file, skip removing polygons based on minimum_dem_reduction_area'
)
# remove polygons based on occurrence
min_ocurr = parameters.get_digit_parameters_None_if_absence(
para_file, 'threshold_occurrence_multi_observation', 'int')
b_smaller = True
if min_ocurr is not None:
rm_occur_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'RmOccur')
if remove_polygons(polygons_shp, 'time_occur', min_ocurr, b_smaller,
rm_occur_save_shp) is False:
basic.outputlogMessage(
"error, removing polygons based on time_occur failed")
else:
polygons_shp = rm_occur_save_shp
else:
basic.outputlogMessage(
'warning, threshold_occurrence_multi_observation is absent in the para file, '
'skip removing polygons based on it')
# remove polygons not in the extent
outline_shp = parameters.get_string_parameters_None_if_absence(
para_file, 'target_outline_shp')
if outline_shp is not None:
rm_outline_save_shp = io_function.get_name_by_adding_tail(
polygons_shp_backup, 'rmOutline')
remove_polygons_outside_extent(polygons_shp, outline_shp,
rm_outline_save_shp)
polygons_shp = rm_outline_save_shp
else:
basic.outputlogMessage(
'warning, target_outline_shp is absent in the para file, skip removing polygons based on outlines'
)
# copy to final output
copy_shape_file(polygons_shp, output)
pass
