def run_evaluation_main(para_file,
b_new_validation_data=False,
train_dir=None):
print("run evaluation")
SECONDS = time.time()
gpu_num = 1
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
network_setting_ini = parameters.get_string_parameters(
para_file, 'network_setting_ini')
tf_research_dir = parameters.get_directory_None_if_absence(
network_setting_ini, 'tf_research_dir')
print(tf_research_dir)
if tf_research_dir is None:
raise ValueError('tf_research_dir is not in %s' % para_file)
if os.path.isdir(tf_research_dir) is False:
raise ValueError('%s does not exist' % tf_research_dir)
# sys.path.insert(0, tf_research_dir)
# sys.path.insert(0, os.path.join(tf_research_dir,'slim'))
# print(sys.path)
# need to change PYTHONPATH, otherwise, deeplab cannot be found
if os.getenv('PYTHONPATH'):
os.environ['PYTHONPATH'] = os.getenv(
'PYTHONPATH') + ':' + tf_research_dir + ':' + os.path.join(
tf_research_dir, 'slim')
else:
os.environ['PYTHONPATH'] = tf_research_dir + ':' + os.path.join(
tf_research_dir, 'slim')
# os.system('echo $PYTHONPATH ')
tf1x_python = parameters.get_file_path_parameters(network_setting_ini,
'tf1x_python')
deeplab_train.tf1x_python = tf1x_python
deeplab_dir = os.path.join(tf_research_dir, 'deeplab')
WORK_DIR = os.getcwd()
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
# prepare data for validation
if b_new_validation_data:
prepare_data_for_evaluation(para_file)
run_evaluation(WORK_DIR,
deeplab_dir,
expr_name,
para_file,
network_setting_ini,
gpu_num,
train_dir=train_dir)
duration = time.time() - SECONDS
os.system(
'echo "$(date): time cost of running evaluation: %.2f seconds">>time_cost.txt'
% duration)
def main(options, args):
para_file = args[0]
network_setting_ini = parameters.get_string_parameters(
para_file, 'network_setting_ini')
global tf1x_python
tf1x_python = parameters.get_file_path_parameters(network_setting_ini,
'tf1x_python')
build_script = os.path.join(code_dir, 'datasets', 'build_TFrecord.py')
call_build_TFrecord(build_script, para_file)
def mmseg_train_main(para_file, gpu_num):
print(datetime.now(), "train MMSegmentation")
SECONDS = time.time()
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in the current folder: %s' %
(para_file, os.getcwd()))
network_setting_ini = parameters.get_string_parameters(
para_file, 'network_setting_ini')
mmseg_repo_dir = parameters.get_directory(network_setting_ini,
'mmseg_repo_dir')
mmseg_config_dir = osp.join(mmseg_repo_dir, 'configs')
if os.path.isdir(mmseg_config_dir) is False:
raise ValueError('%s does not exist' % mmseg_config_dir)
base_config_file = parameters.get_string_parameters(
network_setting_ini, 'base_config')
base_config_file = os.path.join(mmseg_config_dir, base_config_file)
if os.path.isfile(base_config_file) is False:
raise IOError('%s does not exist' % base_config_file)
global open_mmlab_python
open_mmlab_python = parameters.get_file_path_parameters(
network_setting_ini, 'open-mmlab-python')
WORK_DIR = os.getcwd()
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
# copy the base_config_file, then save to to a new one
config_file = osp.join(
WORK_DIR,
osp.basename(
io_function.get_name_by_adding_tail(base_config_file, expr_name)))
if updated_config_file(WORK_DIR, expr_name, base_config_file, config_file,
para_file, network_setting_ini, gpu_num) is False:
raise ValueError('Getting the config file failed')
train_evaluation_mmseg(WORK_DIR, mmseg_repo_dir, config_file, expr_name,
para_file, network_setting_ini, gpu_num)
duration = time.time() - SECONDS
os.system(
'echo "$(date): time cost of training: %.2f seconds">>time_cost.txt' %
duration)
def main():
para_file = 'main_para.ini'
io_function.is_file_exist(para_file)
area_ini_list = [
'area_Willow_River.ini', 'area_Banks_east_nirGB.ini',
'area_Ellesmere_Island_nirGB.ini'
]
for area_ini in area_ini_list:
io_function.is_file_exist(area_ini)
ground_truth_shp = parameters.get_file_path_parameters(
area_ini, 'validation_shape')
# save to current folder
save_info_shp = io_function.get_name_by_adding_tail(
ground_truth_shp, 'post')
save_info_shp = os.path.join(os.getcwd(),
os.path.basename(save_info_shp))
if os.path.isfile(save_info_shp):
basic.outputlogMessage('%s already exist, skip %s' %
(save_info_shp, area_ini))
continue
add_polygon_attributes(ground_truth_shp, save_info_shp, para_file,
area_ini)
def main(options, args):
print(
"%s : prediction using the trained model (run parallel if use multiple GPUs) "
% os.path.basename(sys.argv[0]))
machine_name = os.uname()[1]
start_time = datetime.datetime.now()
para_file = args[0]
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
basic.setlogfile('parallel_predict_Log.txt')
deeplab_inf_script = os.path.join(code_dir, 'deeplabBased',
'deeplab_inference.py')
network_setting_ini = parameters.get_string_parameters(
para_file, 'network_setting_ini')
global tf1x_python
tf1x_python = parameters.get_file_path_parameters(network_setting_ini,
'tf1x_python')
trained_model = options.trained_model
outdir = parameters.get_directory(para_file, 'inf_output_dir')
# remove previous results (let user remove this folder manually or in exe.sh folder)
io_function.mkdir(outdir)
# get name of inference areas
multi_inf_regions = parameters.get_string_list_parameters(
para_file, 'inference_regions')
# max_parallel_inf_task = parameters.get_digit_parameters(para_file,'max_parallel_inf_task','int')
b_use_multiGPUs = parameters.get_bool_parameters(para_file,
'b_use_multiGPUs')
# loop each inference regions
sub_tasks = []
for area_idx, area_ini in enumerate(multi_inf_regions):
area_name = parameters.get_string_parameters(area_ini, 'area_name')
area_remark = parameters.get_string_parameters(area_ini, 'area_remark')
area_time = parameters.get_string_parameters(area_ini, 'area_time')
inf_image_dir = parameters.get_directory(area_ini, 'inf_image_dir')
# it is ok consider a file name as pattern and pass it the following functions to get file list
inf_image_or_pattern = parameters.get_string_parameters(
area_ini, 'inf_image_or_pattern')
inf_img_list = io_function.get_file_list_by_pattern(
inf_image_dir, inf_image_or_pattern)
img_count = len(inf_img_list)
if img_count < 1:
raise ValueError(
'No image for inference, please check inf_image_dir and inf_image_or_pattern in %s'
% area_ini)
area_save_dir = os.path.join(
outdir, area_name + '_' + area_remark + '_' + area_time)
io_function.mkdir(area_save_dir)
# parallel inference images for this area
CUDA_VISIBLE_DEVICES = []
if 'CUDA_VISIBLE_DEVICES' in os.environ.keys():
CUDA_VISIBLE_DEVICES = [
int(item.strip())
for item in os.environ['CUDA_VISIBLE_DEVICES'].split(',')
]
idx = 0
while idx < img_count:
if b_use_multiGPUs:
# get available GPUs # https://github.com/anderskm/gputil
deviceIDs = GPUtil.getAvailable(order='first',
limit=100,
maxLoad=0.5,
maxMemory=0.5,
includeNan=False,
excludeID=[],
excludeUUID=[])
# only use the one in CUDA_VISIBLE_DEVICES
if len(CUDA_VISIBLE_DEVICES) > 0:
deviceIDs = [
item for item in deviceIDs
if item in CUDA_VISIBLE_DEVICES
]
basic.outputlogMessage('on ' + machine_name +
', available GPUs:' +
str(deviceIDs) +
', among visible ones:' +
str(CUDA_VISIBLE_DEVICES))
else:
basic.outputlogMessage('on ' + machine_name +
', available GPUs:' +
str(deviceIDs))
if len(deviceIDs) < 1:
time.sleep(
60
) # wait one minute, then check the available GPUs again
continue
# set only the first available visible
gpuid = deviceIDs[0]
basic.outputlogMessage(
'%d: predict image %s on GPU %d of %s' %
(idx, inf_img_list[idx], gpuid, machine_name))
else:
gpuid = None
basic.outputlogMessage('%d: predict image %s on %s' %
(idx, inf_img_list[idx], machine_name))
# run inference
img_save_dir = os.path.join(area_save_dir, 'I%d' % idx)
inf_list_file = os.path.join(area_save_dir, '%d.txt' % idx)
# if it already exist, then skip
if os.path.isdir(img_save_dir) and is_file_exist_in_folder(
img_save_dir):
basic.outputlogMessage(
'folder of %dth image (%s) already exist, '
'it has been predicted or is being predicted' %
(idx, inf_img_list[idx]))
idx += 1
continue
with open(inf_list_file, 'w') as inf_obj:
inf_obj.writelines(inf_img_list[idx] + '\n')
sub_process = Process(target=predict_one_image_deeplab,
args=(deeplab_inf_script, para_file,
network_setting_ini, img_save_dir,
inf_list_file, gpuid, trained_model))
sub_process.start()
sub_tasks.append(sub_process)
if b_use_multiGPUs is False:
# wait until previous one finished
while sub_process.is_alive():
time.sleep(5)
idx += 1
# wait until predicted image patches exist or exceed 20 minutes
time0 = time.time()
elapsed_time = time.time() - time0
while elapsed_time < 20 * 60:
elapsed_time = time.time() - time0
file_exist = is_file_exist_in_folder(img_save_dir)
if file_exist is True or sub_process.is_alive() is False:
break
else:
time.sleep(5)
if sub_process.exitcode is not None and sub_process.exitcode != 0:
sys.exit(1)
# if 'chpc' in machine_name:
# time.sleep(60) # wait 60 second on ITSC services
# else:
# time.sleep(10)
# check all the tasks already finished
while b_all_task_finish(sub_tasks) is False:
basic.outputlogMessage('wait all tasks to finish')
time.sleep(60)
end_time = datetime.datetime.now()
diff_time = end_time - start_time
out_str = "%s: time cost of total parallel inference on %s: %d seconds" % (
str(end_time), machine_name, diff_time.seconds)
basic.outputlogMessage(out_str)
with open("time_cost.txt", 'a') as t_obj:
t_obj.writelines(out_str + '\n')
def main(options, args):
print("%s : export the frozen inference graph" %
os.path.basename(sys.argv[0]))
para_file = args[0]
if os.path.isfile(para_file) is False:
raise IOError('File %s not exists in current folder: %s' %
(para_file, os.getcwd()))
network_setting_ini = parameters.get_string_parameters(
para_file, 'network_setting_ini')
tf_research_dir = parameters.get_directory_None_if_absence(
network_setting_ini, 'tf_research_dir')
print(tf_research_dir)
if tf_research_dir is None:
raise ValueError('tf_research_dir is not in %s' % para_file)
if os.path.isdir(tf_research_dir) is False:
raise ValueError('%s does not exist' % tf_research_dir)
if os.getenv('PYTHONPATH'):
os.environ['PYTHONPATH'] = os.getenv(
'PYTHONPATH') + ':' + tf_research_dir + ':' + os.path.join(
tf_research_dir, 'slim')
else:
os.environ['PYTHONPATH'] = tf_research_dir + ':' + os.path.join(
tf_research_dir, 'slim')
global tf1x_python
tf1x_python = parameters.get_file_path_parameters(network_setting_ini,
'tf1x_python')
deeplab_dir = os.path.join(tf_research_dir, 'deeplab')
WORK_DIR = os.getcwd()
expr_name = parameters.get_string_parameters(para_file, 'expr_name')
EXP_FOLDER = expr_name
TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train')
EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export')
inf_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_output_stride', 'int')
inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates1', 'int')
inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates2', 'int')
inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'inf_atrous_rates3', 'int')
# depth_multiplier default is 1.0.
depth_multiplier = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'depth_multiplier', 'float')
decoder_output_stride = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'decoder_output_stride', 'int')
aspp_convs_filters = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'aspp_convs_filters', 'int')
model_variant = parameters.get_string_parameters(network_setting_ini,
'model_variant')
num_classes_noBG = parameters.get_digit_parameters_None_if_absence(
para_file, 'NUM_CLASSES_noBG', 'int')
assert num_classes_noBG != None
b_initialize_last_layer = parameters.get_bool_parameters(
para_file, 'b_initialize_last_layer')
if b_initialize_last_layer is False:
pre_trained_tar = parameters.get_string_parameters(
network_setting_ini, 'TF_INIT_CKPT')
if pre_trained_tar in pre_trained_tar_21_classes:
print(
'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21'
% pre_trained_tar)
num_classes_noBG = 20
if pre_trained_tar in pre_trained_tar_19_classes:
print(
'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19'
% pre_trained_tar)
num_classes_noBG = 18
num_of_classes = num_classes_noBG + 1
image_crop_size = parameters.get_string_list_parameters(
para_file, 'image_crop_size')
if len(image_crop_size) != 2 and image_crop_size[0].isdigit(
) and image_crop_size[1].isdigit():
raise ValueError('image_crop_size should be height,width')
iteration_num = get_trained_iteration(TRAIN_LOGDIR)
multi_scale = parameters.get_digit_parameters_None_if_absence(
network_setting_ini, 'export_multi_scale', 'int')
export_script = os.path.join(deeplab_dir, 'export_model.py')
CKPT_PATH = os.path.join(TRAIN_LOGDIR, 'model.ckpt-%s' % iteration_num)
EXPORT_PATH = os.path.join(EXPORT_DIR,
'frozen_inference_graph_%s.pb' % iteration_num)
if os.path.isfile(EXPORT_PATH):
basic.outputlogMessage('%s exists, skipping exporting models' %
EXPORT_PATH)
return
export_graph(export_script, CKPT_PATH, EXPORT_PATH, model_variant,
num_of_classes, inf_atrous_rates1, inf_atrous_rates2,
inf_atrous_rates3, inf_output_stride, image_crop_size[0],
image_crop_size[1], multi_scale, depth_multiplier,
decoder_output_stride, aspp_convs_filters)
def image_translate_train_generate_one_domain(gan_working_dir, gan_para_file,
area_src_ini, area_gan_ini,
gpu_ids, domainB_imgList):
current_dir = os.getcwd()
# get orignal sub-images
_, _, area_ini_sub_images_labels_dict = original_sub_images_labels_list_before_gan(
)
sub_img_label_txt = os.path.join(current_dir,
area_ini_sub_images_labels_dict)
if os.path.isfile(area_ini_sub_images_labels_dict) is False:
raise IOError(
'%s not in the current folder, please get subImages first' %
sub_img_label_txt)
# prepare image list of domain A
# what if the size of some images are not fit with CUT input?
domain_A_images = []
# domain_A_labels = []
# with open(sub_img_label_txt) as txt_obj:
# line_list = [name.strip() for name in txt_obj.readlines()]
# for line in line_list:
# sub_image, sub_label = line.split(':')
# domain_A_images.append(os.path.join(current_dir,sub_image))
# # domain_A_labels.append(os.path.join(current_dir,sub_label))
area_ini_sub_images_labels = io_function.read_dict_from_txt_json(
'area_ini_sub_images_labels.txt')
for line in area_ini_sub_images_labels[os.path.basename(area_src_ini)]:
sub_image, sub_label = line.split(':')
domain_A_images.append(os.path.join(current_dir, sub_image))
# domain_A_labels.append(os.path.join(current_dir,sub_label))
os.chdir(gan_working_dir)
io_function.save_list_to_txt('image_A_list.txt', domain_A_images)
# read target images, that will consider as target domains
# what if there are too many images in domain B?
io_function.save_list_to_txt('image_B_list.txt', domainB_imgList)
gan_python = parameters.get_file_path_parameters(gan_para_file, 'python')
cut_dir = parameters.get_directory(gan_para_file, 'gan_script_dir')
train_script = os.path.join(cut_dir, 'train.py')
generate_script = os.path.join(cut_dir, 'generate_image.py')
# training of CUT
if train_CUT_gan(gan_python, train_script, gan_para_file,
gpu_ids) is False:
os.chdir(current_dir)
return False
# genenerate image using CUT, convert images in domain A to domain B
save_tran_img_folder = 'subImages_translate'
if generate_image_CUT(gan_python, generate_script, gan_para_file, gpu_ids,
domain_A_images, save_tran_img_folder) is False:
os.chdir(current_dir)
return False
# change working directory back
os.chdir(current_dir)
pass
