def run_evaluation_main(para_file, b_new_validation_data=False, train_dir=None): print("run evaluation") SECONDS = time.time() gpu_num = 1 if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd())) network_setting_ini = parameters.get_string_parameters( para_file, 'network_setting_ini') tf_research_dir = parameters.get_directory_None_if_absence( network_setting_ini, 'tf_research_dir') print(tf_research_dir) if tf_research_dir is None: raise ValueError('tf_research_dir is not in %s' % para_file) if os.path.isdir(tf_research_dir) is False: raise ValueError('%s does not exist' % tf_research_dir) # sys.path.insert(0, tf_research_dir) # sys.path.insert(0, os.path.join(tf_research_dir,'slim')) # print(sys.path) # need to change PYTHONPATH, otherwise, deeplab cannot be found if os.getenv('PYTHONPATH'): os.environ['PYTHONPATH'] = os.getenv( 'PYTHONPATH') + ':' + tf_research_dir + ':' + os.path.join( tf_research_dir, 'slim') else: os.environ['PYTHONPATH'] = tf_research_dir + ':' + os.path.join( tf_research_dir, 'slim') # os.system('echo $PYTHONPATH ') tf1x_python = parameters.get_file_path_parameters(network_setting_ini, 'tf1x_python') deeplab_train.tf1x_python = tf1x_python deeplab_dir = os.path.join(tf_research_dir, 'deeplab') WORK_DIR = os.getcwd() expr_name = parameters.get_string_parameters(para_file, 'expr_name') # prepare data for validation if b_new_validation_data: prepare_data_for_evaluation(para_file) run_evaluation(WORK_DIR, deeplab_dir, expr_name, para_file, network_setting_ini, gpu_num, train_dir=train_dir) duration = time.time() - SECONDS os.system( 'echo "$(date): time cost of running evaluation: %.2f seconds">>time_cost.txt' % duration)
def main(options, args): para_file = args[0] network_setting_ini = parameters.get_string_parameters( para_file, 'network_setting_ini') global tf1x_python tf1x_python = parameters.get_file_path_parameters(network_setting_ini, 'tf1x_python') build_script = os.path.join(code_dir, 'datasets', 'build_TFrecord.py') call_build_TFrecord(build_script, para_file)
def mmseg_train_main(para_file, gpu_num): print(datetime.now(), "train MMSegmentation") SECONDS = time.time() if os.path.isfile(para_file) is False: raise IOError('File %s not exists in the current folder: %s' % (para_file, os.getcwd())) network_setting_ini = parameters.get_string_parameters( para_file, 'network_setting_ini') mmseg_repo_dir = parameters.get_directory(network_setting_ini, 'mmseg_repo_dir') mmseg_config_dir = osp.join(mmseg_repo_dir, 'configs') if os.path.isdir(mmseg_config_dir) is False: raise ValueError('%s does not exist' % mmseg_config_dir) base_config_file = parameters.get_string_parameters( network_setting_ini, 'base_config') base_config_file = os.path.join(mmseg_config_dir, base_config_file) if os.path.isfile(base_config_file) is False: raise IOError('%s does not exist' % base_config_file) global open_mmlab_python open_mmlab_python = parameters.get_file_path_parameters( network_setting_ini, 'open-mmlab-python') WORK_DIR = os.getcwd() expr_name = parameters.get_string_parameters(para_file, 'expr_name') # copy the base_config_file, then save to to a new one config_file = osp.join( WORK_DIR, osp.basename( io_function.get_name_by_adding_tail(base_config_file, expr_name))) if updated_config_file(WORK_DIR, expr_name, base_config_file, config_file, para_file, network_setting_ini, gpu_num) is False: raise ValueError('Getting the config file failed') train_evaluation_mmseg(WORK_DIR, mmseg_repo_dir, config_file, expr_name, para_file, network_setting_ini, gpu_num) duration = time.time() - SECONDS os.system( 'echo "$(date): time cost of training: %.2f seconds">>time_cost.txt' % duration)
def main(): para_file = 'main_para.ini' io_function.is_file_exist(para_file) area_ini_list = [ 'area_Willow_River.ini', 'area_Banks_east_nirGB.ini', 'area_Ellesmere_Island_nirGB.ini' ] for area_ini in area_ini_list: io_function.is_file_exist(area_ini) ground_truth_shp = parameters.get_file_path_parameters( area_ini, 'validation_shape') # save to current folder save_info_shp = io_function.get_name_by_adding_tail( ground_truth_shp, 'post') save_info_shp = os.path.join(os.getcwd(), os.path.basename(save_info_shp)) if os.path.isfile(save_info_shp): basic.outputlogMessage('%s already exist, skip %s' % (save_info_shp, area_ini)) continue add_polygon_attributes(ground_truth_shp, save_info_shp, para_file, area_ini)
def main(options, args): print( "%s : prediction using the trained model (run parallel if use multiple GPUs) " % os.path.basename(sys.argv[0])) machine_name = os.uname()[1] start_time = datetime.datetime.now() para_file = args[0] if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd())) basic.setlogfile('parallel_predict_Log.txt') deeplab_inf_script = os.path.join(code_dir, 'deeplabBased', 'deeplab_inference.py') network_setting_ini = parameters.get_string_parameters( para_file, 'network_setting_ini') global tf1x_python tf1x_python = parameters.get_file_path_parameters(network_setting_ini, 'tf1x_python') trained_model = options.trained_model outdir = parameters.get_directory(para_file, 'inf_output_dir') # remove previous results (let user remove this folder manually or in exe.sh folder) io_function.mkdir(outdir) # get name of inference areas multi_inf_regions = parameters.get_string_list_parameters( para_file, 'inference_regions') # max_parallel_inf_task = parameters.get_digit_parameters(para_file,'max_parallel_inf_task','int') b_use_multiGPUs = parameters.get_bool_parameters(para_file, 'b_use_multiGPUs') # loop each inference regions sub_tasks = [] for area_idx, area_ini in enumerate(multi_inf_regions): area_name = parameters.get_string_parameters(area_ini, 'area_name') area_remark = parameters.get_string_parameters(area_ini, 'area_remark') area_time = parameters.get_string_parameters(area_ini, 'area_time') inf_image_dir = parameters.get_directory(area_ini, 'inf_image_dir') # it is ok consider a file name as pattern and pass it the following functions to get file list inf_image_or_pattern = parameters.get_string_parameters( area_ini, 'inf_image_or_pattern') inf_img_list = io_function.get_file_list_by_pattern( inf_image_dir, inf_image_or_pattern) img_count = len(inf_img_list) if img_count < 1: raise ValueError( 'No image for inference, please check inf_image_dir and inf_image_or_pattern in %s' % area_ini) area_save_dir = os.path.join( outdir, area_name + '_' + area_remark + '_' + area_time) io_function.mkdir(area_save_dir) # parallel inference images for this area CUDA_VISIBLE_DEVICES = [] if 'CUDA_VISIBLE_DEVICES' in os.environ.keys(): CUDA_VISIBLE_DEVICES = [ int(item.strip()) for item in os.environ['CUDA_VISIBLE_DEVICES'].split(',') ] idx = 0 while idx < img_count: if b_use_multiGPUs: # get available GPUs # https://github.com/anderskm/gputil deviceIDs = GPUtil.getAvailable(order='first', limit=100, maxLoad=0.5, maxMemory=0.5, includeNan=False, excludeID=[], excludeUUID=[]) # only use the one in CUDA_VISIBLE_DEVICES if len(CUDA_VISIBLE_DEVICES) > 0: deviceIDs = [ item for item in deviceIDs if item in CUDA_VISIBLE_DEVICES ] basic.outputlogMessage('on ' + machine_name + ', available GPUs:' + str(deviceIDs) + ', among visible ones:' + str(CUDA_VISIBLE_DEVICES)) else: basic.outputlogMessage('on ' + machine_name + ', available GPUs:' + str(deviceIDs)) if len(deviceIDs) < 1: time.sleep( 60 ) # wait one minute, then check the available GPUs again continue # set only the first available visible gpuid = deviceIDs[0] basic.outputlogMessage( '%d: predict image %s on GPU %d of %s' % (idx, inf_img_list[idx], gpuid, machine_name)) else: gpuid = None basic.outputlogMessage('%d: predict image %s on %s' % (idx, inf_img_list[idx], machine_name)) # run inference img_save_dir = os.path.join(area_save_dir, 'I%d' % idx) inf_list_file = os.path.join(area_save_dir, '%d.txt' % idx) # if it already exist, then skip if os.path.isdir(img_save_dir) and is_file_exist_in_folder( img_save_dir): basic.outputlogMessage( 'folder of %dth image (%s) already exist, ' 'it has been predicted or is being predicted' % (idx, inf_img_list[idx])) idx += 1 continue with open(inf_list_file, 'w') as inf_obj: inf_obj.writelines(inf_img_list[idx] + '\n') sub_process = Process(target=predict_one_image_deeplab, args=(deeplab_inf_script, para_file, network_setting_ini, img_save_dir, inf_list_file, gpuid, trained_model)) sub_process.start() sub_tasks.append(sub_process) if b_use_multiGPUs is False: # wait until previous one finished while sub_process.is_alive(): time.sleep(5) idx += 1 # wait until predicted image patches exist or exceed 20 minutes time0 = time.time() elapsed_time = time.time() - time0 while elapsed_time < 20 * 60: elapsed_time = time.time() - time0 file_exist = is_file_exist_in_folder(img_save_dir) if file_exist is True or sub_process.is_alive() is False: break else: time.sleep(5) if sub_process.exitcode is not None and sub_process.exitcode != 0: sys.exit(1) # if 'chpc' in machine_name: # time.sleep(60) # wait 60 second on ITSC services # else: # time.sleep(10) # check all the tasks already finished while b_all_task_finish(sub_tasks) is False: basic.outputlogMessage('wait all tasks to finish') time.sleep(60) end_time = datetime.datetime.now() diff_time = end_time - start_time out_str = "%s: time cost of total parallel inference on %s: %d seconds" % ( str(end_time), machine_name, diff_time.seconds) basic.outputlogMessage(out_str) with open("time_cost.txt", 'a') as t_obj: t_obj.writelines(out_str + '\n')
def main(options, args): print("%s : export the frozen inference graph" % os.path.basename(sys.argv[0])) para_file = args[0] if os.path.isfile(para_file) is False: raise IOError('File %s not exists in current folder: %s' % (para_file, os.getcwd())) network_setting_ini = parameters.get_string_parameters( para_file, 'network_setting_ini') tf_research_dir = parameters.get_directory_None_if_absence( network_setting_ini, 'tf_research_dir') print(tf_research_dir) if tf_research_dir is None: raise ValueError('tf_research_dir is not in %s' % para_file) if os.path.isdir(tf_research_dir) is False: raise ValueError('%s does not exist' % tf_research_dir) if os.getenv('PYTHONPATH'): os.environ['PYTHONPATH'] = os.getenv( 'PYTHONPATH') + ':' + tf_research_dir + ':' + os.path.join( tf_research_dir, 'slim') else: os.environ['PYTHONPATH'] = tf_research_dir + ':' + os.path.join( tf_research_dir, 'slim') global tf1x_python tf1x_python = parameters.get_file_path_parameters(network_setting_ini, 'tf1x_python') deeplab_dir = os.path.join(tf_research_dir, 'deeplab') WORK_DIR = os.getcwd() expr_name = parameters.get_string_parameters(para_file, 'expr_name') EXP_FOLDER = expr_name TRAIN_LOGDIR = os.path.join(WORK_DIR, EXP_FOLDER, 'train') EXPORT_DIR = os.path.join(WORK_DIR, EXP_FOLDER, 'export') inf_output_stride = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_output_stride', 'int') inf_atrous_rates1 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates1', 'int') inf_atrous_rates2 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates2', 'int') inf_atrous_rates3 = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'inf_atrous_rates3', 'int') # depth_multiplier default is 1.0. depth_multiplier = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'depth_multiplier', 'float') decoder_output_stride = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'decoder_output_stride', 'int') aspp_convs_filters = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'aspp_convs_filters', 'int') model_variant = parameters.get_string_parameters(network_setting_ini, 'model_variant') num_classes_noBG = parameters.get_digit_parameters_None_if_absence( para_file, 'NUM_CLASSES_noBG', 'int') assert num_classes_noBG != None b_initialize_last_layer = parameters.get_bool_parameters( para_file, 'b_initialize_last_layer') if b_initialize_last_layer is False: pre_trained_tar = parameters.get_string_parameters( network_setting_ini, 'TF_INIT_CKPT') if pre_trained_tar in pre_trained_tar_21_classes: print( 'warning, pretrained model %s is trained with 21 classes, set num_of_classes to 21' % pre_trained_tar) num_classes_noBG = 20 if pre_trained_tar in pre_trained_tar_19_classes: print( 'warning, pretrained model %s is trained with 19 classes, set num_of_classes to 19' % pre_trained_tar) num_classes_noBG = 18 num_of_classes = num_classes_noBG + 1 image_crop_size = parameters.get_string_list_parameters( para_file, 'image_crop_size') if len(image_crop_size) != 2 and image_crop_size[0].isdigit( ) and image_crop_size[1].isdigit(): raise ValueError('image_crop_size should be height,width') iteration_num = get_trained_iteration(TRAIN_LOGDIR) multi_scale = parameters.get_digit_parameters_None_if_absence( network_setting_ini, 'export_multi_scale', 'int') export_script = os.path.join(deeplab_dir, 'export_model.py') CKPT_PATH = os.path.join(TRAIN_LOGDIR, 'model.ckpt-%s' % iteration_num) EXPORT_PATH = os.path.join(EXPORT_DIR, 'frozen_inference_graph_%s.pb' % iteration_num) if os.path.isfile(EXPORT_PATH): basic.outputlogMessage('%s exists, skipping exporting models' % EXPORT_PATH) return export_graph(export_script, CKPT_PATH, EXPORT_PATH, model_variant, num_of_classes, inf_atrous_rates1, inf_atrous_rates2, inf_atrous_rates3, inf_output_stride, image_crop_size[0], image_crop_size[1], multi_scale, depth_multiplier, decoder_output_stride, aspp_convs_filters)
def image_translate_train_generate_one_domain(gan_working_dir, gan_para_file, area_src_ini, area_gan_ini, gpu_ids, domainB_imgList): current_dir = os.getcwd() # get orignal sub-images _, _, area_ini_sub_images_labels_dict = original_sub_images_labels_list_before_gan( ) sub_img_label_txt = os.path.join(current_dir, area_ini_sub_images_labels_dict) if os.path.isfile(area_ini_sub_images_labels_dict) is False: raise IOError( '%s not in the current folder, please get subImages first' % sub_img_label_txt) # prepare image list of domain A # what if the size of some images are not fit with CUT input? domain_A_images = [] # domain_A_labels = [] # with open(sub_img_label_txt) as txt_obj: # line_list = [name.strip() for name in txt_obj.readlines()] # for line in line_list: # sub_image, sub_label = line.split(':') # domain_A_images.append(os.path.join(current_dir,sub_image)) # # domain_A_labels.append(os.path.join(current_dir,sub_label)) area_ini_sub_images_labels = io_function.read_dict_from_txt_json( 'area_ini_sub_images_labels.txt') for line in area_ini_sub_images_labels[os.path.basename(area_src_ini)]: sub_image, sub_label = line.split(':') domain_A_images.append(os.path.join(current_dir, sub_image)) # domain_A_labels.append(os.path.join(current_dir,sub_label)) os.chdir(gan_working_dir) io_function.save_list_to_txt('image_A_list.txt', domain_A_images) # read target images, that will consider as target domains # what if there are too many images in domain B? io_function.save_list_to_txt('image_B_list.txt', domainB_imgList) gan_python = parameters.get_file_path_parameters(gan_para_file, 'python') cut_dir = parameters.get_directory(gan_para_file, 'gan_script_dir') train_script = os.path.join(cut_dir, 'train.py') generate_script = os.path.join(cut_dir, 'generate_image.py') # training of CUT if train_CUT_gan(gan_python, train_script, gan_para_file, gpu_ids) is False: os.chdir(current_dir) return False # genenerate image using CUT, convert images in domain A to domain B save_tran_img_folder = 'subImages_translate' if generate_image_CUT(gan_python, generate_script, gan_para_file, gpu_ids, domain_A_images, save_tran_img_folder) is False: os.chdir(current_dir) return False # change working directory back os.chdir(current_dir) pass