def is_training(main_config_fpath): '''check if complete pipeline should include training and scoring''' with open(main_config_fpath, 'r') as config_file: cfg_parser = ConfigParser.SafeConfigParser() cfg_parser.readfp(config_file) data_dir = preprocess.add_pathsep(cfg_parser.get( 'general', 'data_dir')) if preprocess.is_labeled(data_dir): return True return False
def main(main_config_fpath='../data/example/main_config.cfg', run_type='forward'): '''Get user-specified information from main_config.cfg''' cfg_parser = ConfigParser.SafeConfigParser() cfg_parser.readfp(open(main_config_fpath, 'r')) img_width = cfg_parser.get('general', 'img_width') img_height = cfg_parser.get('general', 'img_height') net_arch_fpath = cfg_parser.get('network', 'net_arch_fpath') train_net_prefix = cfg_parser.get('training', 'training_net_prefix') train_patch_size = cfg_parser.get('training', 'patch_size') learning_rate = cfg_parser.get('training', 'learning_rate') momentum = cfg_parser.get('training', 'momentum') num_iter_per_save = cfg_parser.get('training', 'num_iter_per_save') max_iter = cfg_parser.get('training', 'max_iter') forward_net = cfg_parser.get('forward', 'forward_net') forward_outsz = cfg_parser.get('forward', 'forward_outsz') filter_size = cfg_parser.get('network', 'filter_size') is_squashing = cfg_parser.get('network', 'is_squashing') time_equalize = cfg_parser.get('preprocessing', 'time_equalize') # Get and make user-specified input/output directories input_dir, output_dir = get_io_dirs(run_type, cfg_parser) if not os.path.isdir(input_dir): os.makedirs(input_dir) if not os.path.isdir(output_dir): os.makedirs(output_dir) #docker_dir = dockerize_path(output_dir) # Build file name dictionary if is_labeled(input_dir): file_dict = get_labeled_split(input_dir) else: file_dict = build_unlabeled_file_dict(input_dir) add_indices_to_dict(file_dict) train_indices, val_indices, forward_indices = get_train_val_forward_split_indices_as_str( file_dict) # Create znn files and save to output_dir new_net_fpath = output_dir + net_arch_fpath.split(os.sep)[-1] num_file_pairs = create_dataset_spec(input_dir, output_dir, file_dict, cfg_parser, main_config_fpath, run_type) create_znn_config_file(output_dir, train_indices, val_indices, forward_indices, new_net_fpath, train_net_prefix, train_patch_size, learning_rate, momentum, num_iter_per_save, max_iter, forward_net, forward_outsz, num_file_pairs) copy_net_and_set_conv_filter_size(net_arch_fpath, new_net_fpath, filter_size) if is_squashing: set_network_3D_to_2D_squashing_filter_size(new_net_fpath, time_equalize)
def main(main_config_fpath="../data/example/main_config.cfg"): '''use configuration file to find all image and ROI paths then start GUI application''' cfg_parser = ConfigParser.SafeConfigParser() cfg_parser.readfp(open(main_config_fpath, 'r')) data_dir = add_pathsep(cfg_parser.get('general', 'data_dir')) preprocess_dir = data_dir[0:-1] + "_preprocessed" + os.sep postprocess_dir = data_dir[0:-1] + "_postprocessed" + os.sep ttv_list = ['training' + os.sep, 'validation' + os.sep, 'test' + os.sep] if not os.path.isdir(data_dir): sys.exit("Specified data directory " + data_dir + " does not exist.") files = defaultdict(lambda: [None, None, None]) for ttv in ttv_list if is_labeled(data_dir) else ['']: for f in os.listdir(preprocess_dir + ttv): basename, ext = os.path.splitext(f) if ext.lower() == '.tif' or ext.lower() == '.tiff': if basename[-4:].lower() != "_roi": files[basename][0] = preprocess_dir + ttv + f for f in os.listdir(postprocess_dir + ttv): basename, ext = os.path.splitext(f) if ext == '.npz' and basename[-7:] != "_MANUAL": files[basename][1] = postprocess_dir + ttv + f for f in os.listdir(data_dir + ttv): basename, ext = os.path.splitext(f) if ext == '.zip': files[basename][2] = data_dir + ttv + f img_width = cfg_parser.getint('general', 'img_width') img_height = cfg_parser.getint('general', 'img_height') root = Tk() root.wm_title("ConvnetCellDetection") app = App(root, files, img_width, img_height, is_labeled(data_dir)) root.mainloop()
def main(main_config_fpath='../data/example/main_config.cfg'): cfg_parser = ConfigParser.SafeConfigParser() cfg_parser.readfp(open(main_config_fpath, 'r')) # read parameters data_dir = add_pathsep(cfg_parser.get('general', 'data_dir')) postprocess_dir = data_dir[0:-1] + "_postprocessed" + os.sep img_width = cfg_parser.getint('general', 'img_width') img_height = cfg_parser.getint('general', 'img_height') # if not is_labeled(data_dir) or not is_labeled(postprocess_dir): #we haven't been putting test/train/val into a "labeled" folder #Let's discuss if we should. -AR 09/13/16 if not is_labeled(data_dir): print "original data_dir was not called 'labeled'. Aborting scoring." return else: print "Scoring labeled data" if postprocess_dir[-1] != os.path.sep: postprocess_dir += os.path.sep if data_dir[-1] != os.path.sep: data_dir += os.path.sep score_labeled_data(postprocess_dir, data_dir, img_width, img_height)
def main(main_config_fpath='../data/example/main_config.cfg'): '''Get user-specified information from main_config.cfg''' cfg_parser = ConfigParser.SafeConfigParser() cfg_parser.readfp(open(main_config_fpath,'r')) # get directory paths data_dir = add_pathsep(cfg_parser.get('general', 'data_dir')) parent_dir = add_pathsep(os.path.dirname(data_dir[0:-1])) downsample_dir = data_dir[0:-1] + "_downsampled" + os.sep preprocess_dir = data_dir[0:-1] + "_preprocessed" + os.sep network_output_dir = data_dir[0:-1] + "_training_output" + os.sep postprocess_dir = data_dir[0:-1] + "_postprocessed" + os.sep ttv_list = ['training' + os.sep, 'validation' + os.sep, 'test' + os.sep] # ensure directories exist if not os.path.isdir(data_dir): sys.exit("Specified data directory " + data_dir + " does not exist.") for ttv in ttv_list if is_labeled(data_dir) else ['']: if not os.path.isdir(postprocess_dir + ttv): os.makedirs(postprocess_dir + ttv) # split training output directory if necessary if is_labeled(data_dir): split_dict = get_labeled_split(data_dir) split_labeled_directory(split_dict, network_output_dir, False, False) # get non-optimized postprocessing parameters img_width = cfg_parser.getint('general','img_width') img_height = cfg_parser.getint('general', 'img_height') do_gridsearch_postprocess_params = cfg_parser.getboolean('general', 'do_gridsearch_postprocess_params') min_size_wand = cfg_parser.getfloat('postprocessing', 'min_size_wand') max_size_wand = cfg_parser.getfloat('postprocessing', 'max_size_wand') # locate optimized postprocessing parameters or run grid search optimization params_cfg_parser = ConfigParser.SafeConfigParser() opt_params_cfg_fn = parent_dir + "optimized_postprocess_params.cfg" if do_gridsearch_postprocess_params and os.path.isfile(opt_params_cfg_fn): params_cfg_parser.readfp(open(opt_params_cfg_fn, 'r')) elif (do_gridsearch_postprocess_params and not os.path.isfile(opt_params_cfg_fn) and is_labeled(data_dir)): params_cfg_parser = parameter_optimization(data_dir + ttv_list[1], preprocess_dir + ttv_list[1], network_output_dir + ttv_list[1], postprocess_dir + ttv_list[1], min_size_wand, max_size_wand, img_width, img_height, opt_params_cfg_fn, cfg_parser) else: params_cfg_parser = cfg_parser # read postprocessing-specific parameters threshold = params_cfg_parser.getfloat('postprocessing', 'probability_threshold') min_size_watershed = params_cfg_parser.getfloat('postprocessing', 'min_size_watershed') merge_size_watershed = params_cfg_parser.getfloat('postprocessing', 'merge_size_watershed') max_footprint_str = params_cfg_parser.get('postprocessing', 'max_footprint') max_footprint = tuple([int(float(c)) for c in max_footprint_str.strip().strip(')').strip('(').split(",")]) max_size_wand = params_cfg_parser.getfloat('postprocessing', 'max_size_wand') assert(len(max_footprint) == 2) # run postprocessing for ttv in ttv_list if is_labeled(data_dir) else ['']: final_rois, final_roi_probs, filenames = postprocessing(preprocess_dir + ttv, network_output_dir + ttv, postprocess_dir + ttv, threshold, min_size_watershed, merge_size_watershed, max_footprint, min_size_wand, max_size_wand) # Save final ROIs for i,roi in enumerate(final_rois): r = roi.max(axis=0) roi_name = postprocess_dir + ttv + filenames[i] + '.tif' tifffile.imsave(roi_name, r.astype(np.float32)) np.savez_compressed(postprocess_dir + ttv + filenames[i] + '.npz', rois=roi, roi_probabilities=final_roi_probs[i])