def is_training(main_config_fpath):
'''check if complete pipeline should include training and scoring'''
with open(main_config_fpath, 'r') as config_file:
cfg_parser = ConfigParser.SafeConfigParser()
cfg_parser.readfp(config_file)
data_dir = preprocess.add_pathsep(cfg_parser.get(
'general', 'data_dir'))
if preprocess.is_labeled(data_dir):
return True
return False
def main(main_config_fpath='../data/example/main_config.cfg',
run_type='forward'):
'''Get user-specified information from main_config.cfg'''
cfg_parser = ConfigParser.SafeConfigParser()
cfg_parser.readfp(open(main_config_fpath, 'r'))
img_width = cfg_parser.get('general', 'img_width')
img_height = cfg_parser.get('general', 'img_height')
net_arch_fpath = cfg_parser.get('network', 'net_arch_fpath')
train_net_prefix = cfg_parser.get('training', 'training_net_prefix')
train_patch_size = cfg_parser.get('training', 'patch_size')
learning_rate = cfg_parser.get('training', 'learning_rate')
momentum = cfg_parser.get('training', 'momentum')
num_iter_per_save = cfg_parser.get('training', 'num_iter_per_save')
max_iter = cfg_parser.get('training', 'max_iter')
forward_net = cfg_parser.get('forward', 'forward_net')
forward_outsz = cfg_parser.get('forward', 'forward_outsz')
filter_size = cfg_parser.get('network', 'filter_size')
is_squashing = cfg_parser.get('network', 'is_squashing')
time_equalize = cfg_parser.get('preprocessing', 'time_equalize')
# Get and make user-specified input/output directories
input_dir, output_dir = get_io_dirs(run_type, cfg_parser)
if not os.path.isdir(input_dir):
os.makedirs(input_dir)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
#docker_dir = dockerize_path(output_dir)
# Build file name dictionary
if is_labeled(input_dir):
file_dict = get_labeled_split(input_dir)
else:
file_dict = build_unlabeled_file_dict(input_dir)
add_indices_to_dict(file_dict)
train_indices, val_indices, forward_indices = get_train_val_forward_split_indices_as_str(
file_dict)
# Create znn files and save to output_dir
new_net_fpath = output_dir + net_arch_fpath.split(os.sep)[-1]
num_file_pairs = create_dataset_spec(input_dir, output_dir, file_dict,
cfg_parser, main_config_fpath,
run_type)
create_znn_config_file(output_dir, train_indices, val_indices,
forward_indices, new_net_fpath, train_net_prefix,
train_patch_size, learning_rate, momentum,
num_iter_per_save, max_iter, forward_net,
forward_outsz, num_file_pairs)
copy_net_and_set_conv_filter_size(net_arch_fpath, new_net_fpath,
filter_size)
if is_squashing:
set_network_3D_to_2D_squashing_filter_size(new_net_fpath,
time_equalize)
def main(main_config_fpath="../data/example/main_config.cfg"):
'''use configuration file to find all image and
ROI paths then start GUI application'''
cfg_parser = ConfigParser.SafeConfigParser()
cfg_parser.readfp(open(main_config_fpath, 'r'))
data_dir = add_pathsep(cfg_parser.get('general', 'data_dir'))
preprocess_dir = data_dir[0:-1] + "_preprocessed" + os.sep
postprocess_dir = data_dir[0:-1] + "_postprocessed" + os.sep
ttv_list = ['training' + os.sep, 'validation' + os.sep, 'test' + os.sep]
if not os.path.isdir(data_dir):
sys.exit("Specified data directory " + data_dir + " does not exist.")
files = defaultdict(lambda: [None, None, None])
for ttv in ttv_list if is_labeled(data_dir) else ['']:
for f in os.listdir(preprocess_dir + ttv):
basename, ext = os.path.splitext(f)
if ext.lower() == '.tif' or ext.lower() == '.tiff':
if basename[-4:].lower() != "_roi":
files[basename][0] = preprocess_dir + ttv + f
for f in os.listdir(postprocess_dir + ttv):
basename, ext = os.path.splitext(f)
if ext == '.npz' and basename[-7:] != "_MANUAL":
files[basename][1] = postprocess_dir + ttv + f
for f in os.listdir(data_dir + ttv):
basename, ext = os.path.splitext(f)
if ext == '.zip':
files[basename][2] = data_dir + ttv + f
img_width = cfg_parser.getint('general', 'img_width')
img_height = cfg_parser.getint('general', 'img_height')
root = Tk()
root.wm_title("ConvnetCellDetection")
app = App(root, files, img_width, img_height, is_labeled(data_dir))
root.mainloop()
def main(main_config_fpath='../data/example/main_config.cfg'):
cfg_parser = ConfigParser.SafeConfigParser()
cfg_parser.readfp(open(main_config_fpath, 'r'))
# read parameters
data_dir = add_pathsep(cfg_parser.get('general', 'data_dir'))
postprocess_dir = data_dir[0:-1] + "_postprocessed" + os.sep
img_width = cfg_parser.getint('general', 'img_width')
img_height = cfg_parser.getint('general', 'img_height')
# if not is_labeled(data_dir) or not is_labeled(postprocess_dir): #we haven't been putting test/train/val into a "labeled" folder #Let's discuss if we should. -AR 09/13/16
if not is_labeled(data_dir):
print "original data_dir was not called 'labeled'. Aborting scoring."
return
else:
print "Scoring labeled data"
if postprocess_dir[-1] != os.path.sep:
postprocess_dir += os.path.sep
if data_dir[-1] != os.path.sep:
data_dir += os.path.sep
score_labeled_data(postprocess_dir, data_dir, img_width, img_height)
def main(main_config_fpath='../data/example/main_config.cfg'):
'''Get user-specified information from main_config.cfg'''
cfg_parser = ConfigParser.SafeConfigParser()
cfg_parser.readfp(open(main_config_fpath,'r'))
# get directory paths
data_dir = add_pathsep(cfg_parser.get('general', 'data_dir'))
parent_dir = add_pathsep(os.path.dirname(data_dir[0:-1]))
downsample_dir = data_dir[0:-1] + "_downsampled" + os.sep
preprocess_dir = data_dir[0:-1] + "_preprocessed" + os.sep
network_output_dir = data_dir[0:-1] + "_training_output" + os.sep
postprocess_dir = data_dir[0:-1] + "_postprocessed" + os.sep
ttv_list = ['training' + os.sep, 'validation' + os.sep, 'test' + os.sep]
# ensure directories exist
if not os.path.isdir(data_dir):
sys.exit("Specified data directory " + data_dir + " does not exist.")
for ttv in ttv_list if is_labeled(data_dir) else ['']:
if not os.path.isdir(postprocess_dir + ttv):
os.makedirs(postprocess_dir + ttv)
# split training output directory if necessary
if is_labeled(data_dir):
split_dict = get_labeled_split(data_dir)
split_labeled_directory(split_dict, network_output_dir, False, False)
# get non-optimized postprocessing parameters
img_width = cfg_parser.getint('general','img_width')
img_height = cfg_parser.getint('general', 'img_height')
do_gridsearch_postprocess_params = cfg_parser.getboolean('general', 'do_gridsearch_postprocess_params')
min_size_wand = cfg_parser.getfloat('postprocessing', 'min_size_wand')
max_size_wand = cfg_parser.getfloat('postprocessing', 'max_size_wand')
# locate optimized postprocessing parameters or run grid search optimization
params_cfg_parser = ConfigParser.SafeConfigParser()
opt_params_cfg_fn = parent_dir + "optimized_postprocess_params.cfg"
if do_gridsearch_postprocess_params and os.path.isfile(opt_params_cfg_fn):
params_cfg_parser.readfp(open(opt_params_cfg_fn, 'r'))
elif (do_gridsearch_postprocess_params and
not os.path.isfile(opt_params_cfg_fn) and is_labeled(data_dir)):
params_cfg_parser = parameter_optimization(data_dir + ttv_list[1],
preprocess_dir + ttv_list[1],
network_output_dir + ttv_list[1],
postprocess_dir + ttv_list[1],
min_size_wand,
max_size_wand, img_width, img_height,
opt_params_cfg_fn, cfg_parser)
else:
params_cfg_parser = cfg_parser
# read postprocessing-specific parameters
threshold = params_cfg_parser.getfloat('postprocessing', 'probability_threshold')
min_size_watershed = params_cfg_parser.getfloat('postprocessing', 'min_size_watershed')
merge_size_watershed = params_cfg_parser.getfloat('postprocessing', 'merge_size_watershed')
max_footprint_str = params_cfg_parser.get('postprocessing', 'max_footprint')
max_footprint = tuple([int(float(c)) for c in max_footprint_str.strip().strip(')').strip('(').split(",")])
max_size_wand = params_cfg_parser.getfloat('postprocessing', 'max_size_wand')
assert(len(max_footprint) == 2)
# run postprocessing
for ttv in ttv_list if is_labeled(data_dir) else ['']:
final_rois, final_roi_probs, filenames = postprocessing(preprocess_dir + ttv, network_output_dir + ttv,
postprocess_dir + ttv, threshold,
min_size_watershed, merge_size_watershed,
max_footprint, min_size_wand, max_size_wand)
# Save final ROIs
for i,roi in enumerate(final_rois):
r = roi.max(axis=0)
roi_name = postprocess_dir + ttv + filenames[i] + '.tif'
tifffile.imsave(roi_name, r.astype(np.float32))
np.savez_compressed(postprocess_dir + ttv + filenames[i] + '.npz', rois=roi, roi_probabilities=final_roi_probs[i])
