def main():
start_time = datetime.now()
args = cells_standard_space_cli_parser().parse_args()
args.paths = prep.Paths(args.output_dir)
args.paths.standard_space_output_folder = args.output_dir
args.paths.cells_in_standard_space = join(args.paths.output_dir,
"cells_in_standard_space.xml")
cli_path_update(args.paths, args)
args.paths.make_invert_cell_position_paths()
args = define_pixel_sizes(args)
# TODO: implement a recursive function to remove the need to do this
# (probably using pathlib)
system.ensure_directory_exists(args.paths.output_dir)
system.ensure_directory_exists(args.paths.standard_space_output_folder)
tools.start_logging(
args.paths.output_dir,
args=args,
verbose=args.debug,
filename="cells_to_standard_space",
log_header="CELL TRANSFORMATION TO STANDARD SPACE LOG",
)
logging.info("Starting transformation of cell positions")
transform_cells_to_standard_space(args)
logging.info("Finished. Total time taken: %s", datetime.now() - start_time)
def prep_registration(args, sample_name="amap"):
logging.info("Checking whether the atlas exists")
_, atlas_files_exist = check_atlas_install()
atlas_dir = os.path.join(args.install_path, "atlas")
if not atlas_files_exist:
if args.download_path is None:
atlas_download_path = os.path.join(temp_dir_path, "atlas.tar.gz")
else:
atlas_download_path = os.path.join(
args.download_path, "atlas.tar.gz"
)
if not args.no_atlas:
logging.warning("Atlas does not exist, downloading.")
atlas_download.main(args.atlas, atlas_dir, atlas_download_path)
amend_cfg(new_atlas_folder=atlas_dir, atlas=args.atlas)
if args.registration_config is None:
args.registration_config = source_files.source_custom_config()
args.target_brain_path = args.background_planes_path[0]
args.sample_name = sample_name
logging.debug("Making registration directory")
system.ensure_directory_exists(args.paths.registration_output_folder)
additional_images_downsample = {}
for idx, images in enumerate(args.signal_planes_paths):
channel = args.signal_ch_ids[idx]
additional_images_downsample[f"channel_{channel}"] = images
return args, additional_images_downsample
def prep_candidate_detection(args):
if args.save_planes:
args.plane_directory = os.path.join(
args.output_dir, "processed_planes"
)
system.ensure_directory_exists(args.plane_directory)
else:
args.plane_directory = None # FIXME: remove this fudge
return args
def xml_scale(
xml_file,
x_scale=1,
y_scale=1,
z_scale=1,
output_directory=None,
integer=True,
):
# TODO: add a csv option
"""
To rescale the cell positions within an XML file. For compatibility with
other software, or if data has been scaled after cell detection.
:param xml_file: Any cellfinder xml file
:param x_scale: Rescaling factor in the first dimension
:param y_scale: Rescaling factor in the second dimension
:param z_scale: Rescaling factor in the third dimension
:param output_directory: Directory to save the rescaled XML file.
Defaults to the same directory as the input XML file
:param integer: Force integer cell positions (default: True)
:return:
"""
if x_scale == y_scale == z_scale == 1:
raise CommandLineInputError("All rescaling factors are 1, "
"please check the input.")
else:
input_file = Path(xml_file)
start_time = datetime.now()
cells = cio.get_cells(xml_file)
for cell in cells:
cell.transform(
x_scale=x_scale,
y_scale=y_scale,
z_scale=z_scale,
integer=integer,
)
if output_directory:
output_directory = Path(output_directory)
else:
output_directory = input_file.parent
ensure_directory_exists(output_directory)
output_filename = output_directory / (input_file.stem + "_rescaled")
output_filename = output_filename.with_suffix(input_file.suffix)
cio.save_cells(cells, output_filename)
print("Finished. Total time taken: {}".format(datetime.now() -
start_time))
def main(model, download_path):
model_weight_dir = os.path.join(download_path, "model_weights")
model_path = os.path.join(model_weight_dir, model + ".h5")
if not os.path.exists(model_path):
ensure_directory_exists(model_weight_dir)
download_path = os.path.join(model_weight_dir, model + ".h5")
download(
download_path,
model_weight_urls[model],
model,
download_requires=download_requirements_gb[model],
)
else:
print(f"Model already exists at {model_path}. Skipping download")
return model_path
def main():
# TODO: remove converting to strings when cellfinder is python >3.6 only
start_time = datetime.now()
args = cube_extract_cli_parser().parse_args()
args = define_pixel_sizes(args)
args = prep_channel_ids(args)
# TODO: can these be set in argparse?
args.output_dir = Path(args.output_dir)
args.cells_file_path = Path(args.cells_file_path)
args.cube_extract_cli = True
args.paths = prep.Paths(args.output_dir)
system.ensure_directory_exists(args.output_dir)
tools.start_logging(
str(args.output_dir),
args=args,
verbose=args.debug,
filename="batch_cube_extraction",
log_header="CELLFINDER BATCH CUBE EXTRACTION LOG",
)
logging.info("Starting cube extraction")
num_channels = len(args.raw_data_paths)
if args.cells_file_path.is_dir():
indv_cells_paths = natsorted(subdirs(args.cells_file_path))
elif args.cells_file_path.is_file():
tmp_paths = tools.get_text_lines(args.cells_file_path)
indv_cells_paths = natsorted([Path(path) for path in tmp_paths])
if indv_cells_paths[0].is_dir():
logging.info("Assuming cube definition data is output from ROI Sorter")
cells_dir_names = [path.name for path in indv_cells_paths]
if tools.is_any_list_overlap(cells_dir_names,
train_data_gen_dir_names):
# If there is only training data from one sample, make a list of 1
indv_cells_paths = [args.cells_file_path]
for idx, sample_dir in enumerate(indv_cells_paths):
logging.info("Extracting cubes from: {}".format(sample_dir.name))
args.all_planes_paths = []
system.ensure_directory_exists(
args.output_dir.joinpath(sample_dir.name))
for channel in range(num_channels):
args.all_planes_paths.append(
tools.get_text_lines(
args.raw_data_paths[channel],
return_lines=idx,
sort=True,
))
for sub_dir_name in train_data_gen_dir_names:
if sub_dir_name in subdirs(sample_dir, names_only=True):
logging.info("Extracting: {}".format(sub_dir_name))
args.paths.tmp__cubes_output_dir = str(
args.output_dir.joinpath(sample_dir.name,
sub_dir_name))
args.paths.cells_file_path = str(
args.cells_file_path.joinpath(sample_dir.name,
sub_dir_name))
extract_loop(args)
else:
if indv_cells_paths[0].suffix in SUPPORTED_CELL_FILE_FORMATS:
logging.info("Cube definition file is: '{}', proceeding."
"".format(indv_cells_paths[0].suffix))
raise NotImplementedError(
"Cube definition file is: '{}', NOT YET SUPPORTED."
"".format(indv_cells_paths[0].suffix))
else:
raise NotImplementedError(
"File format: '{}' is not yet supported. Please use the "
"output of ROI Sorter, or provide one of: {}".format(
indv_cells_paths[0].suffix, SUPPORTED_CELL_FILE_FORMATS))
logging.info("Finished. Total time taken: %s", datetime.now() - start_time)
def extract_loop(args):
system.ensure_directory_exists(args.paths.tmp__cubes_output_dir)
extract_cubes.main(args)
def main(max_workers=3):
from cellfinder.main import suppress_tf_logging
suppress_tf_logging(tf_suppress_log_messages)
from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint
from cellfinder.tools import system
from cellfinder.tools.prep import prep_training
from cellfinder.classify.tools import make_lists, get_model
from cellfinder.classify.cube_generator import CubeGeneratorFromDisk
start_time = datetime.now()
args = training_parse()
output_dir = Path(args.output_dir)
system.ensure_directory_exists(output_dir)
args = prep_training(args)
tiff_files = parse_yaml(args.yaml_file)
# Too many workers doesn't increase speed, and uses huge amounts of RAM
workers = system.get_num_processes(min_free_cpu_cores=args.n_free_cpus,
n_max_processes=max_workers)
model = get_model(
existing_model=args.trained_model,
model_weights=args.model_weights,
network_depth=models[args.network_depth],
learning_rate=args.learning_rate,
continue_training=args.continue_training,
)
signal_train, background_train, labels_train = make_lists(tiff_files)
if args.test_fraction > 0:
(
signal_train,
signal_test,
background_train,
background_test,
labels_train,
labels_test,
) = train_test_split(
signal_train,
background_train,
labels_train,
test_size=args.test_fraction,
)
validation_generator = CubeGeneratorFromDisk(
signal_test,
background_test,
labels=labels_test,
batch_size=args.batch_size,
train=True,
)
else:
validation_generator = None
training_generator = CubeGeneratorFromDisk(
signal_train,
background_train,
labels=labels_train,
batch_size=args.batch_size,
shuffle=True,
train=True,
augment=not args.no_augment,
)
callbacks = []
if args.tensorboard:
logdir = output_dir / "tensorboard"
system.ensure_directory_exists(logdir)
tensorboard = TensorBoard(
log_dir=logdir,
histogram_freq=0,
write_graph=True,
update_freq="epoch",
)
callbacks.append(tensorboard)
if args.save_checkpoints:
if args.save_weights:
filepath = str(output_dir /
"weights.{epoch:02d}-{val_loss:.3f}.h5")
else:
filepath = str(output_dir / "model.{epoch:02d}-{val_loss:.3f}.h5")
checkpoints = ModelCheckpoint(filepath,
save_weights_only=args.save_weights)
callbacks.append(checkpoints)
model.fit(
training_generator,
validation_data=validation_generator,
use_multiprocessing=True,
workers=workers,
epochs=args.epochs,
callbacks=callbacks,
)
if args.save_weights:
print("Saving model weights")
model.save_weights(str(output_dir / "model_weights.h5"))
else:
print("Saving model")
model.save(output_dir / "model.h5")
print(
"Finished training, "
"Total time taken: %s",
datetime.now() - start_time,
)
def test_ensure_directory_exists():
home = os.path.expanduser("~")
exist_dir = os.path.join(home, ".cellfinder_test_dir")
system.ensure_directory_exists(exist_dir)
assert os.path.exists(exist_dir)
os.rmdir(exist_dir)
def figures_prep(args):
system.ensure_directory_exists(args.paths.figures_dir)
args.paths.make_figures_paths()
return args
def standard_space_prep(args):
system.ensure_directory_exists(args.paths.standard_space_output_folder)
args.paths.make_invert_cell_position_paths()
return args