def prep_cellfinder_general():
args = parser.cellfinder_parser().parse_args()
arg_groups = get_arg_groups(args, cellfinder_parser())
# args = define_pixel_sizes(args)
check_input_arg_existance(args)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
args.paths = Paths(args.output_dir)
fancylog.start_logging(
args.output_dir,
program_for_log,
variables=[args, args.paths],
verbose=args.debug,
log_header="CELLFINDER LOG",
)
log_metadata(args.paths.metadata_path, args)
what_to_run = CalcWhatToRun(args)
args.signal_ch_ids, args.background_ch_id = check_and_return_ch_ids(
args.signal_ch_ids, args.background_ch_id, args.signal_planes_paths)
args.brainreg_paths = BrainRegPaths(args.paths.registration_output_folder)
atlas = BrainGlobeAtlas(args.atlas)
return args, arg_groups, what_to_run, atlas
def setup(config_path):
if len(sys.argv) == 2 and sys.argv[1] != "-h":
args = experiment_parser.GetArgs(sys.argv[1])
else:
args = cli_parser.get_args()
ensure_directory_exists(args.output_dir)
if args.config_path is None:
args.config_path = config_path
config = config_parser.GetConfig(args.config_path)
options = config_parser.GetOptions(args.config_path)
options.num_processes = get_num_processes(
min_free_cpu_cores=options.n_free_cpus)
fancylog.start_logging(
args.output_dir,
program_for_log,
variables=[args],
verbose=args.verbose,
log_header="OPENDIRECTION LOG",
)
return args, options, config
def main():
start_time = datetime.now()
args = cells_standard_space_cli_parser().parse_args()
args.paths = prep.Paths(args.output_dir)
args.paths.standard_space_output_folder = args.output_dir
args.paths.cells_in_standard_space = join(
args.paths.output_dir, "cells_in_standard_space.xml"
)
cli_path_update(args.paths, args)
args.paths.make_invert_cell_position_paths()
args = define_pixel_sizes(args)
# TODO: implement a recursive function to remove the need to do this
# (probably using pathlib)
ensure_directory_exists(args.paths.output_dir)
ensure_directory_exists(args.paths.standard_space_output_folder)
fancylog.start_logging(
args.paths.output_dir,
log_to_file=False,
package=program_name,
variables=args,
verbose=args.debug,
filename="cells_to_standard_space",
log_header="CELL TRANSFORMATION TO STANDARD SPACE LOG",
)
logging.info("Starting transformation of cell positions")
transform_cells_to_standard_space(args)
logging.info("Finished. Total time taken: %s", datetime.now() - start_time)
def main():
args = extraction_parser().parse_args()
# Get output directory
if args.output_directory is None:
outdir = os.get_cwd()
elif not os.path.isdir(args.output_directory):
raise ValueError("Output directory invalid")
else:
outdir = args.output_directory
# Start log
log_name = "injection_finder_{}".format(
os.path.split(args.registration_folder)[-1])
fancylog.start_logging(outdir, package, filename=log_name, verbose=True)
# Start extraction
Extractor(
args.img_filepath,
args.registration_folder,
logging,
overwrite=args.overwrite,
gaussian_kernel=args.gaussian_kernel,
percentile_threshold=args.percentile_threshold,
threshold_type=args.threshold_type,
obj_path=args.obj_path,
overwrite_registration=args.overwrite_registration,
)
def main(directory):
args = MadeUpArgs()
verbose = True
fancylog.start_logging(
directory, package, variables=[args, args.paths], verbose=verbose
)
logging.info("This is an info message")
logging.debug("This is a debug message")
logging.warning("This fun logging experience is about to end :(")
def run():
start_time = datetime.now()
args = register_cli_parser().parse_args()
args = define_pixel_sizes(args)
args, additional_images_downsample = prep_registration(args)
args = make_paths_absolute(args)
fancylog.start_logging(
args.registration_output_folder,
program_for_log,
variables=[args],
verbose=args.debug,
log_header="AMAP LOG",
multiprocessing_aware=False,
)
logging.info("Starting registration")
register(
args.registration_config,
args.image_paths,
args.registration_output_folder,
x_pixel_um=args.x_pixel_um,
y_pixel_um=args.y_pixel_um,
z_pixel_um=args.z_pixel_um,
orientation=args.orientation,
flip_x=args.flip_x,
flip_y=args.flip_y,
flip_z=args.flip_z,
rotation=args.rotation,
affine_n_steps=args.affine_n_steps,
affine_use_n_steps=args.affine_use_n_steps,
freeform_n_steps=args.freeform_n_steps,
freeform_use_n_steps=args.freeform_use_n_steps,
bending_energy_weight=args.bending_energy_weight,
grid_spacing=args.grid_spacing,
smoothing_sigma_reference=args.smoothing_sigma_reference,
smoothing_sigma_floating=args.smoothing_sigma_floating,
histogram_n_bins_floating=args.histogram_n_bins_floating,
histogram_n_bins_reference=args.histogram_n_bins_reference,
sort_input_file=args.sort_input_file,
n_free_cpus=args.n_free_cpus,
save_downsampled=not (args.no_save_downsampled),
boundaries=not (args.no_boundaries),
additional_images_downsample=additional_images_downsample,
debug=args.debug,
)
logging.info("Finished. Total time taken: %s", datetime.now() - start_time)
def main():
start_time = datetime.now()
args = register_cli_parser().parse_args()
arg_groups = get_arg_groups(args, register_cli_parser())
args = define_pixel_sizes(args)
args, additional_images_downsample = prep_registration(args)
paths = Paths(args.brainreg_directory)
log_metadata(paths.metadata_path, args)
fancylog.start_logging(
paths.registration_output_folder,
program_for_log,
variables=[args],
verbose=args.debug,
log_header="BRAINREG LOG",
multiprocessing_aware=False,
)
logging.info("Starting registration")
atlas = BrainGlobeAtlas(args.atlas)
register(
atlas,
args.orientation,
args.image_paths,
paths,
arg_groups["NiftyReg registration backend options"],
x_pixel_um=args.x_pixel_um,
y_pixel_um=args.y_pixel_um,
z_pixel_um=args.z_pixel_um,
sort_input_file=args.sort_input_file,
n_free_cpus=args.n_free_cpus,
additional_images_downsample=additional_images_downsample,
backend=args.backend,
debug=args.debug,
)
logging.info("Finished. Total time taken: %s", datetime.now() - start_time)
def prep_cellfinder_general():
args = parser.cellfinder_parser().parse_args()
args = define_pixel_sizes(args)
check_input_arg_existance(args)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
args.paths = Paths(args.output_dir)
args.paths.make_reg_paths()
fancylog.start_logging(
args.output_dir,
program_for_log,
variables=[args, args.paths],
verbose=args.debug,
log_header="CELLFINDER LOG",
)
what_to_run = CalcWhatToRun(args)
args.signal_ch_ids, args.background_ch_id = check_and_return_ch_ids(
args.signal_ch_ids, args.background_ch_id, args.signal_planes_paths)
return args, what_to_run
def mult_exp_setup():
args = get_args()
ensure_directory_exists(args.output_dir)
options = config_parser.GetOptions(args.options)
num_processes = get_num_processes(min_free_cpu_cores=args.n_free_cpus)
options.num_processes = num_processes
fancylog.start_logging(
args.output_dir,
program_for_log,
variables=[args],
verbose=args.verbose,
log_header="OPENDIRECTION MULTI EXPERIMENT LOG",
)
experiment_files = glob(os.path.join(args.exp_files, "*.txt"))
logging.info(f"Found {len(experiment_files)} experiment files")
experiment_config_list = [
experiment_parser.GetArgs(experiment_file)
for experiment_file in experiment_files
]
return args, options, experiment_config_list, num_processes
def main():
args = extraction_parser().parse_args()
# Get output directory
if args.output_directory is None:
outdir = os.getcwd()
elif not os.path.isdir(args.output_directory):
raise ValueError("Output directory invalid")
else:
outdir = args.output_directory
if args.obj_path is None:
args.obj_path = Path(args.img_filepath).with_suffix(".obj")
else:
args.obj_path = Path(args.obj_path)
# Start log
fancylog.start_logging(
outdir,
package_for_log,
filename="injection_finder",
verbose=args.debug,
log_to_file=args.save_log,
)
# Start extraction
Extractor(
args.img_filepath,
args.registration_folder,
overwrite=args.overwrite,
gaussian_kernel=args.gaussian_kernel,
gaussian_kernel_z=args.gaussian_kernel_z,
percentile_threshold=args.percentile_threshold,
threshold_type=args.threshold_type,
obj_path=args.obj_path,
overwrite_registration=args.overwrite_registration,
)
def main():
from cellfinder.main import suppress_tf_logging
suppress_tf_logging(tf_suppress_log_messages)
from tensorflow.keras.callbacks import (
TensorBoard,
ModelCheckpoint,
CSVLogger,
)
from cellfinder.tools.prep import prep_training
from cellfinder.classify.tools import make_lists, get_model
from cellfinder.classify.cube_generator import CubeGeneratorFromDisk
start_time = datetime.now()
args = training_parse()
output_dir = Path(args.output_dir)
ensure_directory_exists(output_dir)
args = prep_training(args)
fancylog.start_logging(
args.output_dir,
program_for_log,
variables=[args],
log_header="CELLFINDER TRAINING LOG",
)
yaml_contents = parse_yaml(args.yaml_file)
tiff_files = get_tiff_files(yaml_contents)
logging.info(f"Found {sum(len(imlist) for imlist in tiff_files)} images "
f"from {len(yaml_contents)} datasets "
f"in {len(args.yaml_file)} yaml files")
model = get_model(
existing_model=args.trained_model,
model_weights=args.model_weights,
network_depth=models[args.network_depth],
learning_rate=args.learning_rate,
continue_training=args.continue_training,
)
signal_train, background_train, labels_train = make_lists(tiff_files)
if args.test_fraction > 0:
logging.info("Splitting data into training and validation datasets")
(
signal_train,
signal_test,
background_train,
background_test,
labels_train,
labels_test,
) = train_test_split(
signal_train,
background_train,
labels_train,
test_size=args.test_fraction,
)
logging.info(f"Using {len(signal_train)} images for training and "
f"{len(signal_test)} images for validation")
validation_generator = CubeGeneratorFromDisk(
signal_test,
background_test,
labels=labels_test,
batch_size=args.batch_size,
train=True,
)
# for saving checkpoints
base_checkpoint_file_name = "-epoch.{epoch:02d}-loss-{val_loss:.3f}.h5"
else:
logging.info("No validation data selected.")
validation_generator = None
base_checkpoint_file_name = "-epoch.{epoch:02d}.h5"
training_generator = CubeGeneratorFromDisk(
signal_train,
background_train,
labels=labels_train,
batch_size=args.batch_size,
shuffle=True,
train=True,
augment=not args.no_augment,
)
callbacks = []
if args.tensorboard:
logdir = output_dir / "tensorboard"
ensure_directory_exists(logdir)
tensorboard = TensorBoard(
log_dir=logdir,
histogram_freq=0,
write_graph=True,
update_freq="epoch",
)
callbacks.append(tensorboard)
if not args.no_save_checkpoints:
if args.save_weights:
filepath = str(output_dir / ("weight" + base_checkpoint_file_name))
else:
filepath = str(output_dir / ("model" + base_checkpoint_file_name))
checkpoints = ModelCheckpoint(
filepath,
save_weights_only=args.save_weights,
)
callbacks.append(checkpoints)
if args.save_progress:
filepath = str(output_dir / "training.csv")
csv_logger = CSVLogger(filepath)
callbacks.append(csv_logger)
logging.info("Beginning training.")
model.fit(
training_generator,
validation_data=validation_generator,
use_multiprocessing=False,
epochs=args.epochs,
callbacks=callbacks,
)
if args.save_weights:
logging.info("Saving model weights")
model.save_weights(str(output_dir / "model_weights.h5"))
else:
logging.info("Saving model")
model.save(output_dir / "model.h5")
logging.info(
"Finished training, "
"Total time taken: %s",
datetime.now() - start_time,
)
def sort_mantis_files():
exp_dirs = get_subdirs(raw_tosort_fld)
if not exp_dirs:
return
# Start logging
logging.disable(logging.NOTSET)
fancylog.start_logging(raw_tosort_fld,
package,
verbose=True,
filename='mantis_sorter')
logging.info("Starting to process mantis files")
# Loop over subdirs
for subdir in exp_dirs:
# --------------------------------- GET FILES -------------------------------- #
logging.info(" processing: {}".format(subdir))
files = [f for f in listdir(subdir) if f.endswith('.tdms')]
if not files: continue
if len(files) > 3:
raise NotImplementedError("Can't deal with this many files!")
elif len(files) < 3:
raise ValueError("Found too few files")
for f in files:
for i in range(10):
if "({})".format(i + 1) in f:
raise NotImplementedError(
"Cannot deal with how files are organised in the folder, sorry. "
)
# Infer what the experiment name is
metadata_file = [f for f in files if 'meta.tdms' in f]
if not metadata_file:
logging.warning("Failed to find metadata file")
raise FileNotFoundError("Could not find metadata file")
else:
metadata_file = metadata_file[0]
# Get AI file
exp_name = os.path.split(metadata_file)[-1].split("(0)")[0]
inputs_file = [f for f in files if f.endswith(exp_name + '(0).tdms')]
if not inputs_file:
logging.warning("Failed to find analog inputs file")
raise FileNotFoundError("Could not find analog inputs file")
else:
inputs_file = inputs_file[0]
# Get video file
video_file = [
f for f in files if f != inputs_file and f != metadata_file
][0]
# ---------------------- CHECK IF MANTIS DROPPED FRAMES ---------------------- #
camera_name = os.path.split(video_file)[-1].split("(0)-")[-1].split(
".")[0]
check = check_mantis_dropped_frames(subdir,
camera_name,
exp_name,
skip_analog_inputs=True)
if check:
logging.info(" Mantis didn't drop any frames for video file")
else:
logging.info(" Mantis dropped some frames, darn it.")
# -------------------------------- MOVE FILES -------------------------------- #
# Get destination files
subshort = os.path.split(subdir)[-1]
vdest = os.path.join(raw_video_fld, subshort + '_video.tdms')
mdest = os.path.join(raw_metadata_fld,
subshort + '_video_metadata.tdms')
adest = os.path.join(raw_analog_inputs_fld, subshort + '_AI.tdms')
logging.info(" Video file: {} -> {}".format(video_file, vdest))
logging.info(" Metadata file: {} -> {}".format(
metadata_file, mdest))
logging.info(" Analog inputs file: {} -> {}".format(
inputs_file, adest))
# Move files
for src, dest in tqdm(
zip([video_file, metadata_file, inputs_file],
[vdest, mdest, adest])):
if os.path.isfile(dest):
logging.warning(
" The destination file {} already exists, stopping to avoid overwriting"
.format(dest))
raise FileExistsError(
" The destination file {} already exists, stopping to avoid overwriting"
.format(dest))
os.rename(src, dest)
# disable logging
logging.disable(sys.maxsize)
import os
from injection_finder.extraction.extraction import Extractor
import logging
from fancylog import fancylog
import fancylog as package
mouse='AY_254_2'
channel='1'
rgf = 'Z:\\swc\\branco\\BrainSaw\\{}\\cellfinder\\registration'.format(mouse)
data = os.path.join(rgf, 'downsampled_channel_{}.nii'.format(channel))
out_fld = 'Z:\\swc\\branco\\BrainSaw\\injections'
out_path = os.path.join(out_fld, mouse+'_ch{}inj.obj'.format(channel))
fancylog.start_logging(
rgf, package, verbose=True)
Extractor(
data,
rgf,
logging,
overwrite=True,
percentile_threshold=99.95,
threshold_type='perc',
obj_path=out_path,
)
