def main(args, max_workers=3):
signal_paths = args.signal_planes_paths[args.signal_channel]
background_paths = args.background_planes_path[0]
signal_images = get_sorted_file_paths(signal_paths, file_extension="tif")
background_images = get_sorted_file_paths(background_paths,
file_extension="tif")
# Too many workers doesn't increase speed, and uses huge amounts of RAM
workers = get_num_processes(min_free_cpu_cores=args.n_free_cpus,
n_max_processes=max_workers)
logging.debug("Initialising cube generator")
inference_generator = CubeGeneratorFromFile(
args.paths.detected_points,
signal_images,
background_images,
args.voxel_sizes,
args.network_voxel_sizes,
batch_size=args.batch_size,
cube_width=args.cube_width,
cube_height=args.cube_height,
cube_depth=args.cube_depth,
)
model = get_model(
existing_model=args.trained_model,
model_weights=args.model_weights,
network_depth=models[args.network_depth],
inference=True,
)
logging.info("Running inference")
predictions = model.predict(
inference_generator,
use_multiprocessing=True,
workers=workers,
verbose=True,
)
predictions = predictions.round()
predictions = predictions.astype("uint16")
predictions = np.argmax(predictions, axis=1)
cells_list = []
# only go through the "extractable" cells
for idx, cell in enumerate(inference_generator.ordered_cells):
cell.type = predictions[idx] + 1
cells_list.append(cell)
logging.info("Saving classified cells")
save_cells(cells_list,
args.paths.classified_points,
save_csv=args.save_csv)
try:
get_cells(args.paths.classified_points, cells_only=True)
return True
except MissingCellsError:
return False
def run_extraction(
output_filename,
output_directory,
signal_paths,
background_paths,
cube_depth,
cube_width,
cube_height,
x_pixel_um,
y_pixel_um,
z_pixel_um,
x_pixel_um_network,
y_pixel_um_network,
z_pixel_um_network,
max_ram,
n_free_cpus,
save_empty_cubes,
):
planes_paths = {}
planes_paths[0] = get_sorted_file_paths(signal_paths,
file_extension=".tif")
planes_paths[1] = get_sorted_file_paths(background_paths,
file_extension=".tif")
all_candidates = get_cells(str(output_filename))
cells = [c for c in all_candidates if c.is_cell()]
non_cells = [c for c in all_candidates if not c.is_cell()]
to_extract = {"cells": cells, "non_cells": non_cells}
for cell_type, cell_list in to_extract.items():
print(f"Extracting type: {cell_type}")
cell_type_output_directory = output_directory / cell_type
print(f"Saving to: {cell_type_output_directory}")
ensure_directory_exists(str(cell_type_output_directory))
extract_cubes_main(
cell_list,
cell_type_output_directory,
planes_paths,
cube_depth,
cube_width,
cube_height,
x_pixel_um,
y_pixel_um,
z_pixel_um,
x_pixel_um_network,
y_pixel_um_network,
z_pixel_um_network,
max_ram,
n_free_cpus,
save_empty_cubes,
)
def main():
args = parser().parse_args()
img_paths = get_sorted_file_paths(args.img_paths, file_extension=".tif")
cells, non_cells = get_cell_arrays(args.cells_xml)
with napari.gui_qt():
v = napari.Viewer(title="Cellfinder cell viewer")
images = magic_imread(img_paths, use_dask=True, stack=True)
v.add_image(images)
v.add_points(
non_cells,
size=args.marker_size,
n_dimensional=True,
opacity=args.opacity,
symbol=args.symbol,
face_color="lightskyblue",
name="Non-Cells",
)
v.add_points(
cells,
size=args.marker_size,
n_dimensional=True,
opacity=args.opacity,
symbol=args.symbol,
face_color="lightgoldenrodyellow",
name="Cells",
)
def add_raw_image(viewer, image_path, name):
"""
Add a raw image (as a virtual stack) to the napari viewer
:param viewer: Napari viewer object
:param image_path: Path to the raw data
:param str name: Name to give the data
"""
paths = get_sorted_file_paths(image_path, file_extension=".tif")
images = magic_imread(paths, use_dask=True, stack=True)
viewer.add_image(images, name=name, opacity=0.6, blending="additive")
def load_data(self, name="", visible=True):
try:
img_paths = get_sorted_file_paths(self.directory,
file_extension=".tif")
images = magic_imread(img_paths, use_dask=True, stack=True)
return self.viewer.add_image(images)
# return self.viewer.open(
# str(self.directory), name=name, visible=visible
# )
except ValueError:
print(f"The directory ({self.directory}) cannot be "
f"loaded, please try again.")
return
def load_raw_data_directory(self):
self.status_label.setText("Loading...")
options = QFileDialog.Options()
options |= QFileDialog.DontUseNativeDialog
directory = QFileDialog.getExistingDirectory(
self,
"Select data directory",
options=options,
)
# deal with existing dialog
if directory is not "":
directory = Path(directory)
img_paths = get_sorted_file_paths(directory,
file_extension=".tif*")
images = magic_imread(img_paths, use_dask=True, stack=True)
self.viewer.add_image(images, name=directory.stem)
self.status_label.setText("Ready")
def load_from_folder(
src_folder,
x_scaling_factor,
y_scaling_factor,
z_scaling_factor,
anti_aliasing=True,
file_extension="",
load_parallel=False,
n_free_cpus=2,
):
"""
Load a brain from a folder. All tiff files will be read sorted and assumed
to belong to the same sample.
Optionally a name_filter string can be supplied which will have to be
present in the file names for them
to be considered part of the sample
:param str src_folder:
:param float x_scaling_factor: The scaling of the brain along the x
dimension (applied on loading before return)
:param float y_scaling_factor: The scaling of the brain along the y
dimension (applied on loading before return)
:param float z_scaling_factor: The scaling of the brain along the z
dimension
:param bool anti_aliasing: Whether to apply a Gaussian filter to smooth
the image prior to down-scaling. It is crucial to filter when
down-sampling the image to avoid aliasing artifacts.
:param str file_extension: will have to be present in the file names for them\
to be considered part of the sample
:param bool load_parallel: Use multiprocessing to speedup image loading
:param int n_free_cpus: Number of cpu cores to leave free.
:return: The loaded and scaled brain
:rtype: np.ndarray
"""
paths = get_sorted_file_paths(src_folder, file_extension=file_extension)
return load_image_series(
paths,
x_scaling_factor,
y_scaling_factor,
z_scaling_factor,
load_parallel=load_parallel,
n_free_cpus=n_free_cpus,
anti_aliasing=anti_aliasing,
)
def get_size_image_from_file_paths(file_path, file_extension="tif"):
"""
Returns the size of an image (which is a list of 2D files), without loading
the whole image
:param str file_path: File containing file_paths in a text file,
or as a list.
:param str file_extension: Optional file extension (if a directory
is passed)
:return: Dict of image sizes
"""
file_path = str(file_path)
img_paths = get_sorted_file_paths(file_path, file_extension=file_extension)
z_shape = len(img_paths)
logging.debug("Loading file: {} to check raw image size"
"".format(img_paths[0]))
image_0 = load_any(img_paths[0])
y_shape, x_shape = image_0.shape
image_shape = {"x": x_shape, "y": y_shape, "z": z_shape}
return image_shape
def test_get_sorted_file_paths():
# test list
shuffled = sorted_cubes_dir.copy()
shuffle(shuffled)
assert system.get_sorted_file_paths(shuffled) == sorted_cubes_dir
# test dir
assert system.get_sorted_file_paths(cubes_dir) == sorted_cubes_dir
assert (system.get_sorted_file_paths(
cubes_dir, file_extension=".tif") == sorted_cubes_dir)
# test text file
# specifying utf8, as written on linux
assert system.get_sorted_file_paths(
jabberwocky, encoding="utf8") == get_text_lines(jabberwocky_sorted,
encoding="utf8")
# test unsupported
with pytest.raises(NotImplementedError):
system.get_sorted_file_paths(shuffled[0])
def test_cube_extraction(tmpdir, depth=20):
tmpdir = str(tmpdir)
args = CubeExtractArgs(tmpdir)
planes_paths = {}
planes_paths[0] = get_sorted_file_paths(signal_data_dir,
file_extension="tif")
planes_paths[1] = get_sorted_file_paths(background_data_dir,
file_extension="tif")
extract_cubes.main(
get_cells(args.paths.cells_file_path),
args.paths.tmp__cubes_output_dir,
planes_paths,
args.cube_depth,
args.cube_width,
args.cube_height,
args.voxel_sizes,
args.network_voxel_sizes,
args.max_ram,
args.n_free_cpus,
args.save_empty_cubes,
)
validation_cubes = load_cubes_in_dir(validate_cubes_dir)
test_cubes = load_cubes_in_dir(tmpdir)
for idx, test_cube in enumerate(test_cubes):
assert (validation_cubes[idx] == test_cube).all()
delete_directory_contents(tmpdir)
# test cube scaling
args.voxel_sizes = [7.25, 2, 2]
args.x_pixel_um = 2
args.y_pixel_um = 2
args.z_pixel_um = 7.25
extract_cubes.main(
get_cells(args.paths.cells_file_path),
args.paths.tmp__cubes_output_dir,
planes_paths,
args.cube_depth,
args.cube_width,
args.cube_height,
args.voxel_sizes,
args.network_voxel_sizes,
args.max_ram,
args.n_free_cpus,
args.save_empty_cubes,
)
validation_cubes_scale = load_cubes_in_dir(validate_cubes_scale_dir)
test_cubes = load_cubes_in_dir(tmpdir)
for idx, test_cube in enumerate(test_cubes):
assert (validation_cubes_scale[idx] == test_cube).all()
# test edge of data errors
cell = Cell("x0y0z10", 2)
plane_paths = os.listdir(signal_data_dir)
first_plane = tifffile.imread(os.path.join(signal_data_dir,
plane_paths[0]))
stack_shape = first_plane.shape + (depth, )
stacks = {}
stacks[0] = np.zeros(stack_shape, dtype=np.uint16)
stacks[0][:, :, 0] = first_plane
for plane in range(1, depth):
im_path = os.path.join(signal_data_dir, plane_paths[plane])
stacks[0][:, :, plane] = tifffile.imread(im_path)
cube = extract_cubes.Cube(cell, 0, stacks)
assert (cube.data == 0).all()
cell = Cell("x2500y2500z10", 2)
cube = extract_cubes.Cube(cell, 0, stacks)
assert (cube.data == 0).all()
# test insufficient z-planes for a specific cube
stacks[0] = stacks[0][:, :, 1:]
cube = extract_cubes.Cube(cell, 0, stacks)
assert (cube.data == 0).all()
# test insufficient z-planes for any cube to be extracted at all.
delete_directory_contents(tmpdir)
# args.z_pixel_um = 0.1
args.voxel_sizes[0] = 0.1
with pytest.raises(extract_cubes.StackSizeError):
extract_cubes.main(
get_cells(args.paths.cells_file_path),
args.paths.tmp__cubes_output_dir,
planes_paths,
args.cube_depth,
args.cube_width,
args.cube_height,
args.voxel_sizes,
args.network_voxel_sizes,
args.max_ram,
args.n_free_cpus,
args.save_empty_cubes,
)
def main(args):
n_processes = get_num_processes(min_free_cpu_cores=args.n_free_cpus)
start_time = datetime.now()
(
soma_diameter,
max_cluster_size,
ball_xy_size,
ball_z_size,
) = calculate_parameters_in_pixels(
args.x_pixel_um,
args.y_pixel_um,
args.z_pixel_um,
args.soma_diameter,
args.max_cluster_size,
args.ball_xy_size,
args.ball_z_size,
)
# file extension only used if a directory is passed
img_paths = get_sorted_file_paths(args.signal_planes_paths[0],
file_extension="tif")
if args.end_plane == -1:
args.end_plane = len(img_paths)
planes_paths_range = img_paths[args.start_plane:args.end_plane]
workers_queue = MultiprocessingQueue(maxsize=n_processes)
# WARNING: needs to be AT LEAST ball_z_size
mp_3d_filter_queue = MultiprocessingQueue(maxsize=ball_z_size)
for plane_id in range(n_processes):
# place holder for the queue to have the right size on first run
workers_queue.put(None)
clipping_val, threshold_value, ball_filter, cell_detector = setup(
img_paths[0],
soma_diameter,
ball_xy_size,
ball_z_size,
ball_overlap_fraction=args.ball_overlap_fraction,
z_offset=args.start_plane,
)
progress_bar = tqdm(total=len(planes_paths_range),
desc="Processing planes")
mp_3d_filter = Mp3DFilter(
mp_3d_filter_queue,
ball_filter,
cell_detector,
soma_diameter,
args.output_dir,
soma_size_spread_factor=args.soma_spread_factor,
progress_bar=progress_bar,
save_planes=args.save_planes,
plane_directory=args.plane_directory,
start_plane=args.start_plane,
max_cluster_size=max_cluster_size,
outlier_keep=args.outlier_keep,
artifact_keep=args.artifact_keep,
save_csv=args.save_csv,
)
# start 3D analysis (waits for planes in queue)
bf_process = multiprocessing.Process(target=mp_3d_filter.process, args=())
bf_process.start() # needs to be started before the loop
mp_tile_processor = MpTileProcessor(workers_queue, mp_3d_filter_queue)
prev_lock = Lock()
processes = []
# start 2D tile filter (output goes into queue for 3D analysis)
for plane_id, path in enumerate(planes_paths_range):
workers_queue.get()
lock = Lock()
lock.acquire()
p = multiprocessing.Process(
target=mp_tile_processor.process,
args=(
plane_id,
path,
prev_lock,
lock,
clipping_val,
threshold_value,
soma_diameter,
args.log_sigma_size,
args.n_sds_above_mean_thresh,
),
)
prev_lock = lock
processes.append(p)
p.start()
processes[-1].join()
mp_3d_filter_queue.put((None, None, None)) # Signal the end
bf_process.join()
logging.info(
"Detection complete - all planes done in : {}".format(datetime.now() -
start_time))
def main():
args = parser().parse_args()
args = define_pixel_sizes(args)
if args.output is None:
output = Path(args.cells_xml)
output_directory = output.parent
print(f"No output directory given, so setting output "
f"directory to: {output_directory}")
else:
output_directory = args.output
ensure_directory_exists(output_directory)
output_filename = output_directory / OUTPUT_NAME
img_paths = get_sorted_file_paths(args.signal_image_paths,
file_extension=".tif")
cells, labels = get_cell_labels_arrays(args.cells_xml)
properties = {"cell": labels}
with napari.gui_qt():
viewer = napari.Viewer(title="Cellfinder cell curation")
images = magic_imread(img_paths, use_dask=True, stack=True)
viewer.add_image(images)
face_color_cycle = ["lightskyblue", "lightgoldenrodyellow"]
points_layer = viewer.add_points(
cells,
properties=properties,
symbol=args.symbol,
n_dimensional=True,
size=args.marker_size,
face_color="cell",
face_color_cycle=face_color_cycle,
name="Cell candidates",
)
@viewer.bind_key("t")
def toggle_point_property(viewer):
"""Toggle point type"""
selected_points = viewer.layers[1].selected_data
if selected_points:
selected_properties = viewer.layers[1].properties["cell"][
selected_points]
toggled_properties = np.logical_not(selected_properties)
viewer.layers[1].properties["cell"][
selected_points] = toggled_properties
# Add curated cells to list
CURATED_POINTS.extend(selected_points)
print(f"{len(selected_points)} points "
f"toggled and added to the list ")
# refresh the properties colour
viewer.layers[1].refresh_colors(update_color_mapping=False)
@viewer.bind_key("c")
def confirm_point_property(viewer):
"""Confirm point type"""
selected_points = viewer.layers[1].selected_data
if selected_points:
# Add curated cells to list
CURATED_POINTS.extend(selected_points)
print(f"{len(selected_points)} points "
f"confirmed and added to the list ")
@viewer.bind_key("Control-S")
def save_curation(viewer):
"""Save file"""
if not CURATED_POINTS:
print("No cells have been confirmed or toggled, not saving")
else:
unique_cells = unique_elements_lists(CURATED_POINTS)
points = viewer.layers[1].data[unique_cells]
labels = viewer.layers[1].properties["cell"][unique_cells]
labels = labels.astype("int")
labels = labels + 1
cells_to_save = []
for idx, point in enumerate(points):
cell = Cell([point[2], point[1], point[0]], labels[idx])
cells_to_save.append(cell)
print(f"Saving results to: {output_filename}")
save_cells(cells_to_save, output_filename)
@viewer.bind_key("Alt-E")
def start_cube_extraction(viewer):
"""Extract cubes for training"""
if not output_filename.exists():
print("No curation results have been saved. "
"Please save before extracting cubes")
else:
print(f"Saving cubes to: {output_directory}")
run_extraction(
output_filename,
output_directory,
args.signal_image_paths,
args.background_image_paths,
args.cube_depth,
args.cube_width,
args.cube_height,
args.x_pixel_um,
args.y_pixel_um,
args.z_pixel_um,
args.x_pixel_um_network,
args.y_pixel_um_network,
args.z_pixel_um_network,
args.max_ram,
args.n_free_cpus,
args.save_empty_cubes,
)
print("Saving yaml file to use for training")
save_yaml_file(output_directory)
print("Closing window")
QApplication.closeAllWindows()
print("Finished! You may now annotate more "
"datasets, or go straight to training")
