def main(args, max_workers=3): signal_paths = args.signal_planes_paths[args.signal_channel] background_paths = args.background_planes_path[0] signal_images = get_sorted_file_paths(signal_paths, file_extension="tif") background_images = get_sorted_file_paths(background_paths, file_extension="tif") # Too many workers doesn't increase speed, and uses huge amounts of RAM workers = get_num_processes(min_free_cpu_cores=args.n_free_cpus, n_max_processes=max_workers) logging.debug("Initialising cube generator") inference_generator = CubeGeneratorFromFile( args.paths.detected_points, signal_images, background_images, args.voxel_sizes, args.network_voxel_sizes, batch_size=args.batch_size, cube_width=args.cube_width, cube_height=args.cube_height, cube_depth=args.cube_depth, ) model = get_model( existing_model=args.trained_model, model_weights=args.model_weights, network_depth=models[args.network_depth], inference=True, ) logging.info("Running inference") predictions = model.predict( inference_generator, use_multiprocessing=True, workers=workers, verbose=True, ) predictions = predictions.round() predictions = predictions.astype("uint16") predictions = np.argmax(predictions, axis=1) cells_list = [] # only go through the "extractable" cells for idx, cell in enumerate(inference_generator.ordered_cells): cell.type = predictions[idx] + 1 cells_list.append(cell) logging.info("Saving classified cells") save_cells(cells_list, args.paths.classified_points, save_csv=args.save_csv) try: get_cells(args.paths.classified_points, cells_only=True) return True except MissingCellsError: return False
def prep_classification(args, what_to_run): try: get_cells(args.paths.detected_points) n_processes = get_num_processes(min_free_cpu_cores=args.n_free_cpus) prep_tensorflow(n_processes) args = prep_models(args) except MissingCellsError: what_to_run.cells_exist = False what_to_run.candidates_exist = False what_to_run.update_if_cells_required() what_to_run.update_if_candidates_required() return args
def run(args, atlas, downsampled_space): deformation_field_paths = [ args.brainreg_paths.deformation_field_0, args.brainreg_paths.deformation_field_1, args.brainreg_paths.deformation_field_2, ] cells = get_cells(args.paths.classified_points, cells_only=True) cell_list = [] for cell in cells: cell_list.append([cell.z, cell.y, cell.x]) cells = np.array(cell_list) run_analysis( cells, args.signal_planes_paths[0], args.orientation, args.voxel_sizes, atlas, deformation_field_paths, downsampled_space, args.paths.downsampled_points, args.paths.atlas_points, args.paths.brainrender_points, args.brainreg_paths.volume_csv_path, args.paths.all_points_csv, args.paths.summary_csv, )
def run( cells_file, output_filename, pixel_size_x=10, pixel_size_y=10, pixel_size_z=10, max_z=13200, key="df", ): print(f"Converting file: {cells_file}") cells = cells_io.get_cells(cells_file) for cell in cells: cell.transform( x_scale=pixel_size_x, y_scale=pixel_size_y, z_scale=pixel_size_z, integer=True, ) cells = cells_io.cells_to_dataframe(cells) cells.columns = ["z", "y", "x", "type"] cells["x"] = max_z - cells["x"] print("Ensuring output directory exists") ensure_directory_exists(output_filename.parent) print(f"Saving to: {output_filename}") cells.to_hdf(output_filename, key=key, mode="w") print("Finished")
def __load_cells(self): self.cells = get_cells(self.cells_file) if not self.cells: logging.error(f"No cells found, exiting. " f"Please check the file: {self.cells_file}") raise ValueError(f"No cells found, exiting. " f"Please check the file: {self.cells_file}")
def get_cells_data(xml_file_path, cells_only=True): cells = get_cells(xml_file_path, cells_only=cells_only) if not cells: raise CellCountMissingCellsException( "No cells found in file: {}".format(xml_file_path) ) return cells
def test_detection_full(tmpdir): tmpdir = str(tmpdir) cellfinder_args = [ "cellfinder", "-s", signal_data, "-b", background_data, "-o", tmpdir, "-v", z_pix, y_pix, x_pix, "--orientation", "psl", "--n-free-cpus", "0", "--no-register", "--save-planes", ] sys.argv = cellfinder_args cellfinder_run() cells_test_xml = os.path.join(tmpdir, "points", "cell_classification.xml") cells_validation = cell_io.get_cells(cells_validation_xml) cells_test = cell_io.get_cells(cells_test_xml) num_non_cells_validation = sum( [cell.type == 1 for cell in cells_validation] ) num_cells_validation = sum([cell.type == 2 for cell in cells_validation]) num_non_cells_test = sum([cell.type == 1 for cell in cells_test]) num_cells_test = sum([cell.type == 2 for cell in cells_test]) assert isclose( num_non_cells_validation, num_non_cells_test, abs_tol=DETECTION_TOLERANCE, ) assert isclose( num_cells_validation, num_cells_test, abs_tol=DETECTION_TOLERANCE )
def test_get_cells(): cells = cell_io.get_cells(xml_path) assert len(cells) == 65 assert Cell([2536, 523, 1286], 1) == cells[64] cells = cell_io.get_cells(yml_path) assert len(cells) == 250 assert Cell([9170, 2537, 311], 1) == cells[194] cells = cell_io.get_cells(cubes_dir) assert len(cells) == 4 assert natsorted(cubes_cells) == natsorted(cells) cells = cell_io.get_cells(roi_sorter_output_dir) assert len(cells) == 4 assert natsorted(roi_sorter_cells) == natsorted(cells) with pytest.raises(NotImplementedError): assert cell_io.get_cells("misc_format.abc")
def xml_scale( xml_file, x_scale=1, y_scale=1, z_scale=1, output_directory=None, integer=True, ): # TODO: add a csv option """ To rescale the cell positions within an XML file. For compatibility with other software, or if data has been scaled after cell detection. :param xml_file: Any cellfinder xml file :param x_scale: Rescaling factor in the first dimension :param y_scale: Rescaling factor in the second dimension :param z_scale: Rescaling factor in the third dimension :param output_directory: Directory to save the rescaled XML file. Defaults to the same directory as the input XML file :param integer: Force integer cell positions (default: True) :return: """ if x_scale == y_scale == z_scale == 1: raise CommandLineInputError( "All rescaling factors are 1, " "please check the input." ) else: input_file = Path(xml_file) start_time = datetime.now() cells = cell_io.get_cells(xml_file) for cell in cells: cell.transform( x_scale=x_scale, y_scale=y_scale, z_scale=z_scale, integer=integer, ) if output_directory: output_directory = Path(output_directory) else: output_directory = input_file.parent ensure_directory_exists(output_directory) output_filename = output_directory / (input_file.stem + "_rescaled") output_filename = output_filename.with_suffix(input_file.suffix) cell_io.save_cells(cells, output_filename) print( "Finished. Total time taken: {}".format( datetime.now() - start_time ) )
def run_extraction( output_filename, output_directory, signal_paths, background_paths, cube_depth, cube_width, cube_height, x_pixel_um, y_pixel_um, z_pixel_um, x_pixel_um_network, y_pixel_um_network, z_pixel_um_network, max_ram, n_free_cpus, save_empty_cubes, ): planes_paths = {} planes_paths[0] = get_sorted_file_paths(signal_paths, file_extension=".tif") planes_paths[1] = get_sorted_file_paths(background_paths, file_extension=".tif") all_candidates = get_cells(str(output_filename)) cells = [c for c in all_candidates if c.is_cell()] non_cells = [c for c in all_candidates if not c.is_cell()] to_extract = {"cells": cells, "non_cells": non_cells} for cell_type, cell_list in to_extract.items(): print(f"Extracting type: {cell_type}") cell_type_output_directory = output_directory / cell_type print(f"Saving to: {cell_type_output_directory}") ensure_directory_exists(str(cell_type_output_directory)) extract_cubes_main( cell_list, cell_type_output_directory, planes_paths, cube_depth, cube_width, cube_height, x_pixel_um, y_pixel_um, z_pixel_um, x_pixel_um_network, y_pixel_um_network, z_pixel_um_network, max_ram, n_free_cpus, save_empty_cubes, )
def run(args, atlas, downsampled_space): deformation_field_paths = [ args.brainreg_paths.deformation_field_0, args.brainreg_paths.deformation_field_1, args.brainreg_paths.deformation_field_2, ] cells = get_cells(args.paths.classified_points, cells_only=True) cell_list = [] for cell in cells: cell_list.append([cell.z, cell.y, cell.x]) cells = np.array(cell_list) source_shape = tuple( imio.get_size_image_from_file_paths( args.signal_planes_paths[0]).values()) source_shape = (source_shape[2], source_shape[1], source_shape[0]) source_space = bgs.AnatomicalSpace( args.orientation, shape=source_shape, resolution=[float(i) for i in args.voxel_sizes], ) transformed_cells = transform_points_to_atlas_space( cells, source_space, atlas, deformation_field_paths, downsampled_space, downsampled_points_path=args.paths.downsampled_points, atlas_points_path=args.paths.atlas_points, ) logging.info("Exporting cells to brainrender") export_points( transformed_cells, atlas.resolution[0], args.paths.brainrender_points, ) logging.info("Summarising cell positions") summarise_points( cells, transformed_cells, atlas, args.brainreg_paths.volume_csv_path, args.paths.all_points_csv, args.paths.summary_csv, )
def transform_cells_to_standard_space(args): if args.registration_config is None: args.registration_config = source_custom_config_cellfinder() reg_params = RegistrationParams( args.registration_config, affine_n_steps=args.affine_n_steps, affine_use_n_steps=args.affine_use_n_steps, freeform_n_steps=args.freeform_n_steps, freeform_use_n_steps=args.freeform_use_n_steps, bending_energy_weight=args.bending_energy_weight, grid_spacing=args.grid_spacing, smoothing_sigma_reference=args.smoothing_sigma_reference, smoothing_sigma_floating=args.smoothing_sigma_floating, histogram_n_bins_floating=args.histogram_n_bins_floating, histogram_n_bins_reference=args.histogram_n_bins_reference, ) generate_deformation_field(args, reg_params) cells_only = not args.transform_all cells = get_cells( args.paths.classification_out_file, cells_only=cells_only ) logging.info("Loading deformation field") deformation_field = load_any_image( args.paths.tmp__deformation_field, as_numpy=True ) scales = get_scales(args, reg_params) field_scales = get_deformation_field_scales(reg_params) logging.info("Transforming cell positions") transformed_cells = transform_cell_positions( cells, deformation_field, field_scales, scales ) logging.info("Saving transformed cell positions") save_cells( transformed_cells, args.paths.cells_in_standard_space, save_csv=args.save_csv, ) if not args.debug: logging.info("Removing standard space transformation temp files") delete_temp(args.paths.standard_space_output_folder, args.paths)
def get_cell_location_array( cell_file, cell_position_scaling=[None, None, None], cells_only=False, type_str="type", integer=True, ): """ Loads a cell file, and converts to an array, with 3 columns of x,y,z positions :param cell_file: Any supported cell file, e.g. xml :param cell_position_scaling: list of cell scaling (raw -> final) for [x, y, z] :param cells_only: If only cells (rather than unknown or artifacts) should be included :param str type_str: String defining the title of the cell type column in the dataframe. Used to remove non cells (artifacts), and then to clean up the dataframe to be converted into a numpy array. :param integer: Force integer cell positions (default: True) :return: Array of cell positions, with x,y,z columns """ logging.debug("Loading cells") cells = cell_io.get_cells(cell_file) if cell_position_scaling != [None, None, None]: for cell in cells: cell.transform( x_scale=cell_position_scaling[0], y_scale=cell_position_scaling[1], z_scale=cell_position_scaling[2], integer=integer, ) cells = cell_io.cells_to_dataframe(cells) num_cells = len(cells[cells[type_str] == Cell.CELL]) num_non_cells = len(cells[cells[type_str] == Cell.NO_CELL]) logging.debug("{} cells, and {} non-cells".format(num_cells, num_non_cells)) if cells_only: logging.debug("Removing non cells") cells = cells[cells[type_str] == Cell.CELL] logging.debug("Tidying up dataframe to convert to array") cells.drop(type_str, axis=1, inplace=True) return cells.to_numpy()
def run_xml_scale(xml_file, x_scale, y_scale, z_scale, output_dir): cellfinder_xml_scale_args = [ "cellfinder_xml_scale", xml_file, "-x", str(x_scale), "-y", str(y_scale), "-z", str(z_scale), "-o", str(output_dir), ] sys.argv = cellfinder_xml_scale_args cellfinder_xml_scale_run() scaled_cells = get_cells(os.path.join(output_dir, SCALED_XML_FILE_NAME)) return scaled_cells
def main( pixel_size_x=10, pixel_size_y=10, pixel_size_z=10, max_z=13200, ): args = parser().parse_args() print( f"Calculating centroid positions for cells in: {Path(args.directory).stem}" ) if args.all: print("Including all cell positions") else: print(f"Only including cell positions of type: {Cell.CELL}") xml_files = glob(args.directory + "/*.xml") results = [] for xml_file in xml_files: print(f"Processing: {Path(xml_file).stem}") cells = get_cells(xml_file) positions = [] for cell in tqdm(cells): if args.all or (cell.type == Cell.CELL): positions.append([cell.x, cell.y, cell.z]) if positions: # only if cells included positions = np.array(positions) means = positions.mean(axis=0) results.append([Path(xml_file).stem] + means.tolist()) df = pd.DataFrame(results) df.columns = ["file", "x_center_um", "y_center_um", "z_center_um"] df["x_center_um"] = df["x_center_um"] * pixel_size_x df["y_center_um"] = df["y_center_um"] * pixel_size_y df["z_center_um"] = df["z_center_um"] * pixel_size_z # df["z"] = max_z - cells["z"] filename = Path(args.directory) / "summary.csv" df.to_csv(filename, index=False)
def cells_exist(points_file): try: get_cells(points_file, cells_only=True) return True except MissingCellsError: return False
def test_cells_to_xml(tmpdir): cells = cell_io.get_cells(xml_path) tmp_cells_out_path = os.path.join(str(tmpdir), "cells.xml") cell_io.cells_to_xml(cells, tmp_cells_out_path) assert cells == cell_io.get_cells(tmp_cells_out_path)
def test_cube_extraction(tmpdir, depth=20): tmpdir = str(tmpdir) args = CubeExtractArgs(tmpdir) planes_paths = {} planes_paths[0] = get_sorted_file_paths(signal_data_dir, file_extension="tif") planes_paths[1] = get_sorted_file_paths(background_data_dir, file_extension="tif") extract_cubes.main( get_cells(args.paths.cells_file_path), args.paths.tmp__cubes_output_dir, planes_paths, args.cube_depth, args.cube_width, args.cube_height, args.voxel_sizes, args.network_voxel_sizes, args.max_ram, args.n_free_cpus, args.save_empty_cubes, ) validation_cubes = load_cubes_in_dir(validate_cubes_dir) test_cubes = load_cubes_in_dir(tmpdir) for idx, test_cube in enumerate(test_cubes): assert (validation_cubes[idx] == test_cube).all() delete_directory_contents(tmpdir) # test cube scaling args.voxel_sizes = [7.25, 2, 2] args.x_pixel_um = 2 args.y_pixel_um = 2 args.z_pixel_um = 7.25 extract_cubes.main( get_cells(args.paths.cells_file_path), args.paths.tmp__cubes_output_dir, planes_paths, args.cube_depth, args.cube_width, args.cube_height, args.voxel_sizes, args.network_voxel_sizes, args.max_ram, args.n_free_cpus, args.save_empty_cubes, ) validation_cubes_scale = load_cubes_in_dir(validate_cubes_scale_dir) test_cubes = load_cubes_in_dir(tmpdir) for idx, test_cube in enumerate(test_cubes): assert (validation_cubes_scale[idx] == test_cube).all() # test edge of data errors cell = Cell("x0y0z10", 2) plane_paths = os.listdir(signal_data_dir) first_plane = tifffile.imread(os.path.join(signal_data_dir, plane_paths[0])) stack_shape = first_plane.shape + (depth, ) stacks = {} stacks[0] = np.zeros(stack_shape, dtype=np.uint16) stacks[0][:, :, 0] = first_plane for plane in range(1, depth): im_path = os.path.join(signal_data_dir, plane_paths[plane]) stacks[0][:, :, plane] = tifffile.imread(im_path) cube = extract_cubes.Cube(cell, 0, stacks) assert (cube.data == 0).all() cell = Cell("x2500y2500z10", 2) cube = extract_cubes.Cube(cell, 0, stacks) assert (cube.data == 0).all() # test insufficient z-planes for a specific cube stacks[0] = stacks[0][:, :, 1:] cube = extract_cubes.Cube(cell, 0, stacks) assert (cube.data == 0).all() # test insufficient z-planes for any cube to be extracted at all. delete_directory_contents(tmpdir) # args.z_pixel_um = 0.1 args.voxel_sizes[0] = 0.1 with pytest.raises(extract_cubes.StackSizeError): extract_cubes.main( get_cells(args.paths.cells_file_path), args.paths.tmp__cubes_output_dir, planes_paths, args.cube_depth, args.cube_width, args.cube_height, args.voxel_sizes, args.network_voxel_sizes, args.max_ram, args.n_free_cpus, args.save_empty_cubes, )
def test_xml_scale_cli(tmpdir): scaled_cells = run_xml_scale(orig_xml_path, 0.5, 0.5, 1, tmpdir) assert scaled_cells == get_cells(half_scale_scaled_xml_path) scaled_cells = run_xml_scale(orig_xml_path, 10, 100, 1000, tmpdir) assert scaled_cells == get_cells(order_magnitude_scaled_xml_path)
def test_group_cells_by_z(): z_planes_validate = [ 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1294, 1295, 1296, 1297, 1298, ] cell_numbers_in_groups_validate = [ 1, 3, 7, 8, 3, 1, 4, 3, 1, 2, 2, 1, 1, 2, 5, 2, 2, 2, 3, 1, 1, 6, 1, 1, 1, 1, ] cell_list = get_cells(xml_path) cells_groups = cells.group_cells_by_z(cell_list) z_planes_test = list(cells_groups.keys()) z_planes_test.sort() assert z_planes_validate == z_planes_test cell_numbers_in_groups_test = [ len(cells_groups[plane]) for plane in z_planes_test ] assert cell_numbers_in_groups_validate == cell_numbers_in_groups_test
def run_all(args, what_to_run, atlas): from cellfinder_core.detect import detect from cellfinder_core.classify import classify from cellfinder_core.tools import prep from cellfinder_core.tools.IO import read_with_dask from cellfinder.analyse import analyse from cellfinder.figures import figures from cellfinder.tools.prep import ( prep_candidate_detection, prep_channel_specific_general, ) points = None signal_array = None args, what_to_run = prep_channel_specific_general(args, what_to_run) if what_to_run.detect: logging.info("Detecting cell candidates") args = prep_candidate_detection(args) signal_array = read_with_dask( args.signal_planes_paths[args.signal_channel] ) points = detect.main( signal_array, args.start_plane, args.end_plane, args.voxel_sizes, args.soma_diameter, args.max_cluster_size, args.ball_xy_size, args.ball_z_size, args.ball_overlap_fraction, args.soma_spread_factor, args.n_free_cpus, args.log_sigma_size, args.n_sds_above_mean_thresh, ) ensure_directory_exists(args.paths.points_directory) save_cells( points, args.paths.detected_points, save_csv=args.save_csv, artifact_keep=args.artifact_keep, ) else: logging.info("Skipping cell detection") points = get_cells(args.paths.detected_points) if what_to_run.classify: model_weights = prep.prep_classification( args.trained_model, args.model_weights, args.install_path, args.model, args.n_free_cpus, ) if what_to_run.classify: if points is None: points = get_cells(args.paths.detected_points) if signal_array is None: signal_array = read_with_dask( args.signal_planes_paths[args.signal_channel] ) logging.info("Running cell classification") background_array = read_with_dask(args.background_planes_path[0]) points = classify.main( points, signal_array, background_array, args.n_free_cpus, args.voxel_sizes, args.network_voxel_sizes, args.batch_size, args.cube_height, args.cube_width, args.cube_depth, args.trained_model, model_weights, args.network_depth, ) save_cells( points, args.paths.classified_points, save_csv=args.save_csv, ) what_to_run.cells_exist = cells_exist(args.paths.classified_points) else: logging.info("No cells were detected, skipping classification.") else: logging.info("Skipping cell classification") what_to_run.update_if_cells_required() if what_to_run.analyse or what_to_run.figures: downsampled_space = get_downsampled_space( atlas, args.brainreg_paths.boundaries_file_path ) if what_to_run.analyse: points = get_cells(args.paths.classified_points, cells_only=True) if len(points) == 0: logging.info("No cells detected, skipping cell position analysis") else: logging.info("Analysing cell positions") analyse.run(args, points, atlas, downsampled_space) else: logging.info("Skipping cell position analysis") if what_to_run.figures: points = get_cells(args.paths.detected_points, cells_only=True) if len(points) == 0: logging.info("No cells detected, skipping") else: logging.info("Generating figures") figures.run(args, atlas, downsampled_space.shape) else: logging.info("Skipping figure generation")