def test_get_cells():
cells = cell_io.get_cells(xml_path)
assert len(cells) == 65
assert Cell([2536, 523, 1286], 1) == cells[64]
cells = cell_io.get_cells(cubes_dir)
assert len(cells) == 4
assert natsorted(cubes_cells) == natsorted(cells)
cells = cell_io.get_cells(roi_sorter_output_dir)
assert len(cells) == 4
assert natsorted(roi_sorter_cells) == natsorted(cells)
with pytest.raises(NotImplementedError):
assert cell_io.get_cells("misc_format.abc")
def __load_cells(self):
self.cells = get_cells(self.cells_file)
if not self.cells:
logging.error(f"No cells found, exiting. "
f"Please check the file: {self.cells_file}")
raise ValueError(f"No cells found, exiting. "
f"Please check the file: {self.cells_file}")
def cells_to_brainrender(
cells_file,
output_filename,
pixel_size_x=10,
pixel_size_y=10,
pixel_size_z=10,
max_z=13200,
key="df",
):
print(f"Converting file: {cells_file}")
cells = cells_io.get_cells(cells_file)
cells = cells_io.cells_to_dataframe(cells)
cells["x"] = cells["x"] * pixel_size_x
cells["y"] = cells["y"] * pixel_size_y
cells["z"] = cells["z"] * pixel_size_z
cells.columns = ["z", "y", "x", "type"]
cells["x"] = max_z - cells["x"]
print(f"Saving to: {output_filename}")
cells.to_hdf(output_filename, key=key, mode="w")
print("Finished")
def get_cells_data(xml_file_path, structures_file_path, cells_only=True):
cells = get_cells(xml_file_path, cells_only=cells_only)
if not cells:
raise summary_tools.CellCountMissingCellsException(
"No cells found in file: {}".format(xml_file_path))
structures = get_structures_tree(structures_file_path)
root = structures[0]
return root, cells
def test_detection_full(tmpdir):
tmpdir = str(tmpdir)
cellfinder_args = [
"cellfinder_run",
"-s",
signal_data,
"-b",
background_data,
"-o",
tmpdir,
"-x",
x_pix,
"-y",
y_pix,
"-z",
z_pix,
"--n-free-cpus",
"0",
"--no-standard-space",
"--save-planes",
]
sys.argv = cellfinder_args
cellfinder_run()
cells_test_xml = os.path.join(tmpdir, "cell_classification.xml")
cells_validation = cell_io.get_cells(cells_validation_xml)
cells_test = cell_io.get_cells(cells_test_xml)
num_non_cells_validation = sum(
[cell.type == 1 for cell in cells_validation]
)
num_cells_validation = sum([cell.type == 2 for cell in cells_validation])
num_non_cells_test = sum([cell.type == 1 for cell in cells_test])
num_cells_test = sum([cell.type == 2 for cell in cells_test])
assert isclose(
num_non_cells_validation,
num_non_cells_test,
abs_tol=DETECTION_TOLERANCE,
)
assert isclose(
num_cells_validation, num_cells_test, abs_tol=DETECTION_TOLERANCE
)
def xml_scale(
xml_file,
x_scale=1,
y_scale=1,
z_scale=1,
output_directory=None,
integer=True,
):
# TODO: add a csv option
"""
To rescale the cell positions within an XML file. For compatibility with
other software, or if data has been scaled after cell detection.
:param xml_file: Any cellfinder xml file
:param x_scale: Rescaling factor in the first dimension
:param y_scale: Rescaling factor in the second dimension
:param z_scale: Rescaling factor in the third dimension
:param output_directory: Directory to save the rescaled XML file.
Defaults to the same directory as the input XML file
:param integer: Force integer cell positions (default: True)
:return:
"""
if x_scale == y_scale == z_scale == 1:
raise CommandLineInputError("All rescaling factors are 1, "
"please check the input.")
else:
input_file = Path(xml_file)
start_time = datetime.now()
cells = cio.get_cells(xml_file)
for cell in cells:
cell.transform(
x_scale=x_scale,
y_scale=y_scale,
z_scale=z_scale,
integer=integer,
)
if output_directory:
output_directory = Path(output_directory)
else:
output_directory = input_file.parent
ensure_directory_exists(output_directory)
output_filename = output_directory / (input_file.stem + "_rescaled")
output_filename = output_filename.with_suffix(input_file.suffix)
cio.save_cells(cells, output_filename)
print("Finished. Total time taken: {}".format(datetime.now() -
start_time))
def get_cell_location_array(
cell_file,
cell_position_scaling=[None, None, None],
cells_only=False,
type_str="type",
integer=True,
):
"""
Loads a cell file, and converts to an array, with 3 columns of x,y,z
positions
:param cell_file: Any supported cell file, e.g. xml
:param cell_position_scaling: list of cell scaling (raw -> final) for
[x, y, z]
:param cells_only: If only cells (rather than unknown or artifacts)
should be included
:param str type_str: String defining the title of the cell type column
in the dataframe. Used to remove non cells (artifacts), and then to clean
up the dataframe to be converted into a numpy array.
:param integer: Force integer cell positions (default: True)
:return: Array of cell positions, with x,y,z columns
"""
logging.debug("Loading cells")
cells = cell_io.get_cells(cell_file)
if cell_position_scaling != [None, None, None]:
for cell in cells:
cell.transform(
x_scale=cell_position_scaling[0],
y_scale=cell_position_scaling[1],
z_scale=cell_position_scaling[2],
integer=integer,
)
cells = cell_io.cells_to_dataframe(cells)
num_cells = len(cells[cells[type_str] == Cell.CELL])
num_non_cells = len(cells[cells[type_str] == Cell.NO_CELL])
logging.debug("{} cells, and {} non-cells".format(num_cells,
num_non_cells))
if cells_only:
logging.debug("Removing non cells")
cells = cells[cells[type_str] == Cell.CELL]
logging.debug("Tidying up dataframe to convert to array")
cells.drop(type_str, axis=1, inplace=True)
return cells.to_numpy()
def run_xml_scale(xml_file, x_scale, y_scale, z_scale, output_dir):
cellfinder_xml_scale_args = [
"cellfinder_xml_scale",
xml_file,
"-x",
str(x_scale),
"-y",
str(y_scale),
"-z",
str(z_scale),
"-o",
str(output_dir),
]
sys.argv = cellfinder_xml_scale_args
cellfinder_xml_scale_run()
scaled_cells = get_cells(os.path.join(output_dir, SCALED_XML_FILE_NAME))
return scaled_cells
def transform_cells_to_standard_space(args):
if args.registration_config is None:
args.registration_config = source_custom_config()
reg_params = RegistrationParams(
args.registration_config,
affine_n_steps=args.affine_n_steps,
affine_use_n_steps=args.affine_use_n_steps,
freeform_n_steps=args.freeform_n_steps,
freeform_use_n_steps=args.freeform_use_n_steps,
bending_energy_weight=args.bending_energy_weight,
grid_spacing_x=args.grid_spacing_x,
smoothing_sigma_reference=args.smoothing_sigma_reference,
smoothing_sigma_floating=args.smoothing_sigma_floating,
histogram_n_bins_floating=args.histogram_n_bins_floating,
histogram_n_bins_reference=args.histogram_n_bins_reference,
)
generate_deformation_field(args, reg_params)
cells_only = not args.transform_all
cells = get_cells(args.paths.classification_out_file,
cells_only=cells_only)
logging.info("Loading deformation field")
deformation_field = load_any_image(args.paths.tmp__deformation_field)
scales = get_scales(args, reg_params)
field_scales = get_deformation_field_scales(reg_params)
logging.info("Transforming cell positions")
transformed_cells = transform_cell_positions(cells, deformation_field,
field_scales, scales)
logging.info("Saving transformed cell positions")
save_cells(
transformed_cells,
args.paths.cells_in_standard_space,
save_csv=args.save_csv,
)
if not args.debug:
logging.info("Removing standard space transformation temp files")
tools.delete_temp(args.paths.standard_space_output_folder, args.paths)
def test_classify(tmpdir):
tmpdir = str(tmpdir)
tmpdir = os.path.join(os.getcwd(), tmpdir)
new_cubes_dir = os.path.join(tmpdir, "cubes")
# copying so we can test the removal of intermediate files with debug set
# to false. Otherwise will delete cubes in tests/data
shutil.copytree(cubes_dir, new_cubes_dir)
args = ClassifyArgs(tmpdir, new_cubes_dir)
args = prep_classification(args)
classify.main(args)
cells_test = cell_io.get_cells(args.paths.classification_out_file)
cells_test.sort(key=lambda x: x.z)
cells_validation.sort(key=lambda x: x.z)
assert cells_validation == cells_test
def test_xml_scale_cli(tmpdir):
scaled_cells = run_xml_scale(orig_xml_path, 0.5, 0.5, 1, tmpdir)
assert scaled_cells == get_cells(half_scale_scaled_xml_path)
scaled_cells = run_xml_scale(orig_xml_path, 10, 100, 1000, tmpdir)
assert scaled_cells == get_cells(order_magnitude_scaled_xml_path)
def test_cells_to_xml(tmpdir):
cells = cell_io.get_cells(xml_path)
tmp_cells_out_path = os.path.join(str(tmpdir), "cells.xml")
cell_io.cells_to_xml(cells, tmp_cells_out_path)
assert cells == cell_io.get_cells(tmp_cells_out_path)
def main(args):
start_time = datetime.now()
cells = get_cells(args.paths.cells_file_path)
if not cells:
logging.error("No cells found, exiting. Please check your "
"cell xml file path: {}"
" or verify your cell types "
"(maybe use cells-only option to disable)".format(
args.paths.cells_file_path))
raise ValueError("No cells found, exiting. Please check your "
"cell xml file path: {}"
" or verify your cell types "
"(maybe use cells-only option to disable)".format(
args.paths.cells_file_path))
if args.z_pixel_um != args.z_pixel_um_network:
plane_scaling_factor = args.z_pixel_um_network / args.z_pixel_um
num_planes_needed_for_cube = round(args.cube_depth *
plane_scaling_factor)
else:
num_planes_needed_for_cube = args.cube_depth
planes_paths = {}
# Use args.paths for this
all_channel_ids = args.signal_ch_ids + [args.background_ch_id]
for idx, planes_paths_file_path in enumerate(args.all_planes_paths):
channel = all_channel_ids[idx]
if args.cube_extract_cli:
channel_list = all_channel_ids
args.signal_channel = all_channel_ids[0]
else:
# only extract those channels that are necessary for classification
channel_list = [args.signal_channel, args.background_ch_id]
if channel in channel_list:
planes_paths[channel] = system.get_sorted_file_paths(
planes_paths_file_path, file_extension="tif")
if num_planes_needed_for_cube > len(planes_paths[0]):
raise StackSizeError("The number of planes provided is not sufficient "
"for any cubes to be extracted. Please check the "
"input data")
first_plane = tifffile.imread(list(planes_paths.values())[0][0])
planes_shape = first_plane.shape
brain_depth = len(list(planes_paths.values())[0])
# TODO: use to assert all centre planes processed
center_planes = sorted(list(set([cell.z for cell in cells])))
# REFACTOR: rename (clashes with different meaning of planes_to_read below)
planes_to_read = np.zeros(brain_depth, dtype=np.bool)
if tools.is_even(num_planes_needed_for_cube):
half_nz = num_planes_needed_for_cube // 2
# WARNING: not centered because even
for p in center_planes:
planes_to_read[p - half_nz:p + half_nz] = 1
else:
half_nz = num_planes_needed_for_cube // 2
# centered
for p in center_planes:
planes_to_read[p - half_nz:p + half_nz + 1] = 1
planes_to_read = np.where(planes_to_read)[0]
if not planes_to_read.size:
logging.error(
f"No planes found, you need at the very least "
f"{num_planes_needed_for_cube} "
f"planes to proceed (i.e. cube z size)"
f"Brain z dimension is {brain_depth}.",
stack_info=True,
)
raise ValueError(f"No planes found, you need at the very least "
f"{num_planes_needed_for_cube} "
f"planes to proceed (i.e. cube z size)"
f"Brain z dimension is {brain_depth}.")
# TODO: check if needs to flip args.cube_width and args.cube_height
cells_groups = cell_tools.group_cells_by_z(cells)
# copies=2 is set because at all times there is a plane queue (deque)
# and an array passed to `Cube`
ram_per_process = get_ram_requirement_per_process(
planes_paths[args.signal_channel][0],
num_planes_needed_for_cube,
copies=2,
)
n_processes = system.get_num_processes(
min_free_cpu_cores=args.n_free_cpus,
ram_needed_per_process=ram_per_process,
n_max_processes=len(planes_to_read),
fraction_free_ram=0.2,
max_ram_usage=system.memory_in_bytes(args.max_ram, "GB"),
)
# TODO: don't need to extract cubes from all channels if
# n_signal_channels>1
with ProcessPoolExecutor(max_workers=n_processes) as executor:
n_planes_per_chunk = len(planes_to_read) // n_processes
for i in range(n_processes):
start_idx = i * n_planes_per_chunk
end_idx = (start_idx + n_planes_per_chunk +
num_planes_needed_for_cube - 1)
if end_idx > planes_to_read[-1]:
end_idx = None
sub_planes_to_read = planes_to_read[start_idx:end_idx]
executor.submit(
save_cubes,
cells_groups,
planes_paths,
sub_planes_to_read,
planes_shape,
args.x_pixel_um,
args.y_pixel_um,
args.x_pixel_um_network,
args.y_pixel_um_network,
num_planes_for_cube=num_planes_needed_for_cube,
cube_width=args.cube_width,
cube_height=args.cube_height,
cube_depth=args.cube_depth,
thread_id=i,
output_dir=args.paths.tmp__cubes_output_dir,
save_empty_cubes=args.save_empty_cubes,
)
total_cubes = system.get_number_of_files_in_dir(
args.paths.tmp__cubes_output_dir)
time_taken = datetime.now() - start_time
logging.info("All cubes ({}) extracted in: {}".format(
total_cubes, time_taken))
def test_group_cells_by_z():
z_planes_validate = [
1272,
1273,
1274,
1275,
1276,
1277,
1278,
1279,
1280,
1281,
1282,
1283,
1284,
1285,
1286,
1287,
1288,
1289,
1290,
1291,
1292,
1294,
1295,
1296,
1297,
1298,
]
cell_numbers_in_groups_validate = [
1,
3,
7,
8,
3,
1,
4,
3,
1,
2,
2,
1,
1,
2,
5,
2,
2,
2,
3,
1,
1,
6,
1,
1,
1,
1,
]
cells = get_cells(xml_path)
cells_groups = cell_tools.group_cells_by_z(cells)
z_planes_test = list(cells_groups.keys())
z_planes_test.sort()
assert z_planes_validate == z_planes_test
cell_numbers_in_groups_test = [
len(cells_groups[plane]) for plane in z_planes_test
]
assert cell_numbers_in_groups_validate == cell_numbers_in_groups_test