def deep_segment(
image: FileAtom,
deep_segment_pipeline: FileAtom,
anatomical_suffix: str,
count_suffix: str,
outline_suffix: str = None,
cell_min_area: int = None,
cell_mean_area: float = None,
cell_max_area: int = None,
temp_dir: str = None,
):
anatomical = image.newname_with_suffix("_" + anatomical_suffix)
count = image.newname_with_suffix("_" + count_suffix)
outline = image.newname_with_suffix(
"_" + outline_suffix) if outline_suffix else None
stage = CmdStage(inputs=(image, deep_segment_pipeline),
outputs=(anatomical, count),
cmd=['deep_segment.py',
'--segment-intensity 1',
'--temp-dir %s' % temp_dir if temp_dir else "",
'--learner %s' % deep_segment_pipeline.path,
'--image %s' % image.path,
'--image-output %s' % anatomical.path,
'--centroids-output %s' % count.path,
'--outlines-output %s' % outline.path if outline_suffix else ""
'--cell-min-area %s' % cell_min_area if cell_min_area else "",
'--process-clusters --cell-mean-area %s --cell-max-area %s' % (cell_mean_area, cell_max_area)\
if (cell_mean_area and cell_max_area) else ""
])
return Result(stages=Stages([stage]), output=(anatomical, count, outline))
def parse(cmd_str : str) -> CmdStage:
"""Create a CmdStage object from a string. (Per Jason's suggestion, we could make
an even more clever parser that simply guesses the output based on position
or a few flags like -o and tags anything else that looks like a file as input.)
>>> val = 2
>>> c = parse('mincfoo -flag {val} ,input1.mnc ,input2.mnc @output.mnc'.format(**vars()))
>>> c.outputs
['output.mnc']
>>> c.inputs
['input1.mnc', 'input2.mnc']
"""
cmd = shlex.split(cmd_str)
inputs = tuple([FileAtom(s[1:]) for s in cmd if s[0] == ',']) #TODO what about I(), O() ?
outputs = tuple([FileAtom(s[1:]) for s in cmd if s[0] == '@'])
s = CmdStage(inputs = inputs, outputs = outputs, cmd = [c if c[0] not in [',','@'] else c[1:] for c in cmd])
return s
def surface_mask2(input : MincAtom,
surface : FileAtom,
args : List[str] = []) -> Result[MincAtom]:
mask_vol = surface.newname_with_suffix("_mask", ext=".mnc")
stage = CmdStage(inputs=(input, surface),
outputs=(mask_vol,),
cmd=["surface_mask2", "-clobber"] + args + [input.path, surface.path, mask_vol.path])
return Result(stages=Stages([stage]), output=mask_vol)
def surface_mask2(input: MincAtom,
surface: FileAtom,
args: List[str] = []) -> Result[MincAtom]:
mask_vol = surface.newname_with_suffix("_mask", ext=".mnc")
stage = CmdStage(inputs=(input, surface),
outputs=(mask_vol, ),
cmd=["surface_mask2", "-clobber"] + args +
[input.path, surface.path, mask_vol.path])
return Result(stages=Stages([stage]), output=mask_vol)
def transform_objects(
input_obj: FileAtom,
xfm: XfmAtom) -> Result[FileAtom]: # XfmAtom -> XfmHandler??
output_obj = input_obj.newname_with_suffix("_resampled_via_%s" %
xfm.filename_wo_ext)
stage = CmdStage(
inputs=(input_obj, xfm),
outputs=(output_obj, ),
cmd=["transform_objects", input_obj.path, xfm.path, output_obj.path])
return Result(stages=Stages([stage]), output=output_obj)
def make_laplace_grid(input_labels : FileAtom,
label_mapping : FileAtom,
binary_closing : bool = None,
side : Optional[Side] = None):
out_grid = input_labels.newname_with_suffix("_laplace_grid" + ("_%s" % side.name if side else ""))
s = CmdStage(inputs=(input_labels, label_mapping), outputs=(out_grid,),
cmd=["make_laplace_grid", "--clobber"]
+ optional(binary_closing, "--binary_closing")
+ optional(side, "--%s" % side.name)
+ [input_labels.path, label_mapping.path, out_grid.path])
return Result(stages=Stages([s]), output=out_grid)
def tv_slice_recon_pipeline(options):
output_dir = options.application.output_directory
pipeline_name = options.application.pipeline_name
s = Stages()
df = pd.read_csv(options.application.csv_file,
dtype={
"brain_name": str,
"brain_directory": str
})
# transforms = (mbm_result.xfms.assign(
# native_file=lambda df: df.rigid_xfm.apply(lambda x: x.source),
df["mosaic_file"] = df.apply(lambda row: find_mosaic_file(row), axis=1)
df["mosaic_dictionary"] = df.apply(
lambda row: read_mosaic_file(row.mosaic_file), axis=1)
df["number_of_slices"] = df.apply(
lambda row: int(row.mosaic_dictionary["sections"]), axis=1)
df["interslice_distance"] = df.apply(
lambda row: float(row.mosaic_dictionary["sectionres"]) / 1000, axis=1)
df["Zstart"] = df.apply(lambda row: 1 if isnan(row.Zstart) else row.Zstart,
axis=1)
df["Zend"] = df.apply(lambda row: row.number_of_slices - row.Zstart + 1
if isnan(row.Zend) else row.Zend,
axis=1)
df["slice_directory"] = df.apply(lambda row: os.path.join(
output_dir, pipeline_name + "_stitched", row.brain_name),
axis=1)
#############################
# Step 1: Run TV_stitch.py
#############################
#TODO surely theres a way around this?
df = df.assign(TV_stitch_result="")
for index, row in df.iterrows():
df.at[index, "TV_stitch_result"] = s.defer(
TV_stitch_wrap(brain_directory=FileAtom(row.brain_directory),
brain_name=row.brain_name,
slice_directory=row.slice_directory,
TV_stitch_options=options.TV_stitch,
Zstart=row.Zstart,
Zend=row.Zend,
output_dir=output_dir))
df.drop(["mosaic_dictionary", "TV_stitch_result"],
axis=1).to_csv("TV_brains.csv", index=False)
df.explode("TV_stitch_result")\
.assign(slice=lambda df: df.apply(lambda row: row.TV_stitch_result.path, axis=1))\
.drop(["mosaic_dictionary", "TV_stitch_result"], axis=1)\
.to_csv("TV_slices.csv", index=False)
return Result(stages=s, output=())
def diffuse(obj_file : FileAtom,
input_signal : FileAtom,
#output_signal : FileAtom,
kernel : Optional[float] = None,
iterations : Optional[int] = None,
parametric : Optional[int] = None):
output_signal = input_signal.newname_with_suffix("_thickness")
stage = CmdStage(inputs=(obj_file, input_signal), outputs=(output_signal,),
cmd=["diffuse"] # TODO make an abstraction for this; see Nix stdlib's `optional`
+ ["-kernel", str(kernel)] if kernel is not None else []
+ ["-iterations", iterations] if iterations is not None else []
+ ["-parametric", parametric] if parametric is not None else []
+ [obj_file, input_signal, output_signal])
return Result(stages=Stages([stage]), output=output_signal)
def TV_stitch_wrap(brain_directory: FileAtom, brain_name: str,
slice_directory: str, TV_stitch_options, Zstart: int,
Zend: int, output_dir: str):
#TODO inputs should be tiles not just brain_directory
stitched = []
for z in range(Zstart, Zend + 1):
slice_stitched = FileAtom(
os.path.join(slice_directory, brain_name + "_Z%04d.tif" % z))
stitched.append(slice_stitched)
stage = CmdStage(
inputs=(brain_directory, ),
outputs=tuple(stitched),
cmd=[
'TV_stitch.py',
'--clobber',
#'--verbose',
'--Zstart %s' % Zstart,
'--Zend %s' % Zend,
'--save_positions_file %s_positions.txt' %
os.path.join(stitched[0].dir, brain_name + '_Zstart' + str(Zstart))
if TV_stitch_options.save_positions_file else "",
'--keeptmp' if TV_stitch_options.keep_tmp else "",
'--scaleoutput %s' % TV_stitch_options.scale_output
if TV_stitch_options.scale_output else '',
# '--skip_tile_match' if TV_stitch_options.skip_tile_match else '',
# '--Ystart %s' % TV_stitch_options.Ystart if TV_stitch_options.Ystart else '',
# '--Yend %s' % TV_stitch_options.Yend if TV_stitch_options.Yend else '',
# '--Xstart %s' % TV_stitch_options.Xstart if TV_stitch_options.Xstart else '',
# '--Xend %s' % TV_stitch_options.Xend if TV_stitch_options.Xend else '',
# '--nogradimag' if TV_stitch_options.no_gradient_image else '',
# '--use_positions_file %s' % TV_stitch_options.use_positions_file
# if TV_stitch_options.use_positions_file else '',
# '--overlapx %s' % TV_stitch_options.overlapx if TV_stitch_options.overlapx else '',
# '--overlapy %s' % TV_stitch_options.overlapy if TV_stitch_options.overlapy else '',
# '--channel %s' % TV_stitch_options.channel if TV_stitch_options.channel else '',
# '--Zref %s' % TV_stitch_options.Zref if TV_stitch_options.Zref else '',
# '--Zstack_pzIcorr' if TV_stitch_options.inormalize_piezo_stack else '',
# '--fastpiezo' if TV_stitch_options.fast_piezo else '',
# '--short' if TV_stitch_options.short_int else '',
# '--file_type %s' % TV_stitch_options.use_positions_file if TV_stitch_options.use_positions_file else '',
# '--TV_file_type %s' % TV_stitch_options.use_positions_file if TV_stitch_options.use_positions_file el
# '--use_IM' if TV_stitch_options.use_imagemagick else '',
os.path.join(brain_directory.path, brain_name),
os.path.join(stitched[0].dir, brain_name)
],
log_file=os.path.join(output_dir, "TV_stitch.log"))
return Result(stages=Stages([stage]), output=(stitched))
def make_laplace_grid(input_labels: FileAtom,
label_mapping: FileAtom,
binary_closing: bool = None,
side: Optional[Side] = None):
out_grid = input_labels.newname_with_suffix("_laplace_grid" +
("_%s" %
side.name if side else ""))
s = CmdStage(inputs=(input_labels, label_mapping),
outputs=(out_grid, ),
cmd=["make_laplace_grid", "--clobber"] +
optional(binary_closing, "--binary_closing") +
optional(side, "--%s" % side.name) +
[input_labels.path, label_mapping.path, out_grid.path])
return Result(stages=Stages([s]), output=out_grid)
def decimate(in_obj : FileAtom,
reduction : float,
smoothing_method : Optional[Smoothing] = None,
smoothing_iterations : Optional[int] = None):
decimated = in_obj.newname_with_suffix("_decimated_%s" % reduction
+ ("_smooth" if smoothing_method not in (Smoothing.none, None) else ""))
stage = CmdStage(inputs=(in_obj,), outputs=(decimated,),
cmd=["decimate.py"]
+ (["--smoothing-method", smoothing_method.name]
if smoothing_method not in ("none", None) else [])
+ (["--smoothing-iterations", str(smoothing_iterations)]
if smoothing_iterations is not None else [])
+ [str(reduction), in_obj.path, decimated.path])
return Result(stages=Stages([stage]), output=decimated)
def decimate(in_obj: FileAtom,
reduction: float,
smoothing_method: Optional[Smoothing] = None,
smoothing_iterations: Optional[int] = None):
decimated = in_obj.newname_with_suffix("_decimated_%s" % reduction + (
"_smooth" if smoothing_method not in (Smoothing.none, None) else ""))
stage = CmdStage(inputs=(in_obj, ),
outputs=(decimated, ),
cmd=["decimate.py"] +
(["--smoothing-method", smoothing_method.name]
if smoothing_method not in ("none", None) else []) +
(["--smoothing-iterations",
str(smoothing_iterations)]
if smoothing_iterations is not None else []) +
[str(reduction), in_obj.path, decimated.path])
return Result(stages=Stages([stage]), output=decimated)
def diffuse(
obj_file: FileAtom,
input_signal: FileAtom,
#output_signal : FileAtom,
kernel: Optional[float] = None,
iterations: Optional[int] = None,
parametric: Optional[int] = None):
output_signal = input_signal.newname_with_suffix("_thickness")
stage = CmdStage(
inputs=(obj_file, input_signal),
outputs=(output_signal, ),
cmd=[
"diffuse"
] # TODO make an abstraction for this; see Nix stdlib's `optional`
+ ["-kernel", str(kernel)] if kernel is not None else [] +
["-iterations", iterations] if iterations is not None else [] +
["-parametric", parametric] if parametric is not None else [] +
[obj_file, input_signal, output_signal])
return Result(stages=Stages([stage]), output=output_signal)
def marching_cubes(in_volume: MincAtom,
min_threshold: float = None,
max_threshold: float = None,
threshold: float = None):
if not xor(threshold is None, min_threshold is None
and max_threshold is None):
raise ValueError("specify either threshold or min and max thresholds")
out_volume = FileAtom(
in_volume.newname_with(NotImplemented)
) # forget MINCy fields # FIXME this coercion doesn't work
stage = CmdStage(
inputs=(in_volume, ),
outputs=(out_volume, ),
cmd=["marching_cubes", in_volume.path, out_volume.path] +
([str(threshold)] if threshold is not None else
[str(min_threshold), str(max_threshold)]))
return Result(stages=Stages([stage]), output=out_volume)
def saddle_recon_pipeline(options):
output_dir = options.application.output_directory
pipeline_name = options.application.pipeline_name
fid_input_dir = options.saddle_recon.varian_recon.fid_input_directory
# TODO check that the varian recon "output_file_name" doesn't name a directory, or if it does, that it matches the output directory
#The input directory should contain a host of fid files that will be used for the reconstruction of the mnc files
# TODO check that there are fid files in this directory
# TODO check that all mandatory inputs are provided
#Make list of input fid files, with location, and create a "FileAtom" type
varian_fid_files = [
fid_input_dir + "/fid" + str(num_fid)
for num_fid in range(0, options.saddle_recon.varian_recon.num_fids)
]
fids = [FileAtom(name) for name in varian_fid_files]
# Varian recon will spit out images of the format <output_file_name>.<coil#>.<rep#>.mnc
# TODO If .mnc isn't provided at the end of the output_file_name, then there is no "." before the coil number. Need to check and correct for this.
# Al/ the files created will be spit out to the output_dir
# Create list of files we expect to be produced by varian recon
coil_list_0based = [
int(x) for x in options.saddle_recon.varian_recon.mouse_list.split(',')
]
coil_list = [x + 1 for x in coil_list_0based]
file_name_base = os.path.splitext(
options.saddle_recon.varian_recon.output_file_name)[0]
varian_mnc_output = [
os.path.join(
output_dir,
file_name_base + "." + str(coil) + "_" + str(rep) + ".mnc")
for coil in coil_list
for rep in range(0, options.saddle_recon.varian_recon.num_reps)
]
varian_coil_output = [
str(coil) for coil in coil_list
for rep in range(0, options.saddle_recon.varian_recon.num_reps)
]
recon_sub_dir = [
file_name[:-6] + "_processed" for file_name in varian_mnc_output
]
imgs = [
MincAtom(varian_mnc_output[k], pipeline_sub_dir=recon_sub_dir[k])
for k in range(0, len(varian_mnc_output))
]
s = Stages()
#############################
# Step 1: Run varian_recon.py
#############################
varian_recon_results = s.defer(
varian_recon_ge3dmice_saddle(
fids=fids,
imgs=imgs,
varian_recon_options=options.saddle_recon.varian_recon,
output_dir=output_dir))
# Hold results obtained in the loop
all_lsq6_results = []
all_dc_results = []
all_crop_results = []
# Loop through all the coils
for icoil in coil_list:
icoil_imgs = numpy.array(imgs)[numpy.where(
numpy.array(varian_coil_output) == str(icoil))[0]]
icoil_varian_mnc_output = numpy.array(varian_mnc_output)[numpy.where(
numpy.array(varian_coil_output) == str(icoil))[0]]
###########################
# Step 2: lsq6 registration
###########################
# TODO change functionality of get_resolution_from_file so that it can be deferred
lsq6_dir = os.path.join(output_dir,
file_name_base + "." + str(icoil) + "_lsq6")
target_dir = os.path.join(
output_dir, file_name_base + "." + str(icoil) + "_target_file")
resolution = options.registration.resolution
#resolution = (options.registration.resolution or
# get_resolution_from_file(icoil_varian_mnc_output[0]))
options.registration = options.registration.replace(
resolution=resolution)
target_file = MincAtom(name=icoil_varian_mnc_output[0],
pipeline_sub_dir=target_dir)
targets = RegistrationTargets(registration_standard=target_file,
xfm_to_standard=None,
registration_native=None)
lsq6_result = s.defer(
lsq6_nuc_inorm(imgs=icoil_imgs,
resolution=resolution,
registration_targets=targets,
lsq6_dir=lsq6_dir,
lsq6_options=options.saddle_recon.lsq6))
all_lsq6_results.append(lsq6_result)
###########################
# Step 3: distortion correct lsq6 output image
###########################
lsq6_file = MincAtom(os.path.join(lsq6_dir, "average.mnc"),
pipeline_sub_dir=lsq6_dir)
dc_lsq6_file = MincAtom(os.path.join(lsq6_dir,
"average.aug2015_dist_corr.mnc"),
pipeline_sub_dir=lsq6_dir)
dc_result = s.defer(
dist_corr_saddle(img=lsq6_file, dc_img=dc_lsq6_file))
all_dc_results.append(dc_result)
###########################
# Step 4: crop distortion corrected lsq6 output image
###########################
cropped_dc_lsq6_file = MincAtom(os.path.join(
lsq6_dir, "average.aug2015_dist_corr.cropped.mnc"),
pipeline_sub_dir=lsq6_dir)
crop_result = s.defer(
crop_to_brain(
img=dc_lsq6_file,
cropped_img=cropped_dc_lsq6_file,
crop_to_brain_options=options.saddle_recon.crop_to_brain))
all_crop_results.append(crop_result)
return Result(stages=s,
output=Namespace(varian_output=varian_recon_results,
lsq6_output=all_lsq6_results,
dc_output=all_dc_results,
crop_output=all_crop_results))
def mbm(imgs: List[MincAtom],
options: MBMConf,
prefix: str,
output_dir: str = "",
with_maget: bool = True):
# TODO could also allow pluggable pipeline parts e.g. LSQ6 could be substituted out for the modified LSQ6
# for the kidney tips, etc...
# TODO this is tedious and annoyingly similar to the registration chain ...
lsq6_dir = os.path.join(output_dir, prefix + "_lsq6")
lsq12_dir = os.path.join(output_dir, prefix + "_lsq12")
nlin_dir = os.path.join(output_dir, prefix + "_nlin")
s = Stages()
if len(imgs) == 0:
raise ValueError("Please, some files!")
# FIXME: why do we have to call registration_targets *outside* of lsq6_nuc_inorm? is it just because of the extra
# options required? Also, shouldn't options.registration be a required input (as it contains `input_space`) ...?
targets = s.defer(
registration_targets(lsq6_conf=options.mbm.lsq6,
app_conf=options.application,
reg_conf=options.registration,
first_input_file=imgs[0].path))
# TODO this is quite tedious and duplicates stuff in the registration chain ...
resolution = (options.registration.resolution or get_resolution_from_file(
targets.registration_standard.path))
options.registration = options.registration.replace(resolution=resolution)
# FIXME: this needs to go outside of the `mbm` function to avoid being run from within other pipelines (or
# those other pipelines need to turn off this option)
if with_maget:
if options.mbm.segmentation.run_maget or options.mbm.maget.maget.mask:
# temporary fix...?
if options.mbm.maget.maget.mask and not options.mbm.segmentation.run_maget:
# which means that --no-run-maget was specified
if options.mbm.maget.maget.atlas_lib == None:
# clearly you do not want to run MAGeT at any point in this pipeline
err_msg_maget = "\nYou specified not to run MAGeT using the " \
"--no-run-maget flag. However, the code also " \
"wants to use MAGeT to generate masks for your " \
"input files after the 6 parameter alignment (lsq6). " \
"Because you did not specify a MAGeT atlas library " \
"this can not be done. \nTo run the pipeline without " \
"using MAGeT to mask your input files, please also " \
"specify: \n--maget-no-mask\n"
raise ValueError(err_msg_maget)
import copy
maget_options = copy.deepcopy(options) #Namespace(maget=options)
#maget_options
#maget_options.maget = maget_options.mbm
#maget_options.execution = options.execution
#maget_options.application = options.application
#maget_options.application.output_directory = os.path.join(options.application.output_directory, "segmentation")
maget_options.maget = options.mbm.maget
fixup_maget_options(maget_options=maget_options.maget,
nlin_options=maget_options.mbm.nlin,
lsq12_options=maget_options.mbm.lsq12)
del maget_options.mbm
#def with_new_output_dir(img : MincAtom):
#img = copy.copy(img)
#img.pipeline_sub_dir = img.pipeline_sub_dir + img.output_dir
#img.
#return img.newname_with_suffix(suffix="", subdir="segmentation")
# FIXME it probably makes most sense if the lsq6 module itself (even within lsq6_nuc_inorm) handles the run_lsq6
# setting (via use of the identity transform) since then this doesn't have to be implemented for every pipeline
if options.mbm.lsq6.run_lsq6:
lsq6_result = s.defer(
lsq6_nuc_inorm(imgs=imgs,
resolution=resolution,
registration_targets=targets,
lsq6_dir=lsq6_dir,
lsq6_options=options.mbm.lsq6))
else:
# FIXME the code shouldn't branch here based on run_lsq6 (which should probably
# be part of the lsq6 options rather than the MBM ones; see comments on #287.
# TODO don't actually do this resampling if not required (i.e., if the imgs already have the same grids)??
# however, for now need to add the masks:
identity_xfm = s.defer(
param2xfm(
out_xfm=FileAtom(name=os.path.join(lsq6_dir, 'tmp', "id.xfm"),
pipeline_sub_dir=lsq6_dir,
output_sub_dir='tmp')))
lsq6_result = [
XfmHandler(source=img,
target=img,
xfm=identity_xfm,
resampled=s.defer(
mincresample_new(img=img,
like=targets.registration_standard,
xfm=identity_xfm))) for img in imgs
]
# what about running nuc/inorm without a linear registration step??
if with_maget and options.mbm.maget.maget.mask:
masking_imgs = copy.deepcopy([xfm.resampled for xfm in lsq6_result])
masked_img = (s.defer(
maget_mask(imgs=masking_imgs,
resolution=resolution,
maget_options=maget_options.maget,
pipeline_sub_dir=os.path.join(
options.application.output_directory,
"%s_atlases" % prefix))))
masked_img.index = masked_img.apply(lambda x: x.path)
# replace any masks of the resampled images with the newly created masks:
for xfm in lsq6_result:
xfm.resampled = masked_img.loc[xfm.resampled.path]
elif with_maget:
warnings.warn(
"Not masking your images from atlas masks after LSQ6 alignment ... probably not what you want "
"(this can have negative effects on your registration and statistics)"
)
#full_hierarchy = get_nonlinear_configuration_from_options(nlin_protocol=options.mbm.nlin.nlin_protocol,
# flag_nlin_protocol=next(iter(options.mbm.nlin.flags_.nlin_protocol)),
# reg_method=options.mbm.nlin.reg_method,
# file_resolution=resolution)
#I = TypeVar("I")
#X = TypeVar("X")
#def wrap_minc(nlin_module: NLIN[I, X]) -> type[NLIN[MincAtom, XfmAtom]]:
# class N(NLIN[MincAtom, XfmAtom]): pass
# TODO now the user has to call get_nonlinear_component followed by parse_<...>; previously various things
# like lsq12_nlin_pairwise all branched on the reg_method so one didn't have to call get_nonlinear_component;
# they could still do this if it can be done safety (i.e., not breaking assumptions of various nonlinear units)
nlin_module = get_nonlinear_component(
reg_method=options.mbm.nlin.reg_method)
nlin_build_model_component = get_model_building_procedure(
options.mbm.nlin.reg_strategy,
# was: model_building.reg_strategy
reg_module=nlin_module)
# does this belong here?
# def model_building_with_initial_target_generation(prelim_model_building_component,
# final_model_building_component):
# class C(final_model_building_component):
# @staticmethod
# def build_model(imgs,
# conf : BuildModelConf,
# nlin_dir,
# nlin_prefix,
# initial_target,
# output_name_wo_ext = None): pass
#
# return C
#if options.mbm.model_building.prelim_reg_strategy is not None:
# prelim_nlin_build_model_component = get_model_building_procedure(options.mbm.model_building.prelim_reg_strategy,
# reg_module=nlin_module)
# nlin_build_model_component = model_building_with_initial_target_generation(
# final_model_building_component=nlin_build_model_component,
# prelim_model_building_component=prelim_nlin_build_model_component)
# TODO don't use name 'x_module' for something that's technically not a module ... perhaps unit/component?
# TODO tedious: why can't parse_build_model_protocol handle the null protocol case? is this something we want?
nlin_conf = (nlin_build_model_component.parse_build_model_protocol(
options.mbm.nlin.nlin_protocol, resolution=resolution)
if options.mbm.nlin.nlin_protocol is not None else
nlin_build_model_component.get_default_build_model_conf(
resolution=resolution))
lsq12_nlin_result = s.defer(
lsq12_nlin_build_model(
nlin_module=nlin_build_model_component,
imgs=[xfm.resampled for xfm in lsq6_result],
lsq12_dir=lsq12_dir,
nlin_dir=nlin_dir,
nlin_prefix=prefix,
use_robust_averaging=options.mbm.nlin.use_robust_averaging,
resolution=resolution,
lsq12_conf=options.mbm.lsq12,
nlin_conf=nlin_conf)) #options.mbm.nlin
inverted_xfms = [
s.defer(invert_xfmhandler(xfm)) for xfm in lsq12_nlin_result.output
]
if options.mbm.stats.stats_kernels:
determinants = s.defer(
determinants_at_fwhms(xfms=inverted_xfms,
inv_xfms=lsq12_nlin_result.output,
blur_fwhms=options.mbm.stats.stats_kernels))
else:
determinants = None
overall_xfms = [
s.defer(concat_xfmhandlers([rigid_xfm, lsq12_nlin_xfm])) for rigid_xfm,
lsq12_nlin_xfm in zip(lsq6_result, lsq12_nlin_result.output)
]
output_xfms = (
pd.DataFrame({
"rigid_xfm":
lsq6_result, # maybe don't return this if LSQ6 not run??
"lsq12_nlin_xfm": lsq12_nlin_result.output,
"overall_xfm": overall_xfms
}))
# we could `merge` the determinants with this table, but preserving information would cause lots of duplication
# of the transforms (or storing determinants in more columns, but iterating over dynamically known columns
# seems a bit odd ...)
# TODO transpose these fields?})
#avg_img=lsq12_nlin_result.avg_img, # inconsistent w/ WithAvgImgs[...]-style outputs
# "determinants" : determinants })
#output.avg_img = lsq12_nlin_result.avg_img
#output.determinants = determinants # TODO temporary - remove once incorporated properly into `output` proper
# TODO add more of lsq12_nlin_result?
# FIXME moved above rest of registration for debugging ... shouldn't use and destructively modify lsq6_result!!!
if with_maget and options.mbm.segmentation.run_maget:
maget_options = copy.deepcopy(maget_options)
maget_options.maget.maget.mask = maget_options.maget.maget.mask_only = False # already done above
# use the original masks here otherwise the masking step will be re-run due to the previous masking run's
# masks having been applied to the input images:
maget_result = s.defer(
maget(
[xfm.resampled for xfm in lsq6_result],
#[xfm.resampled for _ix, xfm in mbm_result.xfms.rigid_xfm.iteritems()],
options=maget_options,
prefix="%s_MAGeT" % prefix,
output_dir=os.path.join(output_dir, prefix + "_processed")))
# FIXME add pipeline dir to path and uncomment!
#maget.to_csv(path_or_buf="segmentations.csv", columns=['img', 'voted_labels'])
# TODO return some MAGeT stuff from MBM function ??
# if options.mbm.mbm.run_maget:
# import copy
# maget_options = copy.deepcopy(options) #Namespace(maget=options)
# #maget_options
# #maget_options.maget = maget_options.mbm
# #maget_options.execution = options.execution
# #maget_options.application = options.application
# maget_options.maget = options.mbm.maget
# del maget_options.mbm
#
# s.defer(maget([xfm.resampled for xfm in lsq6_result],
# options=maget_options,
# prefix="%s_MAGeT" % prefix,
# output_dir=os.path.join(output_dir, prefix + "_processed")))
# should also move outside `mbm` function ...
#if options.mbm.thickness.run_thickness:
# if not options.mbm.segmentation.run_maget:
# warnings.warn("MAGeT files (atlases, protocols) are needed to run thickness calculation.")
# # run MAGeT to segment the nlin average:
# import copy
# maget_options = copy.deepcopy(options) #Namespace(maget=options)
# maget_options.maget = options.mbm.maget
# del maget_options.mbm
# segmented_avg = s.defer(maget(imgs=[lsq12_nlin_result.avg_img],
# options=maget_options,
# output_dir=os.path.join(options.application.output_directory,
# prefix + "_processed"),
# prefix="%s_thickness_MAGeT" % prefix)).ix[0].img
# thickness = s.defer(cortical_thickness(xfms=pd.Series(inverted_xfms), atlas=segmented_avg,
# label_mapping=FileAtom(options.mbm.thickness.label_mapping),
# atlas_fwhm=0.56, thickness_fwhm=0.56)) # TODO magic fwhms
# # TODO write CSV -- should `cortical_thickness` do this/return a table?
output = Namespace(avg_img=lsq12_nlin_result.avg_img,
xfms=output_xfms,
determinants=determinants)
if with_maget and options.mbm.segmentation.run_maget:
output.maget_result = maget_result
nlin_maget = (
s.defer(
maget(
[lsq12_nlin_result.avg_img],
#[xfm.resampled for _ix, xfm in mbm_result.xfms.rigid_xfm.iteritems()],
options=maget_options,
prefix="%s_nlin_MAGeT" % prefix,
output_dir=os.path.join(
output_dir,
prefix + "_processed")))).iloc[0] #.voted_labels
#output.avg_img.mask = nlin_maget.mask # makes more sense, but might have weird effects elsewhere
output.avg_img.labels = nlin_maget.labels
return Result(stages=s, output=output)
def f():
return FileAtom('/path/to/a/file.ext')
def tv_recon_pipeline(options):
output_dir = options.application.output_directory
pipeline_name = options.application.pipeline_name
s = Stages()
slices_df = pd.read_csv(options.application.csv_file,
dtype={"brain_name": str, "brain_directory": str, "slice_directory": str})
if "qc" not in slices_df.columns:
slices_df["qc"] = False
step = int(1 / options.deep_segment.qc_fraction)
slices_df.qc[0::step] = True
#############################
# Step 1: Run deep_segment.py
#############################
#TODO surely theres a way around deep_segment_result=""?
slices_df = slices_df.assign(
deep_segment_result="",
segmentation_directory = lambda df: df.apply(
lambda row: os.path.join(output_dir, pipeline_name + "_deep_segmentation", row.brain_name), axis=1)
)
for index, row in slices_df.iterrows():
slices_df.at[index,"deep_segment_result"] = s.defer(deep_segment(image = FileAtom(row.slice,
output_sub_dir = row.segmentation_directory),
deep_segment_pipeline = FileAtom(options.deep_segment.deep_segment_pipeline),
anatomical_suffix = options.deep_segment.anatomical_name,
count_suffix = options.deep_segment.count_name,
outline_suffix = options.deep_segment.outline_name if row.qc else None,
cell_min_area = options.deep_segment.cell_min_area,
cell_mean_area = options.deep_segment.cell_mean_area,
cell_max_area = options.deep_segment.cell_max_area,
temp_dir = options.deep_segment.temp_dir
))
#hacky solution requires deep_segment() returns in that order
#https://stackoverflow.com/questions/35491274/pandas-split-column-of-lists-into-multiple-columns
slices_df[["anatomical_result", "count_result", "outline_result"]] = \
pd.DataFrame(slices_df.deep_segment_result.values.tolist())
#############################
# Step 2: Run stacks_to_volume.py
#############################
#This is annoying... If I add anything to slices_df, I will have to delete it here as well
mincs_df = slices_df.drop(['slice', 'deep_segment_result', "anatomical_result", "count_result", "outline_result", "qc"], axis=1) \
.drop_duplicates().reset_index(drop=True)\
.assign(
anatomical_list = slices_df.groupby("brain_name")['anatomical_result'].apply(list).reset_index(drop=True),
count_list=slices_df.groupby("brain_name")['count_result'].apply(list).reset_index(drop=True),
#the above is so hacky...
stacked_directory=lambda df: df.apply(
lambda row: os.path.join(output_dir, pipeline_name + "_stacked", row.brain_name), axis=1),
)
mincs_df = mincs_df.assign(
anatomical_stacked_MincAtom=lambda df: df.apply(
lambda row: MincAtom(
os.path.join(row.stacked_directory,
row.brain_name + "_" + options.deep_segment.anatomical_name + "_stacked.mnc")
), axis=1
),
count_stacked_MincAtom=lambda df: df.apply(
lambda row: MincAtom(
os.path.join(row.stacked_directory,
row.brain_name + "_" + options.deep_segment.count_name + "_stacked.mnc")
), axis=1
)
)
if not options.stacks_to_volume.manual_scale_output:
mincs_df["scale_output"] = mincs_df.interslice_distance/mincs_df.interslice_distance.min()
for index, row in mincs_df.iterrows():
s.defer(stacks_to_volume(
slices = row.anatomical_list,
output_volume = row.anatomical_stacked_MincAtom,
z_resolution = row.interslice_distance,
stacks_to_volume_options=options.stacks_to_volume,
uniform_sum=False
))
s.defer(stacks_to_volume(
slices=row.count_list,
output_volume=row.count_stacked_MincAtom,
z_resolution=row.interslice_distance,
stacks_to_volume_options=options.stacks_to_volume,
scale_output = row.scale_output,
uniform_sum=True
))
#############################
# Step 3: Run autocrop to resample to isotropic
#############################
for index, row in mincs_df.iterrows():
mincs_df.at[index,"anatomical_isotropic_result"] = s.defer(autocrop(
img = row.anatomical_stacked_MincAtom,
isostep = options.stacks_to_volume.plane_resolution,
suffix = "isotropic"
))
mincs_df.at[index, "count_isotropic_result"] = s.defer(autocrop(
img=row.count_stacked_MincAtom,
isostep=options.stacks_to_volume.plane_resolution,
suffix="isotropic",
nearest_neighbour = True
))
#############################
slices_df = slices_df.assign(
anatomical_slice = lambda df: df.apply(lambda row: row.anatomical_result.path, axis=1),
count_slice=lambda df: df.apply(lambda row: row.count_result.path, axis=1),
outline_slice=lambda df: df.apply(lambda row: row.outline_result.path if row.outline_result else None, axis=1),
)
slices_df.drop(slices_df.filter(regex='.*_directory.*|.*_result.*'), axis=1)\
.to_csv("TV_processed_slices.csv", index=False)
mincs_df = mincs_df.assign(
anatomical=lambda df: df.apply(lambda row: row.anatomical_isotropic_result.path, axis=1),
count=lambda df: df.apply(lambda row: row.count_isotropic_result.path, axis=1),
)
mincs_df.drop(mincs_df.filter(regex='.*_result.*|.*_list.*|.*_MincAtom.*'), axis=1)\
.to_csv("TV_mincs.csv", index=False)
#TODO overlay them
# s.defer(create_quality_control_images(imgs=reconstructed_mincs, montage_dir = output_dir,
# montage_output=os.path.join(output_dir, pipeline_name + "_stacked", "reconstructed_montage"),
# message="reconstructed_mincs"))
#TODO
# s.defer(create_quality_control_images(imgs=all_anatomical_pad_results, montage_dir=output_dir,
# montage_output=os.path.join(output_dir, pipeline_name + "_stacked",
# "%s_montage" % anatomical),
# message="%s_mincs" % anatomical))
# s.defer(create_quality_control_images(imgs=all_count_pad_results, montage_dir=output_dir,
# montage_output=os.path.join(output_dir, pipeline_name + "_stacked",
# "%s_montage" % count),
# auto_range=True,
# message="%s_mincs" % count))
return Result(stages=s, output=())
def transform_objects(input_obj : FileAtom,
xfm : XfmAtom) -> Result[FileAtom]: # XfmAtom -> XfmHandler??
output_obj = input_obj.newname_with_suffix("_resampled_via_%s" % xfm.filename_wo_ext)
stage = CmdStage(inputs=(input_obj, xfm), outputs=(output_obj,),
cmd=["transform_objects", input_obj.path, xfm.path, output_obj.path])
return Result(stages=Stages([stage]), output=output_obj)
