def nlin_part(xfm: XfmHandler,
inv_xfm: Optional[XfmHandler] = None) -> Result[XfmHandler]:
"""
*** = non linear deformations
--- = linear (affine) deformations
Input:
xfm : ******------>
inv_xfm : <******------ (optional)
Calculated:
inv_lin_xfm : <------
Returned:
concat : ******------> +
<------
equals : ******>
Compute the nonlinear part of a transform as follows:
go forwards across xfm and then backwards across the linear part
of the inverse xfm (by first calculating the inverse or using the one supplied)
Finally, use minc_displacement to compute the resulting gridfile of the purely
nonlinear part.
The optional inv_xfm (which must be the inverse!) is an optimization -
we don't go looking for an inverse by filename munging and don't programmatically
keep a log of operations applied, so any preexisting inverse must be supplied explicitly.
"""
s = Stages()
inv_xfm = inv_xfm or s.defer(invert_xfmhandler(xfm))
inv_lin_part = s.defer(lin_from_nlin(inv_xfm))
xfm = s.defer(concat_xfmhandlers([xfm, inv_lin_part]))
return Result(stages=s, output=xfm)
def nlin_part(xfm : XfmHandler, inv_xfm : Optional[XfmHandler] = None) -> Result[XfmHandler]:
"""
*** = non linear deformations
--- = linear (affine) deformations
Input:
xfm : ******------>
inv_xfm : <******------ (optional)
Calculated:
inv_lin_xfm : <------
Returned:
concat : ******------> +
<------
equals : ******>
Compute the nonlinear part of a transform as follows:
go forwards across xfm and then backwards across the linear part
of the inverse xfm (by first calculating the inverse or using the one supplied)
Finally, use minc_displacement to compute the resulting gridfile of the purely
nonlinear part.
The optional inv_xfm (which must be the inverse!) is an optimization -
we don't go looking for an inverse by filename munging and don't programmatically
keep a log of operations applied, so any preexisting inverse must be supplied explicitly.
"""
s = Stages()
inv_xfm = inv_xfm or s.defer(invert_xfmhandler(xfm))
inv_lin_part = s.defer(lin_from_nlin(inv_xfm))
xfm = s.defer(concat_xfmhandlers([xfm, inv_lin_part]))
return Result(stages=s, output=xfm)
def NLIN_pipeline(options):
# if options.application.files is None:
# raise ValueError("Please, some files! (or try '--help')") # TODO make a util procedure for this
output_dir = options.application.output_directory
pipeline_name = options.application.pipeline_name
# TODO this is tedious and annoyingly similar to the registration chain and MBM and LSQ6 ...
processed_dir = os.path.join(output_dir, pipeline_name + "_processed")
nlin_dir = os.path.join(output_dir, pipeline_name + "_nlin")
resolution = (options.registration.resolution # TODO does using the finest resolution here make sense?
or min([get_resolution_from_file(f) for f in options.application.files]))
imgs = get_imgs(options.application)
# imgs = [MincAtom(f, pipeline_sub_dir=processed_dir) for f in options.application.files]
# determine NLIN settings by overriding defaults with
# any settings present in protocol file, if it exists
# could add a hook to print a message announcing completion, output files,
# add more stages here to make a CSV
initial_target_mask = MincAtom(options.nlin.target_mask) if options.nlin.target_mask else None
initial_target = MincAtom(options.nlin.target, mask=initial_target_mask)
full_hierarchy = get_nonlinear_configuration_from_options(nlin_protocol=options.nlin.nlin_protocol,
reg_method=options.nlin.reg_method,
file_resolution=resolution)
s = Stages()
nlin_result = s.defer(nlin_build_model(imgs, initial_target=initial_target, conf=full_hierarchy, nlin_dir=nlin_dir))
# TODO return these?
inverted_xfms = [s.defer(invert_xfmhandler(xfm)) for xfm in nlin_result.output]
if options.stats.calc_stats:
# TODO: put the stats part behind a flag ...
determinants = [s.defer(determinants_at_fwhms(
xfm=inv_xfm,
inv_xfm=xfm,
blur_fwhms=options.stats.stats_kernels))
for xfm, inv_xfm in zip(nlin_result.output, inverted_xfms)]
return Result(stages=s,
output=Namespace(nlin_xfms=nlin_result,
avg_img=nlin_result.avg_img,
determinants=determinants))
else:
# there's no consistency in what gets returned, yikes ...
return Result(stages=s, output=Namespace(nlin_xfms=nlin_result, avg_img=nlin_result.avg_img))
def NLIN_pipeline(options):
if options.application.files is None:
raise ValueError("Please, some files! (or try '--help')") # TODO make a util procedure for this
output_dir = options.application.output_directory
pipeline_name = options.application.pipeline_name
# TODO this is tedious and annoyingly similar to the registration chain and MBM and LSQ6 ...
processed_dir = os.path.join(output_dir, pipeline_name + "_processed")
nlin_dir = os.path.join(output_dir, pipeline_name + "_nlin")
resolution = (options.registration.resolution # TODO does using the finest resolution here make sense?
or min([get_resolution_from_file(f) for f in options.application.files]))
imgs = [MincAtom(f, pipeline_sub_dir=processed_dir) for f in options.application.files]
# determine NLIN settings by overriding defaults with
# any settings present in protocol file, if it exists
# could add a hook to print a message announcing completion, output files,
# add more stages here to make a CSV
initial_target_mask = MincAtom(options.nlin.target_mask) if options.nlin.target_mask else None
initial_target = MincAtom(options.nlin.target, mask=initial_target_mask)
full_hierarchy = get_nonlinear_configuration_from_options(nlin_protocol=options.nlin.nlin_protocol,
flag_nlin_protocol=next(iter(options.nlin.flags_.nlin_protocol)),
reg_method=options.nlin.reg_method,
file_resolution=resolution)
s = Stages()
nlin_result = s.defer(nlin_build_model(imgs, initial_target=initial_target, conf=full_hierarchy, nlin_dir=nlin_dir))
# TODO return these?
inverted_xfms = [s.defer(invert_xfmhandler(xfm)) for xfm in nlin_result.output]
if options.stats.calc_stats:
# TODO: put the stats part behind a flag ...
determinants = [s.defer(determinants_at_fwhms(
xfm=inv_xfm,
inv_xfm=xfm,
blur_fwhms=options.stats.stats_kernels))
for xfm, inv_xfm in zip(nlin_result.output, inverted_xfms)]
return Result(stages=s,
output=Namespace(nlin_xfms=nlin_result,
avg_img=nlin_result.avg_img,
determinants=determinants))
else:
# there's no consistency in what gets returned, yikes ...
return Result(stages=s, output=Namespace(nlin_xfms=nlin_result, avg_img=nlin_result.avg_img))
def determinants_at_fwhms(xfms : List[XfmHandler], # TODO change to pd.Series to get indexing (hence safer inv_xfm)?
blur_fwhms : str, # TODO: change back to List[float]; should unblurred dets be found automatically?
inv_xfms : Optional[List[XfmHandler]] = None) \
-> Result[pd.DataFrame]: # TODO how to write down a Pandas type here ?!
"""
The most common way to use this function is by providing
it with transformations that go from the final average
to an individual. I.e.:
*** = non linear deformations
--- = linear (affine) deformations
xfm = final-nlin ******------> individual_input
inv_xfm = final-nlin <******------ individual_input
Takes a transformation (xfm) containing
both lsq12 (scaling and shearing, the 6-parameter
rotations/translations should not be part of this) and
non-linear parts of a subject to a common/shared average
and returns the determinants of both the (forward) nonlinear
part of the xfm at the given fwhms as well as the determinants
of the full (forward) transformation. The inverse transform
may optionally be specified to avoid its recomputation (e.g.,
when passing an inverted xfm to determinants_at_fwhms,
specify the original here).
"""
s = Stages()
inv_xfms = [s.defer(invert_xfmhandler(xfm)) for xfm in xfms] if inv_xfms is None else inv_xfms
fwhms = [float(x) for x in blur_fwhms.split(',')]
df = pd.DataFrame([{"xfm" : xfm, "inv_xfm" : inv_xfm, "fwhm" : fwhm,
"nlin_det" : nlin_det, "log_nlin_det" : nlin_log_det,
"full_det" : full_det, "log_full_det" : full_log_det }
for fwhm in fwhms + [0] # was: None, but this turns to NaN in Pandas ...
for xfm, inv_xfm in zip(xfms, inv_xfms)
for full_det_and_log_det in
[s.defer(det_and_log_det(displacement_grid=s.defer(minc_displacement(xfm)),
fwhm=fwhm,
annotation="_abs"))]
for full_det, full_log_det in [(full_det_and_log_det.det, full_det_and_log_det.log_det)]
for nlin_det_and_log_det in
[s.defer(det_and_log_det(displacement_grid=s.defer(nlin_displacement(xfm,
inv_xfm=inv_xfm)),
fwhm=fwhm,
annotation="_rel"))]
for nlin_det, nlin_log_det in [(nlin_det_and_log_det.det, nlin_det_and_log_det.log_det)]])
# TODO this is terrible, and should probably be done with joins, but one gets the idea ...
# TODO remove 'inv_xfm' column?
# TODO the return of this function is 'everything', not really just 'determinants_at_fwhms' ...
return Result(stages=s, output=df)
def match_embryo_to_4D_atlas(embryo_with_volume_est,
full_4D_atlas_info,
lsq6_conf: LSQ6Conf,
lsq12_conf: MinctraccConf,
nlin_module: NLIN,
resolution: float,
nlin_options):
s = Stages()
# 1 what's the closest match in the 4D atlas?
mid_index = get_index_closest_volume_match(embryo_with_volume_est["rough_volume"].astype(float), full_4D_atlas_info)
print("Best initial match for: \n", embryo_with_volume_est["mincatom"].orig_path, " ", full_4D_atlas_info.loc[mid_index]["timepoint"])
# register embryo to closest match +/- 5 time points
# make sure we don't index outside the possible range
lowest_index = max(0, mid_index - 7)
highest_index = min(full_4D_atlas_info.shape[0] - 1, mid_index + 7)
all_transforms = [s.defer(lsq6_lsq12_nlin(source=embryo_with_volume_est["mincatom"],
target=full_4D_atlas_info.loc[i]["mincatom"],
lsq6_conf=lsq6_conf,
lsq12_conf=lsq12_conf,
nlin_module=nlin_module,
resolution=resolution,
nlin_options=nlin_options.nlin_protocol,
resampled_post_fix_string="E" + str(full_4D_atlas_info.loc[i]["timepoint"]))) for
i in range(lowest_index, highest_index + 1, 1)]
# gather stats on those registrations
# the match is determined by the sum of the magnitude
# of the inverse transformation from 4D instance -> embryo
# using the mask of the 4D instance to limit the total sum
# 1) calculate inverse
all_inv_transforms = [s.defer(invert_xfmhandler(xfm)) for xfm in all_transforms]
minc_displacement_grids = [s.defer(minc_displacement(inv_xfm)) for inv_xfm in all_inv_transforms]
magnitudes = [s.defer(mincblob(op='magnitude', grid=disp_grid)) for disp_grid in minc_displacement_grids]
return Result(stages=s, output=all_transforms)
def two_level(grouped_files_df, options: TwoLevelConf):
"""
grouped_files_df - must contain 'group':<any comparable, sortable type> and 'file':MincAtom columns
""" # TODO weird naming since the grouped_files_df isn't a GroupBy object? just files_df?
s = Stages()
if grouped_files_df.isnull().values.any():
raise ValueError("NaN values in input dataframe; can't go")
if options.mbm.lsq6.target_type == TargetType.bootstrap:
# won't work since the second level part tries to get the resolution of *its* "first input file", which
# hasn't been created. We could instead pass in a resolution to the `mbm` function,
# but instead disable for now:
raise ValueError(
"Bootstrap model building currently doesn't work with this pipeline; "
"just specify an initial target instead")
elif options.mbm.lsq6.target_type == TargetType.pride_of_models:
pride_of_models_mapping = get_pride_of_models_mapping(
pride_csv=options.mbm.lsq6.target_file,
output_dir=options.application.output_directory,
pipeline_name=options.application.pipeline_name)
# FIXME this is the same as in the 'tamarack' except for names of arguments/enclosing variables
def group_options(options, group):
options = copy.deepcopy(options)
if options.mbm.lsq6.target_type == TargetType.pride_of_models:
targets = get_closest_model_from_pride_of_models(
pride_of_models_dict=pride_of_models_mapping, time_point=group)
options.mbm.lsq6 = options.mbm.lsq6.replace(
target_type=TargetType.initial_model,
target_file=targets.registration_standard.path)
else:
# this will ensure that all groups have the same resolution -- is it necessary?
targets = s.defer(
registration_targets(
lsq6_conf=options.mbm.lsq6,
app_conf=options.application,
reg_conf=options.registration,
first_input_file=grouped_files_df.file.iloc[0]))
resolution = (options.registration.resolution
or get_resolution_from_file(
targets.registration_standard.path))
# This must happen after calling registration_targets otherwise it will resample to options.registration.resolution
options.registration = options.registration.replace(
resolution=resolution)
# no need to check common space settings here since they're turned off at the parser level
# (a bit strange)
return options
first_level_results = (
grouped_files_df.groupby(
'group', as_index=False
) # the usual annoying pattern to do a aggregate with access
.aggregate({'file': lambda files: list(files)
}) # to the groupby object's keys ... TODO: fix
.rename(columns={
'file': "files"
}).assign(build_model=lambda df: df.apply(
axis=1,
func=lambda row: s.defer(
mbm(imgs=row.files,
options=group_options(options, row.group),
prefix="%s" % row.group,
output_dir=os.path.join(
options.application.output_directory, options.
application.pipeline_name + "_first_level",
"%s_processed" % row.group))))))
# TODO replace .assign(...apply(...)...) with just an apply, producing a series right away?
# FIXME right now the same options set is being used for both levels -- use options.first/second_level
second_level_options = copy.deepcopy(options)
second_level_options.mbm.lsq6 = second_level_options.mbm.lsq6.replace(
run_lsq6=False)
second_level_options.mbm.segmentation.run_maget = False
second_level_options.mbm.maget.maget.mask_only = False
second_level_options.mbm.maget.maget.mask = False
# FIXME this is probably a hack -- instead add a --second-level-init-model option to specify which timepoint should be used
# as the initial model in the second level ??? (at this point it doesn't matter due to lack of lsq6 ...)
if second_level_options.mbm.lsq6.target_type == TargetType.pride_of_models:
second_level_options.mbm.lsq6 = second_level_options.mbm.lsq6.replace(
target_type=TargetType.
target, # target doesn't really matter as no lsq6 here, just used for resolution...
target_file=list(pride_of_models_mapping.values())
[0].registration_standard.path)
# NOTE: running lsq6_nuc_inorm here doesn't work in general (but possibly with rotational minctracc)
# since the native-space initial model is used, but our images are
# already in standard space (as we resampled there after the 1st-level lsq6).
# On the other hand, we might want to run it here (although of course NOT nuc/inorm!) in the future,
# for instance given a 'pride' of models (one for each group).
second_level_results = s.defer(
mbm(imgs=first_level_results.build_model.map(lambda m: m.avg_img),
options=second_level_options,
prefix=os.path.join(
options.application.output_directory,
options.application.pipeline_name + "_second_level")))
# FIXME sadly, `mbm` doesn't return a pd.Series of xfms, so we don't have convenient indexing ...
overall_xfms = [
s.defer(concat_xfmhandlers([xfm_1, xfm_2])) for xfms_1, xfm_2 in
zip([r.xfms.lsq12_nlin_xfm for r in first_level_results.build_model],
second_level_results.xfms.overall_xfm) for xfm_1 in xfms_1
]
resample = np.vectorize(mincresample_new, excluded={"extra_flags"})
defer = np.vectorize(s.defer)
# TODO using the avg_img here is a bit clunky -- maybe better to propagate group indices ...
# only necessary since `mbm` doesn't return DataFrames but namespaces ...
first_level_determinants = pd.concat(list(
first_level_results.build_model.apply(
lambda x: x.determinants.assign(first_level_avg=x.avg_img))),
ignore_index=True)
# first_level_xfms is only necessary because you otherwise have no access to the input file which is necessary
# for merging with the input csv. lsq12_nlin_xfm can be used to merge, and rigid_xfm contains the input file.
# If for some reason we want to output xfms in the future, just don't drop everything.
first_level_xfms = pd.concat(
list(
first_level_results.build_model.apply(
lambda x: x.xfms.assign(first_level_avg=x.avg_img))),
ignore_index=True)[["lsq12_nlin_xfm", "rigid_xfm"]]
if options.mbm.segmentation.run_maget:
maget_df = pd.DataFrame([
{
"label_file": x.labels.path,
"native_file": x.orig_path
} #, "_merge" : basename(x.orig_path)}
for x in pd.concat([
namespace.maget_result
for namespace in first_level_results.build_model
])
])
first_level_xfms = pd.merge(
left=first_level_xfms.assign(native_file=lambda df: df.rigid_xfm.
apply(lambda x: x.source.path)),
right=maget_df,
on="native_file")
first_level_determinants = (pd.merge(left=first_level_determinants,
right=first_level_xfms,
left_on="inv_xfm",
right_on="lsq12_nlin_xfm").drop(
["rigid_xfm", "lsq12_nlin_xfm"],
axis=1))
resampled_determinants = (pd.merge(
left=first_level_determinants,
right=pd.DataFrame({
'group_xfm': second_level_results.xfms.overall_xfm
}).assign(source=lambda df: df.group_xfm.apply(lambda r: r.source)),
left_on="first_level_avg",
right_on="source").assign(
resampled_log_full_det=lambda df: defer(
resample(img=df.log_full_det,
xfm=df.group_xfm.apply(lambda x: x.xfm),
like=second_level_results.avg_img)),
resampled_log_nlin_det=lambda df: defer(
resample(img=df.log_nlin_det,
xfm=df.group_xfm.apply(lambda x: x.xfm),
like=second_level_results.avg_img))))
# TODO only resamples the log determinants, but still a bit ugly ... abstract somehow?
# TODO shouldn't be called resampled_determinants since this is basically the whole (first_level) thing ...
inverted_overall_xfms = [
s.defer(invert_xfmhandler(xfm)) for xfm in overall_xfms
]
overall_determinants = (s.defer(
determinants_at_fwhms(
xfms=inverted_overall_xfms,
inv_xfms=overall_xfms,
blur_fwhms=options.mbm.stats.stats_kernels)).assign(
overall_log_full_det=lambda df: df.log_full_det,
overall_log_nlin_det=lambda df: df.log_nlin_det).drop(
['log_full_det', 'log_nlin_det'], axis=1))
# TODO return some MAGeT stuff from two_level function ??
# FIXME running MAGeT from within the `two_level` function has the same problem as running it from within `mbm`:
# it will now run when this pipeline is called from within another one (e.g., n-level), which will be
# redundant, create filename clashes, etc. -- this should be moved to `two_level_pipeline`.
# TODO do we need a `pride of atlases` for MAGeT in this pipeline ??
# TODO at the moment MAGeT is run within the MBM code, but it could be disabled there and run here
#if options.mbm.segmentation.run_maget:
# maget_options = copy.deepcopy(options)
# maget_options.maget = options.mbm.maget
# fixup_maget_options(maget_options=maget_options.maget,
# lsq12_options=maget_options.mbm.lsq12,
# nlin_options=maget_options.mbm.nlin)
# maget_options.maget.maget.mask = maget_options.maget.maget.mask_only = False # already done above
# del maget_options.mbm
# again using a weird combination of vectorized and loop constructs ...
# s.defer(maget([xfm.resampled for _ix, m in first_level_results.iterrows()
# for xfm in m.build_model.xfms.rigid_xfm],
# options=maget_options,
# prefix="%s_MAGeT" % options.application.pipeline_name,
# output_dir=os.path.join(options.application.output_directory,
# options.application.pipeline_name + "_processed")))
# TODO resampling to database model ...
# TODO there should be one table containing all determinants (first level, overall, resampled first level) for each file
# and another containing some groupwise information (averages and transforms to the common average)
return Result(stages=s,
output=Namespace(
first_level_results=first_level_results,
resampled_determinants=resampled_determinants,
overall_determinants=overall_determinants))
def two_level(grouped_files_df, options : TwoLevelConf):
"""
grouped_files_df - must contain 'group':<any comparable, sortable type> and 'file':MincAtom columns
""" # TODO weird naming since the grouped_files_df isn't a GroupBy object? just files_df?
s = Stages()
if grouped_files_df.isnull().values.any():
raise ValueError("NaN values in input dataframe; can't go")
if options.mbm.lsq6.target_type == TargetType.bootstrap:
# won't work since the second level part tries to get the resolution of *its* "first input file", which
# hasn't been created. We could instead pass in a resolution to the `mbm` function,
# but instead disable for now:
raise ValueError("Bootstrap model building currently doesn't work with this pipeline; "
"just specify an initial target instead")
elif options.mbm.lsq6.target_type == TargetType.pride_of_models:
pride_of_models_mapping = get_pride_of_models_mapping(pride_csv=options.mbm.lsq6.target_file,
output_dir=options.application.output_directory,
pipeline_name=options.application.pipeline_name)
# FIXME this is the same as in the 'tamarack' except for names of arguments/enclosing variables
def group_options(options, group):
options = copy.deepcopy(options)
if options.mbm.lsq6.target_type == TargetType.pride_of_models:
targets = get_closest_model_from_pride_of_models(pride_of_models_dict=pride_of_models_mapping,
time_point=group)
options.mbm.lsq6 = options.mbm.lsq6.replace(target_type=TargetType.initial_model,
target_file=targets.registration_standard.path)
else:
# this will ensure that all groups have the same resolution -- is it necessary?
targets = registration_targets(lsq6_conf=options.mbm.lsq6,
app_conf=options.application,
first_input_file=grouped_files_df.file.iloc[0])
resolution = (options.registration.resolution
or get_resolution_from_file(targets.registration_standard.path))
options.registration = options.registration.replace(resolution=resolution)
# no need to check common space settings here since they're turned off at the parser level
# (a bit strange)
return options
first_level_results = (
grouped_files_df
.groupby('group', as_index=False, sort=False) # the usual annoying pattern to do a aggregate with access
.aggregate({ 'file' : lambda files: list(files) }) # to the groupby object's keys ... TODO: fix
.rename(columns={ 'file' : "files" })
.assign(build_model=lambda df:
df.apply(axis=1,
func=lambda row:
s.defer(mbm(imgs=row.files,
options=group_options(options, row.group),
prefix="%s" % row.group,
output_dir=os.path.join(
options.application.output_directory,
options.application.pipeline_name + "_first_level",
"%s_processed" % row.group)))))
)
# TODO replace .assign(...apply(...)...) with just an apply, producing a series right away?
# FIXME right now the same options set is being used for both levels -- use options.first/second_level
second_level_options = copy.deepcopy(options)
second_level_options.mbm.lsq6 = second_level_options.mbm.lsq6.replace(run_lsq6=False)
second_level_options.mbm.segmentation.run_maget = False
second_level_options.mbm.maget.maget.mask_only = False
second_level_options.mbm.maget.maget.mask = False
# FIXME this is probably a hack -- instead add a --second-level-init-model option to specify which timepoint should be used
# as the initial model in the second level ??? (at this point it doesn't matter due to lack of lsq6 ...)
if second_level_options.mbm.lsq6.target_type == TargetType.pride_of_models:
second_level_options.mbm.lsq6 = second_level_options.mbm.lsq6.replace(
target_type=TargetType.target, # target doesn't really matter as no lsq6 here, just used for resolution...
target_file=list(pride_of_models_mapping.values())[0].registration_standard.path)
# NOTE: running lsq6_nuc_inorm here doesn't work in general (but possibly with rotational minctracc)
# since the native-space initial model is used, but our images are
# already in standard space (as we resampled there after the 1st-level lsq6).
# On the other hand, we might want to run it here (although of course NOT nuc/inorm!) in the future,
# for instance given a 'pride' of models (one for each group).
second_level_results = s.defer(mbm(imgs=first_level_results.build_model.map(lambda m: m.avg_img),
options=second_level_options,
prefix=os.path.join(options.application.output_directory,
options.application.pipeline_name + "_second_level")))
# FIXME sadly, `mbm` doesn't return a pd.Series of xfms, so we don't have convenient indexing ...
overall_xfms = [s.defer(concat_xfmhandlers([xfm_1, xfm_2]))
for xfms_1, xfm_2 in zip([r.xfms.lsq12_nlin_xfm for r in first_level_results.build_model],
second_level_results.xfms.overall_xfm)
for xfm_1 in xfms_1]
resample = np.vectorize(mincresample_new, excluded={"extra_flags"})
defer = np.vectorize(s.defer)
# TODO using the avg_img here is a bit clunky -- maybe better to propagate group indices ...
# only necessary since `mbm` doesn't return DataFrames but namespaces ...
first_level_determinants = pd.concat(list(first_level_results.build_model.apply(
lambda x: x.determinants.assign(first_level_avg=x.avg_img))),
ignore_index=True)
resampled_determinants = (pd.merge(
left=first_level_determinants,
right=pd.DataFrame({'group_xfm' : second_level_results.xfms.overall_xfm})
.assign(source=lambda df: df.group_xfm.apply(lambda r: r.source)),
left_on="first_level_avg",
right_on="source")
.assign(resampled_log_full_det=lambda df: defer(resample(img=df.log_full_det,
xfm=df.group_xfm.apply(lambda x: x.xfm),
like=second_level_results.avg_img)),
resampled_log_nlin_det=lambda df: defer(resample(img=df.log_nlin_det,
xfm=df.group_xfm.apply(lambda x: x.xfm),
like=second_level_results.avg_img))))
# TODO only resamples the log determinants, but still a bit ugly ... abstract somehow?
# TODO shouldn't be called resampled_determinants since this is basically the whole (first_level) thing ...
inverted_overall_xfms = [s.defer(invert_xfmhandler(xfm)) for xfm in overall_xfms]
overall_determinants = (s.defer(determinants_at_fwhms(
xfms=inverted_overall_xfms,
inv_xfms=overall_xfms,
blur_fwhms=options.mbm.stats.stats_kernels))
.assign(overall_log_full_det=lambda df: df.log_full_det,
overall_log_nlin_det=lambda df: df.log_nlin_det)
.drop(['log_full_det', 'log_nlin_det'], axis=1))
# TODO return some MAGeT stuff from two_level function ??
# FIXME running MAGeT from within the `two_level` function has the same problem as running it from within `mbm`:
# it will now run when this pipeline is called from within another one (e.g., n-level), which will be
# redundant, create filename clashes, etc. -- this should be moved to `two_level_pipeline`.
# TODO do we need a `pride of atlases` for MAGeT in this pipeline ??
# TODO at the moment MAGeT is run within the MBM code, but it could be disabled there and run here
#if options.mbm.segmentation.run_maget:
# maget_options = copy.deepcopy(options)
# maget_options.maget = options.mbm.maget
# fixup_maget_options(maget_options=maget_options.maget,
# lsq12_options=maget_options.mbm.lsq12,
# nlin_options=maget_options.mbm.nlin)
# maget_options.maget.maget.mask = maget_options.maget.maget.mask_only = False # already done above
# del maget_options.mbm
# again using a weird combination of vectorized and loop constructs ...
# s.defer(maget([xfm.resampled for _ix, m in first_level_results.iterrows()
# for xfm in m.build_model.xfms.rigid_xfm],
# options=maget_options,
# prefix="%s_MAGeT" % options.application.pipeline_name,
# output_dir=os.path.join(options.application.output_directory,
# options.application.pipeline_name + "_processed")))
# TODO resampling to database model ...
# TODO there should be one table containing all determinants (first level, overall, resampled first level) for each file
# and another containing some groupwise information (averages and transforms to the common average)
return Result(stages=s, output=Namespace(first_level_results=first_level_results,
resampled_determinants=resampled_determinants,
overall_determinants=overall_determinants))
def tamarack(imgs: pd.DataFrame, options):
# columns of the input df: `img` : MincAtom, `timept` : number, ...
# columns of the pride of models : 'timept' : number, 'model' : MincAtom
s = Stages()
# TODO some assertions that the pride_of_models, if provided, is correct, and that this is intended target type
def group_options(options, timepoint):
options = copy.deepcopy(options)
if options.mbm.lsq6.target_type == TargetType.pride_of_models:
options = copy.deepcopy(options)
targets = get_closest_model_from_pride_of_models(
pride_of_models_dict=get_pride_of_models_mapping(
pride_csv=options.mbm.lsq6.target_file,
output_dir=options.application.output_directory,
pipeline_name=options.application.pipeline_name),
time_point=timepoint)
options.mbm.lsq6 = options.mbm.lsq6.replace(
target_type=TargetType.initial_model,
target_file=targets.registration_standard.path)
# resolution = (options.registration.resolution
# or get_resolution_from_file(targets.registration_standard.path))
# options.registration = options.registration.replace(resolution=resolution)
# FIXME use of registration_standard here is quite wrong ...
# part of the trouble is that mbm calls registration_targets itself,
# so we can't send this RegistrationTargets to `mbm` directly ...
# one option: add yet another optional arg to `mbm` ...
else:
targets = s.defer(
registration_targets(lsq6_conf=options.mbm.lsq6,
app_conf=options.application,
reg_conf=options.registration,
first_input_file=imgs.filename.iloc[0]))
resolution = (options.registration.resolution
or get_resolution_from_file(
targets.registration_standard.path))
# This must happen after calling registration_targets otherwise it will resample to options.registration.resolution
options.registration = options.registration.replace(
resolution=resolution)
return options
# build all first-level models:
first_level_results = (
imgs # TODO 'group' => 'timept' ?
.groupby('group', as_index=False
) # the usual annoying pattern to do an aggregate with access
.aggregate({'file': lambda files: list(files)}
) # to the groupby object's keys ... TODO: fix
.rename(columns={
'file': "files"
}).assign(options=lambda df: df.apply(
axis=1, func=lambda row: group_options(options, row.group))
).assign(build_model=lambda df: df.apply(
axis=1,
func=lambda row: s.defer(
mbm(imgs=row.files,
options=row.options,
prefix="%s" % row.group,
output_dir=os.path.join(
options.application.output_directory, options
.application.pipeline_name + "_first_level",
"%s_processed" % row.group))))
).sort_values(by='group'))
if all(
first_level_results.options.map(
lambda opts: opts.registration.resolution) ==
first_level_results.options.iloc[0].registration.resolution):
options.registration = options.registration.replace(
resolution=first_level_results.options.iloc[0].registration.
resolution)
else:
raise ValueError(
"some first-level models are run at different resolutions, possibly not what you want ..."
)
# construction of the overall inter-average transforms will be done iteratively (for efficiency/aesthetics),
# which doesn't really fit the DataFrame mold ...
full_hierarchy = get_nonlinear_configuration_from_options(
nlin_protocol=options.mbm.nlin.nlin_protocol,
reg_method=options.mbm.nlin.reg_method,
file_resolution=options.registration.resolution)
# FIXME no good can come of this
nlin_protocol = full_hierarchy.confs[-1] if isinstance(
full_hierarchy, MultilevelANTSConf) else full_hierarchy
# first register consecutive averages together:
average_registrations = (
first_level_results[:-1].assign(
next_model=list(first_level_results[1:].build_model))
# TODO: we should be able to do lsq6 registration here as well!
.assign(xfm=lambda df: df.apply(
axis=1,
func=lambda row: s.defer(
lsq12_nlin(source=row.build_model.avg_img,
target=row.next_model.avg_img,
lsq12_conf=get_linear_configuration_from_options(
options.mbm.lsq12,
transform_type=LinearTransType.lsq12,
file_resolution=options.registration.resolution
),
nlin_conf=nlin_protocol)))))
# now compose the above transforms to produce transforms from each average to the common average:
common_time_pt = options.tamarack.common_time_pt
common_model = first_level_results[
first_level_results.group ==
common_time_pt].iloc[0].build_model.avg_img
#common = average_registrations[average_registrations.group == common_time_pt].iloc[0]
before = average_registrations[
average_registrations.group <
common_time_pt] # asymmetry in before/after since
after = average_registrations[
average_registrations.group >=
common_time_pt] # we used `next_`, not `previous_`
# compose 1st and 2nd level transforms and resample into the common average space:
def suffixes(xs):
if len(xs) == 0:
return [[]]
else:
ys = suffixes(xs[1:])
return [[xs[0]] + ys[0]] + ys
def prefixes(xs):
if len(xs) == 0:
return [[]]
else:
ys = prefixes(xs[1:])
return ys + [ys[-1] + [xs[0]]]
xfms_to_common = (first_level_results.assign(
uncomposed_xfms=suffixes(list(before.xfm))[:-1] + [None] +
prefixes(list(after.xfm))[1:]).assign(
xfm_to_common=lambda df: df.apply(
axis=1,
func=lambda row: ((lambda x: s.defer(invert_xfmhandler(
x)) if row.group >= common_time_pt else x)(s.defer(
concat_xfmhandlers(
row.uncomposed_xfms,
name=("%s_to_common" if row.group < common_time_pt
else "%s_from_common") % row.group))))
if row.uncomposed_xfms is not None else None)).drop(
'uncomposed_xfms', axis=1)) # TODO None => identity??
# TODO indexing here is not good ...
first_level_determinants = pd.concat(list(
first_level_results.build_model.apply(
lambda x: x.determinants.assign(first_level_avg=x.avg_img))),
ignore_index=True)
resampled_determinants = (pd.merge(
left=first_level_determinants,
right=xfms_to_common.assign(source=lambda df: df.xfm_to_common.apply(
lambda x: x.source if x is not None else None)),
left_on="first_level_avg",
right_on='source').assign(
resampled_log_full_det=lambda df: df.apply(
axis=1,
func=lambda row: s.defer(
mincresample_new(img=row.log_full_det,
xfm=row.xfm_to_common.xfm,
like=common_model))
if row.xfm_to_common is not None else row.img),
resampled_log_nlin_det=lambda df: df.apply(
axis=1,
func=lambda row: s.defer(
mincresample_new(img=row.log_nlin_det,
xfm=row.xfm_to_common.xfm,
like=common_model))
if row.xfm_to_common is not None else row.img)))
inverted_overall_xfms = pd.Series({
xfm: (s.defer(concat_xfmhandlers([xfm, row.xfm_to_common]))
if row.xfm_to_common is not None else xfm)
for _ix, row in xfms_to_common.iterrows()
for xfm in row.build_model.xfms.lsq12_nlin_xfm
})
overall_xfms = inverted_overall_xfms.apply(
lambda x: s.defer(invert_xfmhandler(x)))
overall_determinants = determinants_at_fwhms(
xfms=overall_xfms,
blur_fwhms=options.mbm.stats.stats_kernels,
inv_xfms=inverted_overall_xfms)
# TODO turn off bootstrap as with two-level code?
# TODO combine into one data frame
return Result(stages=s,
output=Namespace(
first_level_results=first_level_results,
overall_determinants=overall_determinants,
resampled_determinants=resampled_determinants.drop(
['options'], axis=1)))
def tamarack(imgs : pd.DataFrame, options):
# columns of the input df: `img` : MincAtom, `timept` : number, ...
# columns of the pride of models : 'timept' : number, 'model' : MincAtom
s = Stages()
# TODO some assertions that the pride_of_models, if provided, is correct, and that this is intended target type
def group_options(options, timepoint):
options = copy.deepcopy(options)
if options.mbm.lsq6.target_type == TargetType.pride_of_models:
options = copy.deepcopy(options)
targets = get_closest_model_from_pride_of_models(pride_of_models_dict=get_pride_of_models_mapping(
pride_csv=options.mbm.lsq6.target_file,
output_dir=options.application.output_directory,
pipeline_name=options.application.pipeline_name),
time_point=timepoint)
options.mbm.lsq6 = options.mbm.lsq6.replace(target_type=TargetType.initial_model,
target_file=targets.registration_standard.path)
# resolution = (options.registration.resolution
# or get_resolution_from_file(targets.registration_standard.path))
# options.registration = options.registration.replace(resolution=resolution)
# FIXME use of registration_standard here is quite wrong ...
# part of the trouble is that mbm calls registration_targets itself,
# so we can't send this RegistrationTargets to `mbm` directly ...
# one option: add yet another optional arg to `mbm` ...
else:
targets = s.defer(registration_targets(lsq6_conf=options.mbm.lsq6,
app_conf=options.application, reg_conf=options.registration,
first_input_file=imgs.filename.iloc[0]))
resolution = (options.registration.resolution or
get_resolution_from_file(targets.registration_standard.path))
# This must happen after calling registration_targets otherwise it will resample to options.registration.resolution
options.registration = options.registration.replace(resolution=resolution)
return options
# build all first-level models:
first_level_results = (
imgs # TODO 'group' => 'timept' ?
.groupby('group', as_index=False) # the usual annoying pattern to do an aggregate with access
.aggregate({ 'file' : lambda files: list(files) }) # to the groupby object's keys ... TODO: fix
.rename(columns={ 'file' : "files" })
.assign(options=lambda df: df.apply(axis=1, func=lambda row: group_options(options, row.group)))
.assign(build_model=lambda df:
df.apply(axis=1,
func=lambda row: s.defer(
mbm(imgs=row.files,
options=row.options,
prefix="%s" % row.group,
output_dir=os.path.join(
options.application.output_directory,
options.application.pipeline_name + "_first_level",
"%s_processed" % row.group)))))
.sort_values(by='group')
)
if all(first_level_results.options.map(lambda opts: opts.registration.resolution)
== first_level_results.options.iloc[0].registration.resolution):
options.registration = options.registration.replace(
resolution=first_level_results.options.iloc[0].registration.resolution)
else:
raise ValueError("some first-level models are run at different resolutions, possibly not what you want ...")
# construction of the overall inter-average transforms will be done iteratively (for efficiency/aesthetics),
# which doesn't really fit the DataFrame mold ...
full_hierarchy = get_nonlinear_configuration_from_options(
nlin_protocol=options.mbm.nlin.nlin_protocol,
reg_method=options.mbm.nlin.reg_method,
file_resolution=options.registration.resolution)
# FIXME no good can come of this
nlin_protocol = full_hierarchy.confs[-1] if isinstance(full_hierarchy, MultilevelANTSConf) else full_hierarchy
# first register consecutive averages together:
average_registrations = (
first_level_results[:-1]
.assign(next_model=list(first_level_results[1:].build_model))
# TODO: we should be able to do lsq6 registration here as well!
.assign(xfm=lambda df: df.apply(axis=1, func=lambda row: s.defer(
lsq12_nlin(source=row.build_model.avg_img,
target=row.next_model.avg_img,
lsq12_conf=get_linear_configuration_from_options(
options.mbm.lsq12,
transform_type=LinearTransType.lsq12,
file_resolution=options.registration.resolution),
nlin_conf=nlin_protocol)))))
# now compose the above transforms to produce transforms from each average to the common average:
common_time_pt = options.tamarack.common_time_pt
common_model = first_level_results[first_level_results.group == common_time_pt].iloc[0].build_model.avg_img
#common = average_registrations[average_registrations.group == common_time_pt].iloc[0]
before = average_registrations[average_registrations.group < common_time_pt] # asymmetry in before/after since
after = average_registrations[average_registrations.group >= common_time_pt] # we used `next_`, not `previous_`
# compose 1st and 2nd level transforms and resample into the common average space:
def suffixes(xs):
if len(xs) == 0:
return [[]]
else:
ys = suffixes(xs[1:])
return [[xs[0]] + ys[0]] + ys
def prefixes(xs):
if len(xs) == 0:
return [[]]
else:
ys = prefixes(xs[1:])
return ys + [ys[-1] + [xs[0]]]
xfms_to_common = (
first_level_results
.assign(uncomposed_xfms=suffixes(list(before.xfm))[:-1] + [None] + prefixes(list(after.xfm))[1:])
.assign(xfm_to_common=lambda df: df.apply(axis=1, func=lambda row:
((lambda x: s.defer(invert_xfmhandler(x)) if row.group >= common_time_pt else x)
(s.defer(concat_xfmhandlers(row.uncomposed_xfms,
name=("%s_to_common"
if row.group < common_time_pt
else "%s_from_common") % row.group))))
if row.uncomposed_xfms is not None else None))
.drop('uncomposed_xfms', axis=1)) # TODO None => identity??
# TODO indexing here is not good ...
first_level_determinants = pd.concat(list(first_level_results.build_model.apply(
lambda x: x.determinants.assign(first_level_avg=x.avg_img))),
ignore_index=True)
resampled_determinants = (
pd.merge(left=first_level_determinants,
right=xfms_to_common.assign(source=lambda df: df.xfm_to_common.apply(
lambda x:
x.source if x is not None else None)),
left_on="first_level_avg", right_on='source')
.assign(resampled_log_full_det=lambda df: df.apply(axis=1, func=lambda row:
s.defer(mincresample_new(img=row.log_full_det,
xfm=row.xfm_to_common.xfm,
like=common_model))
if row.xfm_to_common is not None else row.img),
resampled_log_nlin_det=lambda df: df.apply(axis=1, func=lambda row:
s.defer(mincresample_new(img=row.log_nlin_det,
xfm=row.xfm_to_common.xfm,
like=common_model))
if row.xfm_to_common is not None else row.img))
)
inverted_overall_xfms = pd.Series({ xfm : (s.defer(concat_xfmhandlers([xfm, row.xfm_to_common]))
if row.xfm_to_common is not None else xfm)
for _ix, row in xfms_to_common.iterrows()
for xfm in row.build_model.xfms.lsq12_nlin_xfm })
overall_xfms = inverted_overall_xfms.apply(lambda x: s.defer(invert_xfmhandler(x)))
overall_determinants = determinants_at_fwhms(xfms=overall_xfms,
blur_fwhms=options.mbm.stats.stats_kernels,
inv_xfms=inverted_overall_xfms)
# TODO turn off bootstrap as with two-level code?
# TODO combine into one data frame
return Result(stages=s, output=Namespace(first_level_results=first_level_results,
overall_determinants=overall_determinants,
resampled_determinants=resampled_determinants.drop(
['options'],
axis=1)))
def mbm(imgs : List[MincAtom], options : MBMConf, prefix : str, output_dir : str = ""):
# TODO could also allow pluggable pipeline parts e.g. LSQ6 could be substituted out for the modified LSQ6
# for the kidney tips, etc...
# TODO this is tedious and annoyingly similar to the registration chain ...
lsq6_dir = os.path.join(output_dir, prefix + "_lsq6")
lsq12_dir = os.path.join(output_dir, prefix + "_lsq12")
nlin_dir = os.path.join(output_dir, prefix + "_nlin")
s = Stages()
if len(imgs) == 0:
raise ValueError("Please, some files!")
# FIXME: why do we have to call registration_targets *outside* of lsq6_nuc_inorm? is it just because of the extra
# options required? Also, shouldn't options.registration be a required input (as it contains `input_space`) ...?
targets = registration_targets(lsq6_conf=options.mbm.lsq6,
app_conf=options.application,
first_input_file=imgs[0].path)
# TODO this is quite tedious and duplicates stuff in the registration chain ...
resolution = (options.registration.resolution or
get_resolution_from_file(targets.registration_standard.path))
options.registration = options.registration.replace(resolution=resolution)
# FIXME it probably makes most sense if the lsq6 module itself (even within lsq6_nuc_inorm) handles the run_lsq6
# setting (via use of the identity transform) since then this doesn't have to be implemented for every pipeline
if options.mbm.lsq6.run_lsq6:
lsq6_result = s.defer(lsq6_nuc_inorm(imgs=imgs,
resolution=resolution,
registration_targets=targets,
lsq6_dir=lsq6_dir,
lsq6_options=options.mbm.lsq6))
else:
# TODO don't actually do this resampling if not required (i.e., if the imgs already have the same grids)
identity_xfm = s.defer(param2xfm(out_xfm=FileAtom(name="identity.xfm")))
lsq6_result = [XfmHandler(source=img, target=img, xfm=identity_xfm,
resampled=s.defer(mincresample_new(img=img,
like=targets.registration_standard,
xfm=identity_xfm)))
for img in imgs]
# what about running nuc/inorm without a linear registration step??
full_hierarchy = get_nonlinear_configuration_from_options(nlin_protocol=options.mbm.nlin.nlin_protocol,
reg_method=options.mbm.nlin.reg_method,
file_resolution=resolution)
lsq12_nlin_result = s.defer(lsq12_nlin_build_model(imgs=[xfm.resampled for xfm in lsq6_result],
resolution=resolution,
lsq12_dir=lsq12_dir,
nlin_dir=nlin_dir,
nlin_prefix=prefix,
lsq12_conf=options.mbm.lsq12,
nlin_conf=full_hierarchy))
inverted_xfms = [s.defer(invert_xfmhandler(xfm)) for xfm in lsq12_nlin_result.output]
determinants = s.defer(determinants_at_fwhms(
xfms=inverted_xfms,
inv_xfms=lsq12_nlin_result.output,
blur_fwhms=options.mbm.stats.stats_kernels))
overall_xfms = [s.defer(concat_xfmhandlers([rigid_xfm, lsq12_nlin_xfm]))
for rigid_xfm, lsq12_nlin_xfm in zip(lsq6_result, lsq12_nlin_result.output)]
output_xfms = (pd.DataFrame({ "rigid_xfm" : lsq6_result, # maybe don't return this if LSQ6 not run??
"lsq12_nlin_xfm" : lsq12_nlin_result.output,
"overall_xfm" : overall_xfms }))
# we could `merge` the determinants with this table, but preserving information would cause lots of duplication
# of the transforms (or storing determinants in more columns, but iterating over dynamically known columns
# seems a bit odd ...)
# TODO transpose these fields?})
#avg_img=lsq12_nlin_result.avg_img, # inconsistent w/ WithAvgImgs[...]-style outputs
# "determinants" : determinants })
#output.avg_img = lsq12_nlin_result.avg_img
#output.determinants = determinants # TODO temporary - remove once incorporated properly into `output` proper
# TODO add more of lsq12_nlin_result?
# FIXME: this needs to go outside of the `mbm` function to avoid being run from within other pipelines (or
# those other pipelines need to turn off this option)
# TODO return some MAGeT stuff from MBM function ??
# if options.mbm.mbm.run_maget:
# import copy
# maget_options = copy.deepcopy(options) #Namespace(maget=options)
# #maget_options
# #maget_options.maget = maget_options.mbm
# #maget_options.execution = options.execution
# #maget_options.application = options.application
# maget_options.maget = options.mbm.maget
# del maget_options.mbm
#
# s.defer(maget([xfm.resampled for xfm in lsq6_result],
# options=maget_options,
# prefix="%s_MAGeT" % prefix,
# output_dir=os.path.join(output_dir, prefix + "_processed")))
# should also move outside `mbm` function ...
#if options.mbm.thickness.run_thickness:
# if not options.mbm.segmentation.run_maget:
# warnings.warn("MAGeT files (atlases, protocols) are needed to run thickness calculation.")
# # run MAGeT to segment the nlin average:
# import copy
# maget_options = copy.deepcopy(options) #Namespace(maget=options)
# maget_options.maget = options.mbm.maget
# del maget_options.mbm
# segmented_avg = s.defer(maget(imgs=[lsq12_nlin_result.avg_img],
# options=maget_options,
# output_dir=os.path.join(options.application.output_directory,
# prefix + "_processed"),
# prefix="%s_thickness_MAGeT" % prefix)).ix[0].img
# thickness = s.defer(cortical_thickness(xfms=pd.Series(inverted_xfms), atlas=segmented_avg,
# label_mapping=FileAtom(options.mbm.thickness.label_mapping),
# atlas_fwhm=0.56, thickness_fwhm=0.56)) # TODO magic fwhms
# # TODO write CSV -- should `cortical_thickness` do this/return a table?
# FIXME: this needs to go outside of the `mbm` function to avoid being run from within other pipelines (or
# those other pipelines need to turn off this option)
if options.mbm.common_space.do_common_space_registration:
warnings.warn("This feature is experimental ...")
if not options.mbm.common_space.common_space_model:
raise ValueError("No common space template provided!")
# TODO allow lsq6 registration as well ...
common_space_model = MincAtom(options.mbm.common_space.common_space_model,
pipeline_sub_dir=os.path.join(options.application.output_directory,
options.application.pipeline_name + "_processed"))
# TODO allow different lsq12/nlin config params than the ones used in MBM ...
# WEIRD ... see comment in lsq12_nlin code ...
nlin_conf = full_hierarchy.confs[-1] if isinstance(full_hierarchy, MultilevelMincANTSConf) else full_hierarchy
# also weird that we need to call get_linear_configuration_from_options here ... ?
lsq12_conf = get_linear_configuration_from_options(conf=options.mbm.lsq12,
transform_type=LinearTransType.lsq12,
file_resolution=resolution)
xfm_to_common = s.defer(lsq12_nlin(source=lsq12_nlin_result.avg_img, target=common_space_model,
lsq12_conf=lsq12_conf, nlin_conf=nlin_conf,
resample_source=True))
model_common = s.defer(mincresample_new(img=lsq12_nlin_result.avg_img,
xfm=xfm_to_common.xfm, like=common_space_model,
postfix="_common"))
overall_xfms_common = [s.defer(concat_xfmhandlers([rigid_xfm, nlin_xfm, xfm_to_common]))
for rigid_xfm, nlin_xfm in zip(lsq6_result, lsq12_nlin_result.output)]
xfms_common = [s.defer(concat_xfmhandlers([nlin_xfm, xfm_to_common]))
for nlin_xfm in lsq12_nlin_result.output]
output_xfms = output_xfms.assign(xfm_common=xfms_common, overall_xfm_common=overall_xfms_common)
log_nlin_det_common, log_full_det_common = [dets.map(lambda d:
s.defer(mincresample_new(
img=d,
xfm=xfm_to_common.xfm,
like=common_space_model,
postfix="_common",
extra_flags=("-keep_real_range",),
interpolation=Interpolation.nearest_neighbour)))
for dets in (determinants.log_nlin_det, determinants.log_full_det)]
determinants = determinants.assign(log_nlin_det_common=log_nlin_det_common,
log_full_det_common=log_full_det_common)
output = Namespace(avg_img=lsq12_nlin_result.avg_img, xfms=output_xfms, determinants=determinants)
if options.mbm.common_space.do_common_space_registration:
output.model_common = model_common
return Result(stages=s, output=output)
def mbm(imgs: List[MincAtom],
options: MBMConf,
prefix: str,
output_dir: str = "",
with_maget: bool = True):
# TODO could also allow pluggable pipeline parts e.g. LSQ6 could be substituted out for the modified LSQ6
# for the kidney tips, etc...
# TODO this is tedious and annoyingly similar to the registration chain ...
lsq6_dir = os.path.join(output_dir, prefix + "_lsq6")
lsq12_dir = os.path.join(output_dir, prefix + "_lsq12")
nlin_dir = os.path.join(output_dir, prefix + "_nlin")
s = Stages()
if len(imgs) == 0:
raise ValueError("Please, some files!")
# FIXME: why do we have to call registration_targets *outside* of lsq6_nuc_inorm? is it just because of the extra
# options required? Also, shouldn't options.registration be a required input (as it contains `input_space`) ...?
targets = s.defer(
registration_targets(lsq6_conf=options.mbm.lsq6,
app_conf=options.application,
reg_conf=options.registration,
first_input_file=imgs[0].path))
# TODO this is quite tedious and duplicates stuff in the registration chain ...
resolution = (options.registration.resolution or get_resolution_from_file(
targets.registration_standard.path))
options.registration = options.registration.replace(resolution=resolution)
# FIXME: this needs to go outside of the `mbm` function to avoid being run from within other pipelines (or
# those other pipelines need to turn off this option)
if with_maget:
if options.mbm.segmentation.run_maget or options.mbm.maget.maget.mask:
# temporary fix...?
if options.mbm.maget.maget.mask and not options.mbm.segmentation.run_maget:
# which means that --no-run-maget was specified
if options.mbm.maget.maget.atlas_lib == None:
# clearly you do not want to run MAGeT at any point in this pipeline
err_msg_maget = "\nYou specified not to run MAGeT using the " \
"--no-run-maget flag. However, the code also " \
"wants to use MAGeT to generate masks for your " \
"input files after the 6 parameter alignment (lsq6). " \
"Because you did not specify a MAGeT atlas library " \
"this can not be done. \nTo run the pipeline without " \
"using MAGeT to mask your input files, please also " \
"specify: \n--maget-no-mask\n"
raise ValueError(err_msg_maget)
import copy
maget_options = copy.deepcopy(options) #Namespace(maget=options)
#maget_options
#maget_options.maget = maget_options.mbm
#maget_options.execution = options.execution
#maget_options.application = options.application
#maget_options.application.output_directory = os.path.join(options.application.output_directory, "segmentation")
maget_options.maget = options.mbm.maget
fixup_maget_options(maget_options=maget_options.maget,
nlin_options=maget_options.mbm.nlin,
lsq12_options=maget_options.mbm.lsq12)
del maget_options.mbm
#def with_new_output_dir(img : MincAtom):
#img = copy.copy(img)
#img.pipeline_sub_dir = img.pipeline_sub_dir + img.output_dir
#img.
#return img.newname_with_suffix(suffix="", subdir="segmentation")
# FIXME it probably makes most sense if the lsq6 module itself (even within lsq6_nuc_inorm) handles the run_lsq6
# setting (via use of the identity transform) since then this doesn't have to be implemented for every pipeline
if options.mbm.lsq6.run_lsq6:
lsq6_result = s.defer(
lsq6_nuc_inorm(imgs=imgs,
resolution=resolution,
registration_targets=targets,
lsq6_dir=lsq6_dir,
lsq6_options=options.mbm.lsq6))
else:
# FIXME the code shouldn't branch here based on run_lsq6 (which should probably
# be part of the lsq6 options rather than the MBM ones; see comments on #287.
# TODO don't actually do this resampling if not required (i.e., if the imgs already have the same grids)??
# however, for now need to add the masks:
identity_xfm = s.defer(
param2xfm(
out_xfm=FileAtom(name=os.path.join(lsq6_dir, 'tmp', "id.xfm"),
pipeline_sub_dir=lsq6_dir,
output_sub_dir='tmp')))
lsq6_result = [
XfmHandler(source=img,
target=img,
xfm=identity_xfm,
resampled=s.defer(
mincresample_new(img=img,
like=targets.registration_standard,
xfm=identity_xfm))) for img in imgs
]
# what about running nuc/inorm without a linear registration step??
if with_maget and options.mbm.maget.maget.mask:
masking_imgs = copy.deepcopy([xfm.resampled for xfm in lsq6_result])
masked_img = (s.defer(
maget_mask(imgs=masking_imgs,
resolution=resolution,
maget_options=maget_options.maget,
pipeline_sub_dir=os.path.join(
options.application.output_directory,
"%s_atlases" % prefix))))
masked_img.index = masked_img.apply(lambda x: x.path)
# replace any masks of the resampled images with the newly created masks:
for xfm in lsq6_result:
xfm.resampled = masked_img.loc[xfm.resampled.path]
elif with_maget:
warnings.warn(
"Not masking your images from atlas masks after LSQ6 alignment ... probably not what you want "
"(this can have negative effects on your registration and statistics)"
)
#full_hierarchy = get_nonlinear_configuration_from_options(nlin_protocol=options.mbm.nlin.nlin_protocol,
# flag_nlin_protocol=next(iter(options.mbm.nlin.flags_.nlin_protocol)),
# reg_method=options.mbm.nlin.reg_method,
# file_resolution=resolution)
#I = TypeVar("I")
#X = TypeVar("X")
#def wrap_minc(nlin_module: NLIN[I, X]) -> type[NLIN[MincAtom, XfmAtom]]:
# class N(NLIN[MincAtom, XfmAtom]): pass
# TODO now the user has to call get_nonlinear_component followed by parse_<...>; previously various things
# like lsq12_nlin_pairwise all branched on the reg_method so one didn't have to call get_nonlinear_component;
# they could still do this if it can be done safety (i.e., not breaking assumptions of various nonlinear units)
nlin_module = get_nonlinear_component(
reg_method=options.mbm.nlin.reg_method)
nlin_build_model_component = get_model_building_procedure(
options.mbm.nlin.reg_strategy,
# was: model_building.reg_strategy
reg_module=nlin_module)
# does this belong here?
# def model_building_with_initial_target_generation(prelim_model_building_component,
# final_model_building_component):
# class C(final_model_building_component):
# @staticmethod
# def build_model(imgs,
# conf : BuildModelConf,
# nlin_dir,
# nlin_prefix,
# initial_target,
# output_name_wo_ext = None): pass
#
# return C
#if options.mbm.model_building.prelim_reg_strategy is not None:
# prelim_nlin_build_model_component = get_model_building_procedure(options.mbm.model_building.prelim_reg_strategy,
# reg_module=nlin_module)
# nlin_build_model_component = model_building_with_initial_target_generation(
# final_model_building_component=nlin_build_model_component,
# prelim_model_building_component=prelim_nlin_build_model_component)
# TODO don't use name 'x_module' for something that's technically not a module ... perhaps unit/component?
# TODO tedious: why can't parse_build_model_protocol handle the null protocol case? is this something we want?
nlin_conf = (nlin_build_model_component.parse_build_model_protocol(
options.mbm.nlin.nlin_protocol, resolution=resolution)
if options.mbm.nlin.nlin_protocol is not None else
nlin_build_model_component.get_default_build_model_conf(
resolution=resolution))
lsq12_nlin_result = s.defer(
lsq12_nlin_build_model(
nlin_module=nlin_build_model_component,
imgs=[xfm.resampled for xfm in lsq6_result],
lsq12_dir=lsq12_dir,
nlin_dir=nlin_dir,
nlin_prefix=prefix,
use_robust_averaging=options.mbm.nlin.use_robust_averaging,
resolution=resolution,
lsq12_conf=options.mbm.lsq12,
nlin_conf=nlin_conf)) #options.mbm.nlin
inverted_xfms = [
s.defer(invert_xfmhandler(xfm)) for xfm in lsq12_nlin_result.output
]
if options.mbm.stats.stats_kernels:
determinants = s.defer(
determinants_at_fwhms(xfms=inverted_xfms,
inv_xfms=lsq12_nlin_result.output,
blur_fwhms=options.mbm.stats.stats_kernels))
else:
determinants = None
overall_xfms = [
s.defer(concat_xfmhandlers([rigid_xfm, lsq12_nlin_xfm])) for rigid_xfm,
lsq12_nlin_xfm in zip(lsq6_result, lsq12_nlin_result.output)
]
output_xfms = (
pd.DataFrame({
"rigid_xfm":
lsq6_result, # maybe don't return this if LSQ6 not run??
"lsq12_nlin_xfm": lsq12_nlin_result.output,
"overall_xfm": overall_xfms
}))
# we could `merge` the determinants with this table, but preserving information would cause lots of duplication
# of the transforms (or storing determinants in more columns, but iterating over dynamically known columns
# seems a bit odd ...)
# TODO transpose these fields?})
#avg_img=lsq12_nlin_result.avg_img, # inconsistent w/ WithAvgImgs[...]-style outputs
# "determinants" : determinants })
#output.avg_img = lsq12_nlin_result.avg_img
#output.determinants = determinants # TODO temporary - remove once incorporated properly into `output` proper
# TODO add more of lsq12_nlin_result?
# FIXME moved above rest of registration for debugging ... shouldn't use and destructively modify lsq6_result!!!
if with_maget and options.mbm.segmentation.run_maget:
maget_options = copy.deepcopy(maget_options)
maget_options.maget.maget.mask = maget_options.maget.maget.mask_only = False # already done above
# use the original masks here otherwise the masking step will be re-run due to the previous masking run's
# masks having been applied to the input images:
maget_result = s.defer(
maget(
[xfm.resampled for xfm in lsq6_result],
#[xfm.resampled for _ix, xfm in mbm_result.xfms.rigid_xfm.iteritems()],
options=maget_options,
prefix="%s_MAGeT" % prefix,
output_dir=os.path.join(output_dir, prefix + "_processed")))
# FIXME add pipeline dir to path and uncomment!
#maget.to_csv(path_or_buf="segmentations.csv", columns=['img', 'voted_labels'])
# TODO return some MAGeT stuff from MBM function ??
# if options.mbm.mbm.run_maget:
# import copy
# maget_options = copy.deepcopy(options) #Namespace(maget=options)
# #maget_options
# #maget_options.maget = maget_options.mbm
# #maget_options.execution = options.execution
# #maget_options.application = options.application
# maget_options.maget = options.mbm.maget
# del maget_options.mbm
#
# s.defer(maget([xfm.resampled for xfm in lsq6_result],
# options=maget_options,
# prefix="%s_MAGeT" % prefix,
# output_dir=os.path.join(output_dir, prefix + "_processed")))
# should also move outside `mbm` function ...
#if options.mbm.thickness.run_thickness:
# if not options.mbm.segmentation.run_maget:
# warnings.warn("MAGeT files (atlases, protocols) are needed to run thickness calculation.")
# # run MAGeT to segment the nlin average:
# import copy
# maget_options = copy.deepcopy(options) #Namespace(maget=options)
# maget_options.maget = options.mbm.maget
# del maget_options.mbm
# segmented_avg = s.defer(maget(imgs=[lsq12_nlin_result.avg_img],
# options=maget_options,
# output_dir=os.path.join(options.application.output_directory,
# prefix + "_processed"),
# prefix="%s_thickness_MAGeT" % prefix)).ix[0].img
# thickness = s.defer(cortical_thickness(xfms=pd.Series(inverted_xfms), atlas=segmented_avg,
# label_mapping=FileAtom(options.mbm.thickness.label_mapping),
# atlas_fwhm=0.56, thickness_fwhm=0.56)) # TODO magic fwhms
# # TODO write CSV -- should `cortical_thickness` do this/return a table?
output = Namespace(avg_img=lsq12_nlin_result.avg_img,
xfms=output_xfms,
determinants=determinants)
if with_maget and options.mbm.segmentation.run_maget:
output.maget_result = maget_result
nlin_maget = (
s.defer(
maget(
[lsq12_nlin_result.avg_img],
#[xfm.resampled for _ix, xfm in mbm_result.xfms.rigid_xfm.iteritems()],
options=maget_options,
prefix="%s_nlin_MAGeT" % prefix,
output_dir=os.path.join(
output_dir,
prefix + "_processed")))).iloc[0] #.voted_labels
#output.avg_img.mask = nlin_maget.mask # makes more sense, but might have weird effects elsewhere
output.avg_img.labels = nlin_maget.labels
return Result(stages=s, output=output)
def common_space(mbm_result, options):
s = Stages()
# TODO: the interface of this function (basically a destructive 'id' function) is horrific
# TODO: instead, copy the mbm_result here ??
if not options.mbm.common_space.common_space_model:
raise ValueError("No common space template provided!")
if not options.mbm.common_space.common_space_mask:
warnings.warn(
"No common space mask provided ... might be OK if your consensus average mask is OK"
)
# TODO allow lsq6 registration as well ...
common_space_model = MincAtom(
options.mbm.common_space.common_space_model,
# TODO fix the subdirectories!
mask=MincAtom(options.mbm.common_space.common_space_mask,
pipeline_sub_dir=os.path.join(
options.application.output_directory,
options.application.pipeline_name + "_processed"))
if options.mbm.common_space.common_space_mask else None,
pipeline_sub_dir=os.path.join(
options.application.output_directory,
options.application.pipeline_name + "_processed"))
# TODO allow different lsq12/nlin config params than the ones used in MBM ...
# full_hierarchy = get_nonlinear_configuration_from_options(nlin_protocol=options.mbm.nlin.nlin_protocol,
# reg_method=options.mbm.nlin.reg_method,
# file_resolution=options.registration.resolution)
# WEIRD ... see comment in lsq12_nlin code ...
# nlin_conf = full_hierarchy.confs[-1] if isinstance(full_hierarchy, MultilevelANTSConf) else full_hierarchy
# also weird that we need to call get_linear_configuration_from_options here ... ?
# nlin_build_model_component = model_building_with_initial_target_generation(
# final_model_building_component=nlin_build_model_component,
# prelim_model_building_component=prelim_nlin_build_model_component)
# TODO don't use name 'x_module' for something that's technically not a module ... perhaps unit/component?
nlin_component = get_nonlinear_component(
reg_method=options.mbm.nlin.reg_method)
lsq12_conf = get_linear_configuration_from_options(
conf=options.mbm.lsq12,
transform_type=LinearTransType.lsq12,
file_resolution=options.registration.resolution)
# N.B.: options.registration.resolution has been *updated* correctly by mbm( ). sigh ...
model_to_common = s.defer(
lsq12_nlin(
source=mbm_result.avg_img,
target=common_space_model,
lsq12_conf=lsq12_conf,
nlin_module=nlin_component,
resolution=options.registration.resolution,
nlin_options=options.mbm.nlin.nlin_protocol, # =nlin_conf,
resample_source=True))
model_common = s.defer(
mincresample_new(img=mbm_result.avg_img,
xfm=model_to_common.xfm,
like=common_space_model,
postfix="_common"))
overall_xfms_to_common = [
s.defer(concat_xfmhandlers([rigid_xfm, nlin_xfm, model_to_common]))
for rigid_xfm, nlin_xfm in zip(mbm_result.xfms.rigid_xfm,
mbm_result.xfms.lsq12_nlin_xfm)
]
overall_xfms_to_common_inv = [
s.defer(invert_xfmhandler(xfmhandler)) for xfmhandler in [
s.defer(concat_xfmhandlers([rigid_xfm, nlin_xfm, model_to_common]))
for rigid_xfm, nlin_xfm in zip(mbm_result.xfms.rigid_xfm,
mbm_result.xfms.lsq12_nlin_xfm)
]
]
xfms_to_common = [
s.defer(concat_xfmhandlers([nlin_xfm, model_to_common]))
for nlin_xfm in mbm_result.xfms.lsq12_nlin_xfm
]
mbm_result.xfms = mbm_result.xfms.assign(
xfm_to_common=xfms_to_common,
overall_xfm_to_common=overall_xfms_to_common)
if options.mbm.stats.calc_stats:
log_nlin_det_common, log_full_det_common = ([
dets.map(lambda d: s.defer(
mincresample_new(img=d,
xfm=model_to_common.xfm,
like=common_space_model,
postfix="_common")))
for dets in (mbm_result.determinants.log_nlin_det,
mbm_result.determinants.log_full_det)
])
overall_determinants = s.defer(
determinants_at_fwhms(xfms=overall_xfms_to_common_inv,
blur_fwhms=options.mbm.stats.stats_kernels))
mbm_result.determinants = \
mbm_result.determinants.assign(log_nlin_det_common=log_nlin_det_common,
log_full_det_common=log_full_det_common,
log_nlin_overall_det_common=overall_determinants.log_nlin_det,
log_full_overall_det_common=overall_determinants.log_full_det
)
mbm_result.model_common = model_common
return Result(stages=s, output=mbm_result)
def NLIN_pipeline(options):
output_dir = options.application.output_directory
pipeline_name = options.application.pipeline_name
# TODO this is tedious and annoyingly similar to the registration chain and MBM and LSQ6 ...
processed_dir = os.path.join(output_dir, pipeline_name + "_processed")
nlin_dir = os.path.join(output_dir, pipeline_name + "_nlin")
resolution = (
options.registration.
resolution # TODO does using the finest resolution here make sense?
or min(
[get_resolution_from_file(f) for f in options.application.files]))
imgs = get_imgs(options.application)
initial_target_mask = MincAtom(
options.nlin.target_mask) if options.nlin.target_mask else None
initial_target = MincAtom(options.nlin.target, mask=initial_target_mask)
nlin_module = get_nonlinear_component(reg_method=options.nlin.reg_method)
nlin_build_model_component = get_model_building_procedure(
options.nlin.reg_strategy, reg_module=nlin_module)
nlin_conf = (nlin_build_model_component.parse_build_model_protocol(
options.nlin.nlin_protocol, resolution=resolution)
if options.nlin.nlin_protocol is not None else
nlin_build_model_component.get_default_build_model_conf(
resolution=resolution))
s = Stages()
nlin_result = s.defer(
nlin_build_model_component.build_model(
imgs=imgs,
initial_target=initial_target,
conf=nlin_conf,
nlin_dir=nlin_dir,
use_robust_averaging=options.nlin.use_robust_averaging,
nlin_prefix=""))
inverted_xfms = [
s.defer(invert_xfmhandler(xfm)) for xfm in nlin_result.output
]
if options.stats.calc_stats:
determinants = s.defer(
determinants_at_fwhms(xfms=inverted_xfms,
inv_xfms=nlin_result.output,
blur_fwhms=options.stats.stats_kernels))
return Result(stages=s,
output=Namespace(nlin_xfms=nlin_result,
avg_img=nlin_result.avg_img,
determinants=determinants))
else:
# there's no consistency in what gets returned, yikes ...
return Result(stages=s,
output=Namespace(nlin_xfms=nlin_result,
avg_img=nlin_result.avg_img))
def determinants_at_fwhms(xfms : List[XfmHandler], # TODO change to pd.Series to get indexing (hence safer inv_xfm)?
blur_fwhms : str, # TODO: change back to List[float]; should unblurred dets be found automatically?
inv_xfms : Optional[List[XfmHandler]] = None) \
-> Result[pd.DataFrame]: # TODO how to write down a Pandas type here ?!
"""
The most common way to use this function is by providing
it with transformations that go from the final average
to an individual. I.e.:
*** = non linear deformations
--- = linear (affine) deformations
xfm = final-nlin ******------> individual_input
inv_xfm = final-nlin <******------ individual_input
Takes a transformation (xfm) containing
both lsq12 (scaling and shearing, the 6-parameter
rotations/translations should not be part of this) and
non-linear parts of a subject to a common/shared average
and returns the determinants of both the (forward) nonlinear
part of the xfm at the given fwhms as well as the determinants
of the full (forward) transformation. The inverse transform
may optionally be specified to avoid its recomputation (e.g.,
when passing an inverted xfm to determinants_at_fwhms,
specify the original here).
"""
s = Stages()
inv_xfms = [s.defer(invert_xfmhandler(xfm))
for xfm in xfms] if inv_xfms is None else inv_xfms
fwhms = [float(x) for x in blur_fwhms.split(',')]
df = pd.DataFrame([
{
"xfm": xfm,
"inv_xfm": inv_xfm,
"fwhm": fwhm,
"nlin_det": nlin_det,
"log_nlin_det": nlin_log_det,
"full_det": full_det,
"log_full_det": full_log_det
} for fwhm in fwhms +
[0] # was: None, but this turns to NaN in Pandas ...
for xfm, inv_xfm in zip(xfms, inv_xfms) for full_det_and_log_det in [
s.defer(
det_and_log_det(displacement_grid=s.defer(
minc_displacement(xfm)),
fwhm=fwhm,
annotation="_abs"))
] for full_det, full_log_det in [(full_det_and_log_det.det,
full_det_and_log_det.log_det)]
for nlin_det_and_log_det in [
s.defer(
det_and_log_det(displacement_grid=s.defer(
nlin_displacement(xfm, inv_xfm=inv_xfm)),
fwhm=fwhm,
annotation="_rel"))
] for nlin_det, nlin_log_det in [(nlin_det_and_log_det.det,
nlin_det_and_log_det.log_det)]
])
# TODO this is terrible, and should probably be done with joins, but one gets the idea ...
# TODO remove 'inv_xfm' column?
# TODO the return of this function is 'everything', not really just 'determinants_at_fwhms' ...
return Result(stages=s, output=df)
def nonlinear_midpoint_xfm(nlin_algorithm : Type[NLIN],
img_A: MincAtom,
img_B: MincAtom,
conf, #: nlin_algorithm.Conf,
out_name_wo_ext: str,
out_dir: str,
mincaverage = mincbigaverage) -> Result[MincAtom]:
"""
:param img_A:
:param img_B:
:return: the midway point between img_A and img_B --> img_AB
"""
s = Stages()
# # invariant: all the `xfms_A` have the same `resampled` field, and same for `xfms_B`:
# start with an antRegistration call between the two files
#xfm_handlers_antsReg = s.defer(antsRegistration(source=img_A,
# target=img_B,
# subdir='tmp'))
A_to_B = s.defer(nlin_algorithm.register(source=img_A,
# all the xfms have the same `resampled` field
target=img_B,
conf=conf,
transform_name_wo_ext="%s_A_to_B" % out_name_wo_ext,
####resample_name_wo_ext="%s_A_to_B" % out_name_wo_ext,
resample_subdir=out_dir)) # TODO 'tmp' ?
# N.B.: this uses `invert_xfmhandler` magic in either extracting the inverse from A_to_B
# if it's present or inverting the transform if not ... slightly uncomfortable with this for some reason.
B_to_A = s.defer(invert_xfmhandler(A_to_B))
algorithms = nlin_algorithm.Algorithms
# generate halfway transformations
transform_A_to_B_halfway = s.defer(algorithms.scale_transform(xfm=A_to_B,
scale=0.5, newname_wo_ext="%s_AtoB_half" % out_name_wo_ext))
transform_B_to_A_halfway = s.defer(algorithms.scale_transform(xfm=B_to_A,
scale=0.5, newname_wo_ext="%s_BtoA_half" % out_name_wo_ext))
# resample A and B halfway:
A_halfway_to_B = s.defer(algorithms.resample(img=img_A,
xfm=transform_A_to_B_halfway,
like=img_B,
subdir='tmp',
new_name_wo_ext=out_name_wo_ext + "_AtoB"))
B_halfway_to_A = s.defer(algorithms.resample(img=img_B,
xfm=transform_B_to_A_halfway,
like=img_A,
subdir='tmp',
new_name_wo_ext=out_name_wo_ext + "_BtoA"))
# the output file (avg of both files resampled to the midway point)
avg_mid_point = s.defer(algorithms.average(imgs=[A_halfway_to_B, B_halfway_to_A],
output_dir=out_dir,
name_wo_ext=out_name_wo_ext))
return Result(stages=s, output=(XfmHandler(source=img_A, target=None,
xfm=transform_A_to_B_halfway, resampled=A_halfway_to_B),
XfmHandler(source=img_B, target=None,
xfm=transform_B_to_A_halfway, resampled=B_halfway_to_A),
avg_mid_point))
