def mbm_pipeline(options : MBMConf):
s = Stages()
imgs = [MincAtom(name, pipeline_sub_dir=os.path.join(options.application.output_directory,
options.application.pipeline_name + "_processed"))
for name in options.application.files]
check_MINC_input_files([img.path for img in imgs])
prefix = options.application.pipeline_name
mbm_result = s.defer(mbm(imgs=imgs, options=options,
prefix=prefix,
output_dir=options.application.output_directory))
# create useful CSVs (note the files listed therein won't yet exist ...)
for filename, dataframe in (("transforms.csv", mbm_result.xfms),
("determinants.csv", mbm_result.determinants)):
with open(filename, 'w') as f:
f.write(dataframe.applymap(maybe_deref_path).to_csv(index=False))
# TODO moved here from inside `mbm` for now ... does this make most sense?
if options.mbm.segmentation.run_maget:
import copy
maget_options = copy.deepcopy(options) #Namespace(maget=options)
#maget_options
#maget_options.maget = maget_options.mbm
#maget_options.execution = options.execution
#maget_options.application = options.application
maget_options.application.output_directory = os.path.join(options.application.output_directory, "segmentation")
maget_options.maget = options.mbm.maget
fixup_maget_options(maget_options=maget_options.maget,
nlin_options=maget_options.mbm.nlin,
lsq12_options=maget_options.mbm.lsq12)
del maget_options.mbm
s.defer(maget([xfm.resampled for _ix, xfm in mbm_result.xfms.rigid_xfm.iteritems()],
options=maget_options,
prefix="%s_MAGeT" % prefix,
output_dir=os.path.join(options.application.output_directory, prefix + "_processed")))
return Result(stages=s, output=mbm_result)
def mbm(imgs: List[MincAtom],
options: MBMConf,
prefix: str,
output_dir: str = "",
with_maget: bool = True):
# TODO could also allow pluggable pipeline parts e.g. LSQ6 could be substituted out for the modified LSQ6
# for the kidney tips, etc...
# TODO this is tedious and annoyingly similar to the registration chain ...
lsq6_dir = os.path.join(output_dir, prefix + "_lsq6")
lsq12_dir = os.path.join(output_dir, prefix + "_lsq12")
nlin_dir = os.path.join(output_dir, prefix + "_nlin")
s = Stages()
if len(imgs) == 0:
raise ValueError("Please, some files!")
# FIXME: why do we have to call registration_targets *outside* of lsq6_nuc_inorm? is it just because of the extra
# options required? Also, shouldn't options.registration be a required input (as it contains `input_space`) ...?
targets = s.defer(
registration_targets(lsq6_conf=options.mbm.lsq6,
app_conf=options.application,
reg_conf=options.registration,
first_input_file=imgs[0].path))
# TODO this is quite tedious and duplicates stuff in the registration chain ...
resolution = (options.registration.resolution or get_resolution_from_file(
targets.registration_standard.path))
options.registration = options.registration.replace(resolution=resolution)
# FIXME: this needs to go outside of the `mbm` function to avoid being run from within other pipelines (or
# those other pipelines need to turn off this option)
if with_maget:
if options.mbm.segmentation.run_maget or options.mbm.maget.maget.mask:
# temporary fix...?
if options.mbm.maget.maget.mask and not options.mbm.segmentation.run_maget:
# which means that --no-run-maget was specified
if options.mbm.maget.maget.atlas_lib == None:
# clearly you do not want to run MAGeT at any point in this pipeline
err_msg_maget = "\nYou specified not to run MAGeT using the " \
"--no-run-maget flag. However, the code also " \
"wants to use MAGeT to generate masks for your " \
"input files after the 6 parameter alignment (lsq6). " \
"Because you did not specify a MAGeT atlas library " \
"this can not be done. \nTo run the pipeline without " \
"using MAGeT to mask your input files, please also " \
"specify: \n--maget-no-mask\n"
raise ValueError(err_msg_maget)
import copy
maget_options = copy.deepcopy(options) #Namespace(maget=options)
#maget_options
#maget_options.maget = maget_options.mbm
#maget_options.execution = options.execution
#maget_options.application = options.application
#maget_options.application.output_directory = os.path.join(options.application.output_directory, "segmentation")
maget_options.maget = options.mbm.maget
fixup_maget_options(maget_options=maget_options.maget,
nlin_options=maget_options.mbm.nlin,
lsq12_options=maget_options.mbm.lsq12)
del maget_options.mbm
#def with_new_output_dir(img : MincAtom):
#img = copy.copy(img)
#img.pipeline_sub_dir = img.pipeline_sub_dir + img.output_dir
#img.
#return img.newname_with_suffix(suffix="", subdir="segmentation")
# FIXME it probably makes most sense if the lsq6 module itself (even within lsq6_nuc_inorm) handles the run_lsq6
# setting (via use of the identity transform) since then this doesn't have to be implemented for every pipeline
if options.mbm.lsq6.run_lsq6:
lsq6_result = s.defer(
lsq6_nuc_inorm(imgs=imgs,
resolution=resolution,
registration_targets=targets,
lsq6_dir=lsq6_dir,
lsq6_options=options.mbm.lsq6))
else:
# FIXME the code shouldn't branch here based on run_lsq6 (which should probably
# be part of the lsq6 options rather than the MBM ones; see comments on #287.
# TODO don't actually do this resampling if not required (i.e., if the imgs already have the same grids)??
# however, for now need to add the masks:
identity_xfm = s.defer(
param2xfm(
out_xfm=FileAtom(name=os.path.join(lsq6_dir, 'tmp', "id.xfm"),
pipeline_sub_dir=lsq6_dir,
output_sub_dir='tmp')))
lsq6_result = [
XfmHandler(source=img,
target=img,
xfm=identity_xfm,
resampled=s.defer(
mincresample_new(img=img,
like=targets.registration_standard,
xfm=identity_xfm))) for img in imgs
]
# what about running nuc/inorm without a linear registration step??
if with_maget and options.mbm.maget.maget.mask:
masking_imgs = copy.deepcopy([xfm.resampled for xfm in lsq6_result])
masked_img = (s.defer(
maget_mask(imgs=masking_imgs,
resolution=resolution,
maget_options=maget_options.maget,
pipeline_sub_dir=os.path.join(
options.application.output_directory,
"%s_atlases" % prefix))))
masked_img.index = masked_img.apply(lambda x: x.path)
# replace any masks of the resampled images with the newly created masks:
for xfm in lsq6_result:
xfm.resampled = masked_img.loc[xfm.resampled.path]
elif with_maget:
warnings.warn(
"Not masking your images from atlas masks after LSQ6 alignment ... probably not what you want "
"(this can have negative effects on your registration and statistics)"
)
#full_hierarchy = get_nonlinear_configuration_from_options(nlin_protocol=options.mbm.nlin.nlin_protocol,
# flag_nlin_protocol=next(iter(options.mbm.nlin.flags_.nlin_protocol)),
# reg_method=options.mbm.nlin.reg_method,
# file_resolution=resolution)
#I = TypeVar("I")
#X = TypeVar("X")
#def wrap_minc(nlin_module: NLIN[I, X]) -> type[NLIN[MincAtom, XfmAtom]]:
# class N(NLIN[MincAtom, XfmAtom]): pass
# TODO now the user has to call get_nonlinear_component followed by parse_<...>; previously various things
# like lsq12_nlin_pairwise all branched on the reg_method so one didn't have to call get_nonlinear_component;
# they could still do this if it can be done safety (i.e., not breaking assumptions of various nonlinear units)
nlin_module = get_nonlinear_component(
reg_method=options.mbm.nlin.reg_method)
nlin_build_model_component = get_model_building_procedure(
options.mbm.nlin.reg_strategy,
# was: model_building.reg_strategy
reg_module=nlin_module)
# does this belong here?
# def model_building_with_initial_target_generation(prelim_model_building_component,
# final_model_building_component):
# class C(final_model_building_component):
# @staticmethod
# def build_model(imgs,
# conf : BuildModelConf,
# nlin_dir,
# nlin_prefix,
# initial_target,
# output_name_wo_ext = None): pass
#
# return C
#if options.mbm.model_building.prelim_reg_strategy is not None:
# prelim_nlin_build_model_component = get_model_building_procedure(options.mbm.model_building.prelim_reg_strategy,
# reg_module=nlin_module)
# nlin_build_model_component = model_building_with_initial_target_generation(
# final_model_building_component=nlin_build_model_component,
# prelim_model_building_component=prelim_nlin_build_model_component)
# TODO don't use name 'x_module' for something that's technically not a module ... perhaps unit/component?
# TODO tedious: why can't parse_build_model_protocol handle the null protocol case? is this something we want?
nlin_conf = (nlin_build_model_component.parse_build_model_protocol(
options.mbm.nlin.nlin_protocol, resolution=resolution)
if options.mbm.nlin.nlin_protocol is not None else
nlin_build_model_component.get_default_build_model_conf(
resolution=resolution))
lsq12_nlin_result = s.defer(
lsq12_nlin_build_model(
nlin_module=nlin_build_model_component,
imgs=[xfm.resampled for xfm in lsq6_result],
lsq12_dir=lsq12_dir,
nlin_dir=nlin_dir,
nlin_prefix=prefix,
use_robust_averaging=options.mbm.nlin.use_robust_averaging,
resolution=resolution,
lsq12_conf=options.mbm.lsq12,
nlin_conf=nlin_conf)) #options.mbm.nlin
inverted_xfms = [
s.defer(invert_xfmhandler(xfm)) for xfm in lsq12_nlin_result.output
]
if options.mbm.stats.stats_kernels:
determinants = s.defer(
determinants_at_fwhms(xfms=inverted_xfms,
inv_xfms=lsq12_nlin_result.output,
blur_fwhms=options.mbm.stats.stats_kernels))
else:
determinants = None
overall_xfms = [
s.defer(concat_xfmhandlers([rigid_xfm, lsq12_nlin_xfm])) for rigid_xfm,
lsq12_nlin_xfm in zip(lsq6_result, lsq12_nlin_result.output)
]
output_xfms = (
pd.DataFrame({
"rigid_xfm":
lsq6_result, # maybe don't return this if LSQ6 not run??
"lsq12_nlin_xfm": lsq12_nlin_result.output,
"overall_xfm": overall_xfms
}))
# we could `merge` the determinants with this table, but preserving information would cause lots of duplication
# of the transforms (or storing determinants in more columns, but iterating over dynamically known columns
# seems a bit odd ...)
# TODO transpose these fields?})
#avg_img=lsq12_nlin_result.avg_img, # inconsistent w/ WithAvgImgs[...]-style outputs
# "determinants" : determinants })
#output.avg_img = lsq12_nlin_result.avg_img
#output.determinants = determinants # TODO temporary - remove once incorporated properly into `output` proper
# TODO add more of lsq12_nlin_result?
# FIXME moved above rest of registration for debugging ... shouldn't use and destructively modify lsq6_result!!!
if with_maget and options.mbm.segmentation.run_maget:
maget_options = copy.deepcopy(maget_options)
maget_options.maget.maget.mask = maget_options.maget.maget.mask_only = False # already done above
# use the original masks here otherwise the masking step will be re-run due to the previous masking run's
# masks having been applied to the input images:
maget_result = s.defer(
maget(
[xfm.resampled for xfm in lsq6_result],
#[xfm.resampled for _ix, xfm in mbm_result.xfms.rigid_xfm.iteritems()],
options=maget_options,
prefix="%s_MAGeT" % prefix,
output_dir=os.path.join(output_dir, prefix + "_processed")))
# FIXME add pipeline dir to path and uncomment!
#maget.to_csv(path_or_buf="segmentations.csv", columns=['img', 'voted_labels'])
# TODO return some MAGeT stuff from MBM function ??
# if options.mbm.mbm.run_maget:
# import copy
# maget_options = copy.deepcopy(options) #Namespace(maget=options)
# #maget_options
# #maget_options.maget = maget_options.mbm
# #maget_options.execution = options.execution
# #maget_options.application = options.application
# maget_options.maget = options.mbm.maget
# del maget_options.mbm
#
# s.defer(maget([xfm.resampled for xfm in lsq6_result],
# options=maget_options,
# prefix="%s_MAGeT" % prefix,
# output_dir=os.path.join(output_dir, prefix + "_processed")))
# should also move outside `mbm` function ...
#if options.mbm.thickness.run_thickness:
# if not options.mbm.segmentation.run_maget:
# warnings.warn("MAGeT files (atlases, protocols) are needed to run thickness calculation.")
# # run MAGeT to segment the nlin average:
# import copy
# maget_options = copy.deepcopy(options) #Namespace(maget=options)
# maget_options.maget = options.mbm.maget
# del maget_options.mbm
# segmented_avg = s.defer(maget(imgs=[lsq12_nlin_result.avg_img],
# options=maget_options,
# output_dir=os.path.join(options.application.output_directory,
# prefix + "_processed"),
# prefix="%s_thickness_MAGeT" % prefix)).ix[0].img
# thickness = s.defer(cortical_thickness(xfms=pd.Series(inverted_xfms), atlas=segmented_avg,
# label_mapping=FileAtom(options.mbm.thickness.label_mapping),
# atlas_fwhm=0.56, thickness_fwhm=0.56)) # TODO magic fwhms
# # TODO write CSV -- should `cortical_thickness` do this/return a table?
output = Namespace(avg_img=lsq12_nlin_result.avg_img,
xfms=output_xfms,
determinants=determinants)
if with_maget and options.mbm.segmentation.run_maget:
output.maget_result = maget_result
nlin_maget = (
s.defer(
maget(
[lsq12_nlin_result.avg_img],
#[xfm.resampled for _ix, xfm in mbm_result.xfms.rigid_xfm.iteritems()],
options=maget_options,
prefix="%s_nlin_MAGeT" % prefix,
output_dir=os.path.join(
output_dir,
prefix + "_processed")))).iloc[0] #.voted_labels
#output.avg_img.mask = nlin_maget.mask # makes more sense, but might have weird effects elsewhere
output.avg_img.labels = nlin_maget.labels
return Result(stages=s, output=output)
def two_level(grouped_files_df, options : TwoLevelConf):
"""
grouped_files_df - must contain 'group':<any comparable, sortable type> and 'file':MincAtom columns
""" # TODO weird naming since the grouped_files_df isn't a GroupBy object? just files_df?
s = Stages()
if grouped_files_df.isnull().values.any():
raise ValueError("NaN values in input dataframe; can't go")
if options.mbm.lsq6.target_type == TargetType.bootstrap:
# won't work since the second level part tries to get the resolution of *its* "first input file", which
# hasn't been created. We could instead pass in a resolution to the `mbm` function,
# but instead disable for now:
raise ValueError("Bootstrap model building currently doesn't work with this pipeline; "
"just specify an initial target instead")
elif options.mbm.lsq6.target_type == TargetType.pride_of_models:
pride_of_models_mapping = get_pride_of_models_mapping(pride_csv=options.mbm.lsq6.target_file,
output_dir=options.application.output_directory,
pipeline_name=options.application.pipeline_name)
# FIXME this is the same as in the 'tamarack' except for names of arguments/enclosing variables
def group_options(options, group):
options = copy.deepcopy(options)
if options.mbm.lsq6.target_type == TargetType.pride_of_models:
targets = get_closest_model_from_pride_of_models(pride_of_models_dict=pride_of_models_mapping,
time_point=group)
options.mbm.lsq6 = options.mbm.lsq6.replace(target_type=TargetType.initial_model,
target_file=targets.registration_standard.path)
else:
# this will ensure that all groups have the same resolution -- is it necessary?
targets = registration_targets(lsq6_conf=options.mbm.lsq6,
app_conf=options.application,
first_input_file=grouped_files_df.file.iloc[0])
resolution = (options.registration.resolution
or get_resolution_from_file(targets.registration_standard.path))
options.registration = options.registration.replace(resolution=resolution)
return options
first_level_results = (
grouped_files_df
.groupby('group', as_index=False, sort=False) # the usual annoying pattern to do a aggregate with access
.aggregate({ 'file' : lambda files: list(files) }) # to the groupby object's keys ... TODO: fix
.rename(columns={ 'file' : "files" })
.assign(build_model=lambda df:
df.apply(axis=1,
func=lambda row:
s.defer(mbm(imgs=row.files,
options=group_options(options, row.group),
prefix="%s" % row.group,
output_dir=os.path.join(
options.application.output_directory,
options.application.pipeline_name + "_first_level",
"%s_processed" % row.group)))))
)
# TODO replace .assign(...apply(...)...) with just an apply, producing a series right away?
# FIXME right now the same options set is being used for both levels -- use options.first/second_level
second_level_options = copy.deepcopy(options)
second_level_options.mbm.lsq6 = second_level_options.mbm.lsq6.replace(run_lsq6=False)
# FIXME this is probably a hack -- instead add a --second-level-init-model option to specify which timepoint should be used
# as the initial model in the second level ??? (at this point it doesn't matter due to lack of lsq6 ...)
if second_level_options.mbm.lsq6.target_type == TargetType.pride_of_models:
second_level_options.mbm.lsq6 = second_level_options.mbm.lsq6.replace(
target_type=TargetType.target, # target doesn't really matter as no lsq6 here, just used for resolution...
target_file=list(pride_of_models_mapping.values())[0].registration_standard.path)
# NOTE: running lsq6_nuc_inorm here doesn't work in general (but possibly with rotational minctracc)
# since the native-space initial model is used, but our images are
# already in standard space (as we resampled there after the 1st-level lsq6).
# On the other hand, we might want to run it here (although of course NOT nuc/inorm!) in the future,
# for instance given a 'pride' of models (one for each group).
second_level_results = s.defer(mbm(imgs=first_level_results.build_model.map(lambda m: m.avg_img),
options=second_level_options,
prefix=os.path.join(options.application.output_directory,
options.application.pipeline_name + "_second_level")))
# FIXME sadly, `mbm` doesn't return a pd.Series of xfms, so we don't have convenient indexing ...
overall_xfms = [s.defer(concat_xfmhandlers([xfm_1, xfm_2]))
for xfms_1, xfm_2 in zip([r.xfms.lsq12_nlin_xfm for r in first_level_results.build_model],
second_level_results.xfms.overall_xfm)
for xfm_1 in xfms_1]
resample = np.vectorize(mincresample_new, excluded={"extra_flags"})
defer = np.vectorize(s.defer)
# TODO using the avg_img here is a bit clunky -- maybe better to propagate group indices ...
# only necessary since `mbm` doesn't return DataFrames but namespaces ...
first_level_determinants = pd.concat(list(first_level_results.build_model.apply(
lambda x: x.determinants.assign(first_level_avg=x.avg_img))),
ignore_index=True)
resampled_determinants = (pd.merge(
left=first_level_determinants,
right=pd.DataFrame({'group_xfm' : second_level_results.xfms.overall_xfm})
.assign(source=lambda df: df.group_xfm.apply(lambda r: r.source)),
left_on="first_level_avg",
right_on="source")
.assign(resampled_log_full_det=lambda df: defer(resample(img=df.log_full_det,
xfm=df.group_xfm.apply(lambda x: x.xfm),
like=second_level_results.avg_img)),
resampled_log_nlin_det=lambda df: defer(resample(img=df.log_nlin_det,
xfm=df.group_xfm.apply(lambda x: x.xfm),
like=second_level_results.avg_img))))
# TODO only resamples the log determinants, but still a bit ugly ... abstract somehow?
# TODO shouldn't be called resampled_determinants since this is basically the whole (first_level) thing ...
inverted_overall_xfms = [s.defer(invert_xfmhandler(xfm)) for xfm in overall_xfms]
overall_determinants = (s.defer(determinants_at_fwhms(
xfms=inverted_overall_xfms,
inv_xfms=overall_xfms,
blur_fwhms=options.mbm.stats.stats_kernels))
.assign(overall_log_full_det=lambda df: df.log_full_det,
overall_log_nlin_det=lambda df: df.log_nlin_det)
.drop(['log_full_det', 'log_nlin_det'], axis=1))
# TODO return some MAGeT stuff from two_level function ??
# FIXME running MAGeT from within the `two_level` function has the same problem as running it from within `mbm`:
# it will now run when this pipeline is called from within another one (e.g., n-level), which will be
# redundant, create filename clashes, etc. -- this should be moved to `two_level_pipeline`.
if options.mbm.segmentation.run_maget:
maget_options = copy.deepcopy(options)
maget_options.maget = options.mbm.maget
fixup_maget_options(maget_options=maget_options.maget,
lsq12_options=maget_options.mbm.lsq12,
nlin_options=maget_options.mbm.nlin)
del maget_options.mbm
# again using a weird combination of vectorized and loop constructs ...
s.defer(maget([xfm.resampled for _ix, m in first_level_results.iterrows()
for xfm in m.build_model.xfms.rigid_xfm],
options=maget_options,
prefix="%s_MAGeT" % options.application.pipeline_name,
output_dir=os.path.join(options.application.output_directory,
options.application.pipeline_name + "_processed")))
# TODO resampling to database model ...
# TODO there should be one table containing all determinants (first level, overall, resampled first level) for each file
# and another containing some groupwise information (averages and transforms to the common average)
return Result(stages=s, output=Namespace(first_level_results=first_level_results,
resampled_determinants=resampled_determinants,
overall_determinants=overall_determinants))