def __init__(self,
name,
extension=None,
desc='',
directory=False,
within_dir_exts=None,
converters=None):
if not name.islower():
raise NiAnalysisUsageError(
"All data format names must be lower case ('{}')".format(name))
if extension is None and not directory:
raise NiAnalysisUsageError(
"Extension for '{}' format can only be None if it is a "
"directory".format(name))
self._name = name
self._extension = extension
self._desc = desc
self._directory = directory
if within_dir_exts is not None:
if not directory:
raise NiAnalysisUsageError(
"'within_dir_exts' keyword arg is only valid "
"for directory data formats, not '{}'".format(name))
within_dir_exts = frozenset(within_dir_exts)
self._within_dir_exts = within_dir_exts
self._converters = converters if converters is not None else {}
def __init__(
self,
name,
format,
pattern=None, # @ReservedAssignment @IgnorePep8
frequency='per_session',
derived=False,
id=None, # @ReservedAssignment @IgnorePep8
order=None,
dicom_tags=None,
is_regex=False,
study=None):
if pattern is None and id is None:
raise NiAnalysisUsageError(
"Either 'pattern' or 'id' need to be provided to "
"DatasetMatch constructor")
BaseDataset.__init__(self, name, format, frequency)
BaseMatch.__init__(self, pattern, derived, order, study, is_regex)
if dicom_tags is not None and format.name != 'dicom':
raise NiAnalysisUsageError(
"Cannot use 'dicom_tags' kwarg with non-DICOM "
"format ({})".format(format))
self._dicom_tags = dicom_tags
if order is not None and id is not None:
raise NiAnalysisUsageError(
"Cannot provide both 'order' and 'id' to a dataset"
"match")
self._id = str(id) if id is not None else id
def _list_outputs(self):
outputs = self._outputs().get()
new_files = set(os.listdir(os.getcwd())) - self.listdir_before
if len(new_files) > 1:
raise NiAnalysisUsageError(
"Zip archives can only contain a single directory, found '{}'"
.format("', '".join(new_files)))
try:
unzipped = next(iter(new_files))
except StopIteration:
raise NiAnalysisUsageError(
"No files or directories found in unzipped directory")
outputs['gunzipped'] = os.path.join(os.getcwd(), unzipped)
return outputs
def __init__(self, name, value):
self._name = name
if value is None:
self._dtype = None
else:
if not isinstance(value, (int, float, basestring, tuple, list)):
raise NiAnalysisUsageError(
"Invalid type for '{}' option default ({}), {}, "
"can be one of int, float or str".format(
name, value, type(value)))
self._dtype = (str
if isinstance(value, basestring) else type(value))
self._value = value
def __init__(self, name, archive, runner, inputs, options=None,
**kwargs):
options = [] if options is None else options
try:
if not issubclass(type(self).__dict__['__metaclass__'],
MultiStudyMetaClass):
raise KeyError
except KeyError:
raise NiAnalysisUsageError(
"Need to set MultiStudyMetaClass (or sub-class) as "
"the metaclass of all classes derived from "
"MultiStudy")
super(MultiStudy, self).__init__(name, archive, runner, inputs,
options=options, **kwargs)
self._sub_studies = {}
for sub_study_spec in self.sub_study_specs():
# Create copies of the input datasets to pass to the
# __init__ method of the generated sub-studies
sub_study_cls = sub_study_spec.study_class
mapped_inputs = []
for inpt in inputs:
try:
mapped_inputs.append(
inpt.renamed(sub_study_spec.map(inpt.name)))
except NiAnalysisNameError:
pass # Ignore datasets not required for sub-study
mapped_options = []
for opt_name in sub_study_cls.option_spec_names():
mapped_name = sub_study_spec.inverse_map(opt_name)
option = self._get_option(mapped_name)
mapped_options.append(option.renamed(opt_name))
# Create sub-study
sub_study = sub_study_spec.study_class(
name + '_' + sub_study_spec.name,
archive, runner, mapped_inputs,
options=mapped_options,
enforce_inputs=False)
# # Set sub-study as attribute
# setattr(self, sub_study_spec.name, sub_study)
# Append to dictionary of sub_studies
if sub_study_spec.name in self._sub_studies:
raise NiAnalysisNameError(
sub_study_spec.name,
"Duplicate sub-study names '{}'"
.format(sub_study_spec.name))
self._sub_studies[sub_study_spec.name] = sub_study
def create_pipeline(self, *args, **kwargs):
"""
Creates a Pipeline object, passing the study (self) as the first
argument
"""
pipeline = Pipeline(self, *args, **kwargs)
# Register options used before the pipeline was created
try:
pipeline._used_options.update(self._pre_options.pop(pipeline.name))
except KeyError:
pass
if self._pre_options:
raise NiAnalysisUsageError(
"Orphanned pre-options for '{}' pipeline(s) remain in "
"'{}' {} after creating '{}' pipeline. Please check "
"pipeline generation code".format(
"', '".join(self._pre_options.keys()), self.name,
type(self).__name__, pipeline.name))
return pipeline
def __init__(self,
name,
archive,
runner,
inputs,
options=None,
subject_ids=None,
visit_ids=None,
enforce_inputs=True,
reprocess=False):
try:
if not issubclass(
type(self).__dict__['__metaclass__'], StudyMetaClass):
raise KeyError
except KeyError:
raise NiAnalysisUsageError(
"Need to have StudyMetaClass (or a sub-class) as "
"the metaclass of all classes derived from Study")
if isinstance(options, dict):
# Convert dictionary of key-value pairs into list of Option
# objects
options = [Option(k, v) for k, v in options.items()]
elif options is None:
options = []
self._name = name
self._archive = archive
self._runner = runner
self._inputs = {}
# "Bind" data specs in the class to the current study object
# this will allow them to prepend the study name to the name
# of the dataset
self._bound_specs = {}
for opt in options:
try:
opt_spec = self._option_specs[opt.name]
except KeyError:
raise NiAnalysisNameError(
"Provided option '{}' is not present in the "
"allowable options for {} classes ('{}')".format(
opt.name,
type(self).__name__,
"', '".join(self.default_options)))
if not isinstance(opt.value, opt_spec.dtype):
raise NiAnalysisUsageError(
"Incorrect datatype for '{}' option provided "
"to '{}' {}, {} ({}). Should be {}".format(
opt.name, name,
type(self).__name__, type(opt.value), opt_spec.dtype))
if (opt_spec.choices is not None
and opt.value not in opt_spec.choices):
raise NiAnalysisUsageError(
"Provided value for '{}' option in '{}' {}, {}, is "
"not a valid choice. Can be one of {}".format(
opt.name, name,
type(self).__name__, opt.value,
','.join(opt_spec.choices)))
self._options = dict((o.name, o) for o in options)
self._subject_ids = subject_ids
self._visit_ids = visit_ids
self._tree_cache = None
# Add each "input dataset" checking to see whether the given
# dataset_spec name is valid for the study type
for inpt in inputs:
if inpt.name not in self._data_specs:
raise NiAnalysisNameError(
inpt.name,
"Match name '{}' isn't in data specs of {} ('{}')".format(
inpt.name, self.__class__.__name__,
"', '".join(self._data_specs)))
self._inputs[inpt.name] = inpt.bind(self)
for spec in self.data_specs():
if not spec.derived:
# Emit a warning if an acquired dataset has not been
# provided for an "acquired dataset"
if spec.name not in self._inputs:
msg = (" acquired dataset '{}' was not given as an "
"input of {}.".format(spec.name, self))
if spec.optional:
logger.info('Optional' + msg)
else:
if enforce_inputs:
raise NiAnalysisMissingInputError(
'Non-optional' + msg + " Pipelines "
"depending on this dataset will not run")
else:
self._bound_specs[spec.name] = spec.bind(self)
self._reprocess = reprocess
# Record options accessed before a pipeline is created
# so they can be attributed to the pipeline after creation
self._pre_options = defaultdict(list)
def __new__(metacls, name, bases, dct): # @NoSelf @UnusedVariable
if not any(issubclass(b, Study) for b in bases):
raise NiAnalysisUsageError(
"StudyMetaClass can only be used for classes that "
"have Study as a base class")
try:
add_data_specs = dct['add_data_specs']
except KeyError:
add_data_specs = []
try:
add_option_specs = dct['add_option_specs']
except KeyError:
add_option_specs = []
combined_attrs = set()
combined_data_specs = {}
combined_option_specs = {}
for base in reversed(bases):
# Get the combined class dictionary including base dicts
# excluding auto-added properties for data and option specs
combined_attrs.update(
a for a in dir(base)
if (not issubclass(base, Study) or a not in base.spec_names()))
try:
combined_data_specs.update(
(d.name, d) for d in base.data_specs())
except AttributeError:
pass
try:
combined_option_specs.update(
(o.name, o) for o in base.option_specs())
except AttributeError:
pass
combined_attrs.update(dct.keys())
combined_data_specs.update((d.name, d) for d in add_data_specs)
combined_option_specs.update((o.name, o) for o in add_option_specs)
# Check that the pipeline names in data specs correspond to a
# pipeline method in the class
for spec in add_data_specs:
pipe_name = spec.pipeline_name
if pipe_name is not None and pipe_name not in combined_attrs:
raise NiAnalysisUsageError(
"Pipeline to generate '{}', '{}', is not present"
" in '{}' class".format(spec.name, spec.pipeline_name,
name))
# Check for name clashes between data and option specs
spec_name_clashes = (set(combined_data_specs)
& set(combined_option_specs))
if spec_name_clashes:
raise NiAnalysisUsageError(
"'{}' name both data and option specs in '{}' class".format(
"', '".join(spec_name_clashes), name))
dct['_data_specs'] = combined_data_specs
dct['_option_specs'] = combined_option_specs
# Add properties to the class corresponding to each data spec
# for spec in combined_data_specs.values():
# if spec.name in combined_attrs:
# raise NiAnalysisUsageError(
# "'{}' in data_specs clashes with existing class "
# "member of the same name so cannot add property. "
# "Please rename.".format(spec.name))
# dct[spec.name] = make_data_property(spec)
# # Add properties to the class corresponding to each option spec
# for spec in combined_option_specs.values():
# if spec.name in combined_attrs:
# raise NiAnalysisUsageError(
# "'{}' in option_specs clashes with existing class "
# "member of the same name so cannot add property. "
# "Please rename.".format(spec.name))
# dct[spec.name] = make_option_property(spec)
return type(name, bases, dct)
def data(self, name, subject_id=None, visit_id=None):
"""
Returns the Dataset or Field associated with the name,
generating derived datasets as required
Parameters
----------
name : str
The name of the DatasetSpec|FieldSpec to retried the
datasets for
subject_id : int | str | List[int|str] | None
The subject ID or subject IDs to return. If None all are
returned
visit_id : int | str | List[int|str] | None
The visit ID or visit IDs to return. If None all are
returned
Returns
-------
data : Dataset | Field | List[Dataset|Field]
Either a single Dataset or field if a single subject_id
and visit_id are provided, otherwise a list of datasets or
fields corresponding to the given name
"""
def is_single(id_):
return isinstance(id_, (str, int))
subject_ids = ([subject_id] if is_single(subject_id) else subject_id)
visit_ids = ([visit_id] if is_single(visit_id) else visit_id)
spec = self.bound_data_spec(name)
if isinstance(spec, BaseMatch):
data = spec.matches
else:
try:
self.runner.run(spec.pipeline(),
subject_ids=subject_ids,
visit_ids=visit_ids)
except NiAnalysisNoRunRequiredException:
pass
if isinstance(spec, BaseDataset):
data = chain(*((d for d in n.datasets
if d.name == spec.prefixed_name)
for n in self.tree.nodes(spec.frequency)))
elif isinstance(spec, BaseField):
data = chain(*((f for f in n.fields
if f.name == spec.prefixed_name)
for n in self.tree.nodes(spec.frequency)))
else:
assert False
if subject_ids is not None and spec.frequency in ('per_session',
'per_subject'):
data = [d for d in data if d.subject_id in subject_ids]
if visit_ids is not None and spec.frequency in ('per_session',
'per_visit'):
data = [d for d in data if d.visit_id in visit_ids]
if not data:
raise NiAnalysisUsageError(
"No matching data found (subject_id={}, visit_id={})".format(
subject_id, visit_id))
if is_single(subject_id) and is_single(visit_id):
assert len(data) == 1
data = data[0]
else:
data = list(data)
return data
def __new__(metacls, name, bases, dct): # @NoSelf @UnusedVariable
if not any(issubclass(b, MultiStudy) for b in bases):
raise NiAnalysisUsageError(
"MultiStudyMetaClass can only be used for classes that "
"have MultiStudy as a base class")
try:
add_sub_study_specs = dct['add_sub_study_specs']
except KeyError:
add_sub_study_specs = dct['add_sub_study_specs'] = []
try:
add_data_specs = dct['add_data_specs']
except KeyError:
add_data_specs = dct['add_data_specs'] = []
try:
add_option_specs = dct['add_option_specs']
except KeyError:
add_option_specs = dct['add_option_specs'] = []
dct['_sub_study_specs'] = sub_study_specs = {}
for base in reversed(bases):
try:
add_sub_study_specs.update(
(d.name, d) for d in base.sub_study_specs())
except AttributeError:
pass
sub_study_specs.update(
(s.name, s) for s in add_sub_study_specs)
explicitly_added_data_specs = [s.name for s in add_data_specs]
explicitly_added_option_specs = [s.name
for s in add_option_specs]
# Loop through all data specs that haven't been explicitly
# mapped and add a data spec in the multi class.
for sub_study_spec in sub_study_specs.values():
for data_spec in sub_study_spec.auto_data_specs:
trans_sname = sub_study_spec.apply_prefix(
data_spec.name)
if trans_sname not in explicitly_added_data_specs:
initkwargs = data_spec.initkwargs()
initkwargs['name'] = trans_sname
if data_spec.pipeline_name is not None:
trans_pname = sub_study_spec.apply_prefix(
data_spec.pipeline_name)
initkwargs['pipeline_name'] = trans_pname
# Check to see whether pipeline has already been
# translated or always existed in the class (when
# overriding default options for example)
if trans_pname not in dct:
dct[trans_pname] = MultiStudy.translate(
sub_study_spec.name,
data_spec.pipeline_name)
add_data_specs.append(type(data_spec)(**initkwargs))
for opt_spec in sub_study_spec.auto_option_specs:
trans_sname = sub_study_spec.apply_prefix(
opt_spec.name)
if trans_sname not in explicitly_added_option_specs:
add_option_specs.append(
opt_spec.renamed(trans_sname))
cls = StudyMetaClass(name, bases, dct)
# Check all names in name-map correspond to data or option
# specs
for sub_study_spec in sub_study_specs.values():
local_spec_names = list(
sub_study_spec.study_class.spec_names())
for global_name, local_name in sub_study_spec._name_map.items():
if local_name not in local_spec_names:
raise NiAnalysisUsageError(
"'{}' in name-map for '{}' sub study spec in {}"
"MultiStudy class does not name a data or "
"option spec in {} class"
.format(local_name, sub_study_spec.name,
name, sub_study_spec.study_class))
if global_name not in cls.spec_names():
raise NiAnalysisUsageError(
"'{}' in name-map for '{}' sub study spec in {}"
"MultiStudy class does not name data or option spec"
.format(global_name, sub_study_spec.name, name))
return cls