def get_field(self, field):
self._check_repository(field)
with self:
xsession = self.get_xsession(field)
val = xsession.fields[field.name]
val = val.replace('"', '"')
val = parse_value(val)
return val
def __init__(self,
name,
value=None,
dtype=None,
frequency='per_session',
array=None,
subject_id=None,
visit_id=None,
dataset=None,
from_analysis=None,
exists=True,
record=None):
# Try to determine dtype and array from value if they haven't
# been provided.
if value is None:
if dtype is None:
raise ArcanaUsageError(
"Either 'value' or 'dtype' must be provided to "
"Field init")
array = bool(array) # Convert to array is None to False
else:
value = parse_value(value)
if isinstance(value, list):
if array is False:
raise ArcanaUsageError(
"Array value passed to '{}', which is explicitly not "
"an array ({})".format(name, value))
array = True
else:
if array:
raise ArcanaUsageError(
"Non-array value ({}) passed to '{}', which expects "
"array{}".format(value, name,
('of type {}'.format(dtype)
if dtype is not None else '')))
array = False
if dtype is None:
if array:
dtype = type(value[0])
else:
dtype = type(value)
else:
# Ensure everything is cast to the correct type
if array:
value = [dtype(v) for v in value]
else:
value = dtype(value)
BaseField.__init__(self, name, dtype, frequency, array)
BaseItemMixin.__init__(self, subject_id, visit_id, dataset,
from_analysis, exists, record)
self._value = value
def run(cls, args):
set_loggers(args.logger)
study_class = resolve_class(args.study_class)
if args.scratch is not None:
scratch_dir = args.scratch
else:
scratch_dir = op.join(op.expanduser('~'), 'banana-scratch')
# Ensure scratch dir exists
os.makedirs(scratch_dir, exist_ok=True)
work_dir = op.join(scratch_dir, 'work')
if args.repository is None:
if args.input:
repository_type = 'basic'
else:
repository_type = 'bids'
else:
repository_type = args.repository[0]
# Load subject_ids from file if single value is provided with
# a '/' in the string
if (args.subject_ids is not None and len(args.subject_ids)
and '/' in args.subject_ids[0]):
with open(args.subject_ids[0]) as f:
subject_ids = f.read().split()
else:
subject_ids = args.subject_ids
# Load visit_ids from file if single value is provided with
# a '/' in the string
if (args.visit_ids is not None and len(args.visit_ids)
and '/' in args.visit_ids[0]):
with open(args.visit_ids[0]) as f:
visit_ids = f.read().split()
else:
visit_ids = args.visit_ids
def init_repo(repo_path,
repo_type,
option_str,
*repo_args,
create_root=False):
if repo_type == 'bids':
if create_root:
os.makedirs(repo_path, exist_ok=True)
repo = BidsRepo(repo_path)
elif repo_type == 'basic':
if len(repo_args) != 1:
raise BananaUsageError(
"Unrecognised arguments passed to '--{}' option "
"({}) exactly 1 additional argument is required for "
"'basic' type repository (DEPTH)".format(
option_str, args.respository))
if create_root:
os.makedirs(repo_path, exist_ok=True)
repo = BasicRepo(repo_path, depth=repo_args[0])
elif repo_type == 'xnat':
nargs = len(repo_args)
if nargs < 1:
raise BananaUsageError(
"Not enough arguments passed to '--{}' option "
"({}), at least 1 additional argument is required for "
"'xnat' type repository (SERVER)".format(
option_str, args.respository))
elif nargs > 3:
raise BananaUsageError(
"Unrecognised arguments passed to '--{}' option "
"({}), at most 3 additional arguments are accepted for"
" 'xnat' type repository (SERVER, USER, PASSWORD)".
format(option_str, args.respository))
repo = XnatRepo(project_id=repo_path,
server=repo_args[0],
user=(repo_args[1] if nargs > 2 else None),
password=(repo_args[2] if nargs > 3 else None),
cache_dir=op.join(scratch_dir, 'cache'))
else:
raise BananaUsageError(
"Unrecognised repository type provided as first argument "
"to '--{}' option ({})".format(option_str, repo_args[0]))
return repo
repository = init_repo(args.repository_path, repository_type,
'repository', *args.repository)
if args.output_repository is not None:
input_repository = repository
tree = repository.cached_tree()
if subject_ids is None:
subject_ids = list(tree.subject_ids)
if visit_ids is None:
visit_ids = list(tree.visit_ids)
fill_tree = True
nargs = len(args.output_repository)
if nargs == 1:
repo_type = 'basic'
out_path = args.output_repository[0]
out_repo_args = [input_repository.depth]
else:
repo_type = args.output_repository[0]
out_path = args.output_repository[1]
out_repo_args = args.output_repository[2:]
repository = init_repo(out_path,
repo_type,
'output_repository',
*out_repo_args,
create_root=True)
else:
input_repository = None
fill_tree = False
if args.email is not None:
email = args.email
else:
try:
email = os.environ['EMAIL']
except KeyError:
email = None
proc_args = {'reprocess': args.reprocess}
if args.processor[0] == 'single':
processor = SingleProc(work_dir, **proc_args)
elif args.processor[0] == 'multi':
if len(args.processor) > 1:
num_processes = args.processor[1]
elif len(args.processor) > 2:
raise BananaUsageError(
"Unrecognised arguments passed to '--processor' option "
"({}) expected at most 1 additional argument for 'multi' "
"type processor (NUM_PROCS)".format(args.processor))
else:
num_processes = cpu_count()
processor = MultiProc(work_dir,
num_processes=num_processes,
**proc_args)
elif args.processor[0] == 'slurm':
if email is None:
raise BananaUsageError(
"Email needs to be provided either via '--email' argument "
"or set in 'EMAIL' environment variable for SLURM "
"processor")
nargs = len(args.processor)
if nargs > 3:
raise BananaUsageError(
"Unrecognised arguments passed to '--processor' option "
"with 'slurm' type ({}), expected at most 2 additional "
"arguments [ACCOUNT, PARTITION]".format(args.processor))
processor = SlurmProc(
work_dir,
account=(args.processor[1] if nargs >= 2 else None),
partition=(args.processor[2] if nargs >= 3 else None),
email=email,
mail_on=('FAIL', ),
**proc_args)
else:
raise BananaUsageError(
"Unrecognised processor type provided as first argument to "
"'--processor' option ({})".format(args.processor[0]))
if args.environment == 'static':
environment = StaticEnv()
else:
environment = ModulesEnv()
parameters = {}
for name, value in args.parameter:
parameters[name] = parse_value(
value, dtype=study_class.param_spec(name).dtype)
if input_repository is not None and input_repository.type == 'bids':
inputs = study_class.get_bids_inputs(args.bids_task,
repository=input_repository)
else:
inputs = {}
for name, pattern in args.input:
spec = study_class.data_spec(name)
if spec.is_fileset:
inpt_cls = InputFilesets
else:
inpt_cls = InputFields
inputs[name] = inpt_cls(name,
pattern=pattern,
is_regex=True,
repository=input_repository)
study = study_class(name=args.study_name,
repository=repository,
processor=processor,
environment=environment,
inputs=inputs,
parameters=parameters,
subject_ids=subject_ids,
visit_ids=visit_ids,
enforce_inputs=args.enforce_inputs,
fill_tree=fill_tree,
bids_task=args.bids_task)
for spec_name in args.cache:
spec = study.bound_spec(spec_name)
if not isinstance(spec, InputFilesets):
raise BananaUsageError(
"Cannot cache non-input fileset '{}'".format(spec_name))
spec.cache()
# Generate data
study.data(args.derivatives)
logger.info("Generated derivatives for '{}'".format(args.derivatives))