def from_path(cls,
path,
frequency='per_session',
subject_id=None,
visit_id=None,
archive=None):
if os.path.isdir(path):
within_exts = frozenset(
split_extension(f)[1] for f in os.listdir(path)
if not f.startswith('.'))
try:
data_format = DataFormat.by_within_dir_exts(within_exts)
except NiAnalysisDataFormatNotRegisteredError:
# Fall back to general directory format
data_format = directory_format
name = os.path.basename(path)
else:
filename = os.path.basename(path)
name, ext = split_extension(filename)
data_format = DataFormat.by_ext(ext)
return cls(name,
data_format,
frequency=frequency,
path=path,
derived=False,
subject_id=subject_id,
visit_id=visit_id,
archive=archive)
def _gen_outfilename(self):
if isdefined(self.inputs.out_file):
out_name = self.inputs.out_file
else:
first, ext = split_extension(
os.path.basename(self.inputs.first_scan))
second, _ = split_extension(
os.path.basename(self.inputs.second_scan))
out_name = os.path.join(
os.getcwd(), "{}_{}_concat{}".format(first, second, ext))
return out_name
def _gen_outfilename(self):
if isdefined(self.inputs.out_file):
filename = self.inputs.out_file
else:
_, ext = split_extension(os.path.basename(self.inputs.operands[0]))
filename = os.getcwd()
for op in self.inputs.operands:
try:
op_str = split_extension(os.path.basename(op))[0]
except Exception:
op_str = str(float(op))
filename += '_' + op_str
filename += '_' + self.inputs.operation + ext
return filename
def download_resource(download_path, dataset, data_format_name, session_label):
data_format = DataFormat.by_name(data_format_name)
try:
resource = dataset.resources[data_format.xnat_resource_name]
except KeyError:
raise NiAnalysisError(
"Didn't find {} resource in {} dataset matching '{}' in {}".format(
data_format.xnat_resource_name, dataset.type))
tmp_dir = download_path + '.download'
resource.download_dir(tmp_dir)
dataset_label = dataset.id + '-' + special_char_re.sub('_', dataset.type)
src_path = os.path.join(tmp_dir, session_label, 'scans', dataset_label,
'resources', data_format.xnat_resource_name,
'files')
if not data_format.directory:
fnames = os.listdir(src_path)
match_fnames = [
f for f in fnames
if lower(split_extension(f)[-1]) == lower(data_format.extension)
]
if len(match_fnames) == 1:
src_path = os.path.join(src_path, match_fnames[0])
else:
raise NiAnalysisMissingDataException(
"Did not find single file with extension '{}' "
"(found '{}') in resource '{}'".format(data_format.extension,
"', '".join(fnames),
src_path))
shutil.move(src_path, download_path)
shutil.rmtree(tmp_dir)
def _gen_outfilename(self):
if isdefined(self.inputs.out_file):
filename = self.inputs.out_file
else:
base, ext = split_extension(os.path.basename(self.inputs.in_file))
filename = os.path.join(os.getcwd(), "{}_pad{}".format(base, ext))
return filename
def setUp(self):
self.reset_dirs()
shutil.rmtree(self.archive_cache_dir, ignore_errors=True)
os.makedirs(self.archive_cache_dir)
self._delete_test_subjects()
download_all_datasets(self.cache_dir,
SERVER,
'{}_{}'.format(self.XNAT_TEST_PROJECT,
self.name),
overwrite=False)
with self._connect() as mbi_xnat:
project = mbi_xnat.projects[self.PROJECT]
subject = mbi_xnat.classes.SubjectData(label='{}_{}'.format(
self.PROJECT, self.SUBJECT),
parent=project)
session = mbi_xnat.classes.MrSessionData(
label=self.session_label(), parent=subject)
for fname in os.listdir(self.cache_dir):
if fname == FIELDS_FNAME:
continue
name, ext = split_extension(fname)
dataset = mbi_xnat.classes.MrScanData(type=name,
parent=session)
resource = dataset.create_resource(
DataFormat.by_ext(ext).name.upper())
resource.upload(os.path.join(self.cache_dir, fname), fname)
def _gen_grad_filename(self, comp):
filename = getattr(self.inputs, comp + 's_file')
if not isdefined(filename):
base, _ = split_extension(os.path.basename(self.inputs.in_file))
filename = os.path.join(
os.getcwd(), "{base}_{comp}s.{comp}".format(base=base,
comp=comp))
return filename
def _gen_outfilename(self):
if isdefined(self.inputs.out_file):
fpath = self.inputs.out_file
else:
fname = (
split_extension(os.path.basename(self.inputs.in_file))[0] +
'_dicom')
fpath = os.path.join(os.getcwd(), fname)
return fpath
def _gen_outfilename(self):
if isdefined(self.inputs.out_file):
out_name = self.inputs.out_file
else:
base, orig_ext = split_extension(
os.path.basename(self.inputs.in_file))
ext = (self.inputs.out_ext
if isdefined(self.inputs.out_ext) else orig_ext)
out_name = os.path.join(os.getcwd(), "{}_conv{}".format(base, ext))
return out_name
def _gen_outfilename(self):
if isdefined(self.inputs.out_file):
filename = self.inputs.out_file
else:
base, ext = split_extension(os.path.basename(self.inputs.in_file))
if isdefined(self.inputs.bzero):
suffix = 'b0'
else:
suffix = 'dw'
filename = os.path.join(os.getcwd(),
"{}_{}{}".format(base, suffix, ext))
return filename
def download_dataset(cls, tmp_dir, xresource, xdataset, dataset,
session_label, cache_path):
# Download resource to zip file
zip_path = os.path.join(tmp_dir, 'download.zip')
with open(zip_path, 'w') as f:
xresource.xnat_session.download_stream(xresource.uri + '/files',
f,
format='zip',
verbose=True)
digests = cls.get_digests(xresource)
# Extract downloaded zip file
expanded_dir = os.path.join(tmp_dir, 'expanded')
try:
with ZipFile(zip_path) as zip_file:
zip_file.extractall(expanded_dir)
except BadZipfile as e:
raise NiAnalysisError("Could not unzip file '{}' ({})".format(
xresource.id, e))
data_path = os.path.join(
expanded_dir, session_label, 'scans',
(xdataset.id + '-' + special_char_re.sub('_', xdataset.type)),
'resources', dataset.format.xnat_resource_name, 'files')
if not dataset.format.directory:
# If the dataformat is not a directory (e.g. DICOM),
# attempt to locate a single file within the resource
# directory with the appropriate filename and add that
# to be the complete data path.
fnames = os.listdir(data_path)
match_fnames = [
f for f in fnames if (lower(split_extension(f)[-1]) == lower(
dataset.format.extension))
]
if len(match_fnames) == 1:
data_path = os.path.join(data_path, match_fnames[0])
else:
raise NiAnalysisMissingDataException(
"Did not find single file with extension '{}' "
"(found '{}') in resource '{}'".format(
dataset.format.extension, "', '".join(fnames),
data_path))
shutil.move(data_path, cache_path)
with open(cache_path + XnatArchive.MD5_SUFFIX, 'w') as f:
json.dump(digests, f)
shutil.rmtree(tmp_dir)
def _list_outputs(self):
if (not isdefined(self.inputs.compression)
or (self.inputs.compression == 'y'
or self.inputs.compression == 'i')):
im_ext = '.nii.gz'
else:
im_ext = '.nii'
outputs = self._outputs().get()
# As Dcm2niix sometimes prepends a prefix onto the filenames to avoid
# name clashes with multiple echos, we need to check the output folder
# for all filenames that end with the "generated filename".
out_dir = self._gen_filename('out_dir')
fname = self._gen_filename('filename') + im_ext
base, ext = split_extension(fname)
match_re = re.compile(r'(_e\d+)?{}(_(?:e|c)\d+)?{}'.format(
base, ext if ext is not None else ''))
products = [
os.path.join(out_dir, f) for f in os.listdir(out_dir)
if match_re.match(f) is not None
]
if len(products) == 1:
converted = products[0]
elif len(products) > 1 and self.inputs.multifile_concat:
ex_file = nib.load(products[0])
data = ex_file.get_data()
merged_file = np.zeros(
(data.shape[0], data.shape[1], data.shape[2], len(products)))
for i, el in enumerate(products):
f = nib.load(el)
merged_file[:, :, :, i] = f.get_data()
im2save = nib.Nifti1Image(merged_file, ex_file.affine)
nib.save(im2save, out_dir + fname)
converted = out_dir + fname
elif len(products) > 1 and not self.inputs.multifile_concat:
converted = products[-1]
else:
raise NiAnalysisError(
"No products produced by dcm2niix ({})".format(', '.join(
os.listdir(out_dir))))
outputs['converted'] = converted
return outputs
def _list_outputs(self):
"""Execute this module.
"""
# Initiate output
outputs = self._base_outputs()
out_files = []
missing_files = []
# Open XNAT session
sess_kwargs = {}
if 'user' in self.inputs.trait_names(): # Because InputSpec is dynamic
sess_kwargs['user'] = self.inputs.user
if 'password' in self.inputs.trait_names():
sess_kwargs['password'] = self.inputs.password
logger.debug("Session kwargs: {}".format(sess_kwargs))
with xnat.connect(server=self.inputs.server,
**sess_kwargs) as xnat_login:
# Add session for derived scans if not present
session, cache_dir = self._get_session(xnat_login)
# Make session cache dir
if not os.path.exists(cache_dir):
os.makedirs(cache_dir, stat.S_IRWXU | stat.S_IRWXG)
# Loop through datasets connected to the sink and copy them to the
# cache directory and upload to daris.
for dataset in self.datasets:
assert dataset.frequency == self.frequency
assert dataset.derived, (
"{} (format: {}, freq: {}) isn't derived".format(
dataset.name, dataset.format_name, dataset.frequency))
filename = getattr(self.inputs, dataset.name + PATH_SUFFIX)
if not isdefined(filename):
missing_files.append(dataset.name)
continue # skip the upload for this file
ext = dataset.format.extension
assert split_extension(filename)[1] == ext, (
"Mismatching extension '{}' for format '{}' ('{}')".format(
split_extension(filename)[1], dataset.format.name,
dataset.format.extension))
src_path = os.path.abspath(filename)
out_fname = dataset.fname()
# Copy to local cache
dst_path = os.path.join(cache_dir, out_fname)
out_files.append(dst_path)
shutil.copyfile(src_path, dst_path)
# Create md5 digest
with open(dst_path) as f:
digests = {out_fname: hashlib.md5(f.read()).hexdigest()}
with open(dst_path + XnatArchive.MD5_SUFFIX, 'w') as f:
json.dump(digests, f)
# Upload to XNAT
xdataset = xnat_login.classes.MrScanData(
type=dataset.basename(), parent=session)
# Delete existing resource
# TODO: probably should have check to see if we want to
# override it
try:
xresource = xdataset.resources[dataset.format.name.upper()]
xresource.delete()
except KeyError:
pass
xresource = xdataset.create_resource(
dataset.format.name.upper())
xresource.upload(dst_path, out_fname)
for field in self.fields:
assert field.frequency == self.frequency
assert field.derived, ("{} isn't derived".format(field))
session.fields[field.prefixed_name] = getattr(
self.inputs, field.name + FIELD_SUFFIX)
if missing_files:
# FIXME: Not sure if this should be an exception or not,
# indicates a problem but stopping now would throw
# away the datasets that were created
logger.warning("Missing output datasets '{}' in XnatSink".format(
"', '".join(str(f) for f in missing_files)))
# Return cache file paths
outputs['out_files'] = out_files
return outputs
def _list_outputs(self):
"""Execute this module.
"""
# Initiate outputs
outputs = self._base_outputs()
out_files = []
missing_files = []
# Get output dir from base ArchiveSink class (will change depending on
# whether it is per session/subject/visit/project)
out_path = self._get_output_path()
out_dir = os.path.abspath(os.path.join(*out_path))
# Make session dir
if not os.path.exists(out_dir):
os.makedirs(out_dir, stat.S_IRWXU | stat.S_IRWXG)
# Loop through datasets connected to the sink and copy them to archive
# directory
for spec in self.datasets:
assert spec.derived, (
"Should only be sinking derived datasets, not '{}'".format(
spec.name))
filename = getattr(self.inputs, spec.name + PATH_SUFFIX)
ext = spec.format.extension
if not isdefined(filename):
missing_files.append(spec.name)
continue # skip the upload for this file
if lower(split_extension(filename)[1]) != lower(ext):
raise NiAnalysisError(
"Mismatching extension '{}' for format '{}' ('{}')".format(
split_extension(filename)[1], spec.format, ext))
assert spec.frequency == self.frequency
# Copy to local system
src_path = os.path.abspath(filename)
out_fname = spec.fname()
dst_path = os.path.join(out_dir, out_fname)
out_files.append(dst_path)
if os.path.isfile(src_path):
shutil.copyfile(src_path, dst_path)
elif os.path.isdir(src_path):
shutil.copytree(src_path, dst_path)
else:
assert False
if missing_files:
# FIXME: Not sure if this should be an exception or not,
# indicates a problem but stopping now would throw
# away the datasets that were created
logger.warning("Missing input datasets '{}' in LocalSink".format(
"', '".join(missing_files)))
# Return cache file paths
outputs['out_files'] = out_files
# Loop through fields connected to the sink and save them in the
# fields JSON file
out_fields = []
fpath = self.fields_path(self.frequency)
# Open fields JSON, locking to prevent other processes
# reading or writing
if self.fields:
with InterProcessLock(fpath + LOCK, logger=logger):
try:
with open(fpath, 'rb') as f:
fields = json.load(f)
except IOError as e:
if e.errno == errno.ENOENT:
fields = {}
else:
raise
# Update fields JSON and write back to file.
for spec in self.fields:
value = getattr(self.inputs, spec.name + FIELD_SUFFIX)
qual_name = self.prefix_study_name(spec.name)
if spec.dtype is str:
assert isinstance(value, basestring)
else:
assert isinstance(value, spec.dtype)
fields[qual_name] = value
out_fields.append((qual_name, value))
with open(fpath, 'wb') as f:
json.dump(fields, f)
outputs['out_fields'] = out_fields
return outputs
