def from_path(cls, path, frequency='per_session', subject_id=None, visit_id=None, archive=None): if os.path.isdir(path): within_exts = frozenset( split_extension(f)[1] for f in os.listdir(path) if not f.startswith('.')) try: data_format = DataFormat.by_within_dir_exts(within_exts) except NiAnalysisDataFormatNotRegisteredError: # Fall back to general directory format data_format = directory_format name = os.path.basename(path) else: filename = os.path.basename(path) name, ext = split_extension(filename) data_format = DataFormat.by_ext(ext) return cls(name, data_format, frequency=frequency, path=path, derived=False, subject_id=subject_id, visit_id=visit_id, archive=archive)
def _gen_outfilename(self): if isdefined(self.inputs.out_file): out_name = self.inputs.out_file else: first, ext = split_extension( os.path.basename(self.inputs.first_scan)) second, _ = split_extension( os.path.basename(self.inputs.second_scan)) out_name = os.path.join( os.getcwd(), "{}_{}_concat{}".format(first, second, ext)) return out_name
def _gen_outfilename(self): if isdefined(self.inputs.out_file): filename = self.inputs.out_file else: _, ext = split_extension(os.path.basename(self.inputs.operands[0])) filename = os.getcwd() for op in self.inputs.operands: try: op_str = split_extension(os.path.basename(op))[0] except Exception: op_str = str(float(op)) filename += '_' + op_str filename += '_' + self.inputs.operation + ext return filename
def download_resource(download_path, dataset, data_format_name, session_label): data_format = DataFormat.by_name(data_format_name) try: resource = dataset.resources[data_format.xnat_resource_name] except KeyError: raise NiAnalysisError( "Didn't find {} resource in {} dataset matching '{}' in {}".format( data_format.xnat_resource_name, dataset.type)) tmp_dir = download_path + '.download' resource.download_dir(tmp_dir) dataset_label = dataset.id + '-' + special_char_re.sub('_', dataset.type) src_path = os.path.join(tmp_dir, session_label, 'scans', dataset_label, 'resources', data_format.xnat_resource_name, 'files') if not data_format.directory: fnames = os.listdir(src_path) match_fnames = [ f for f in fnames if lower(split_extension(f)[-1]) == lower(data_format.extension) ] if len(match_fnames) == 1: src_path = os.path.join(src_path, match_fnames[0]) else: raise NiAnalysisMissingDataException( "Did not find single file with extension '{}' " "(found '{}') in resource '{}'".format(data_format.extension, "', '".join(fnames), src_path)) shutil.move(src_path, download_path) shutil.rmtree(tmp_dir)
def _gen_outfilename(self): if isdefined(self.inputs.out_file): filename = self.inputs.out_file else: base, ext = split_extension(os.path.basename(self.inputs.in_file)) filename = os.path.join(os.getcwd(), "{}_pad{}".format(base, ext)) return filename
def setUp(self): self.reset_dirs() shutil.rmtree(self.archive_cache_dir, ignore_errors=True) os.makedirs(self.archive_cache_dir) self._delete_test_subjects() download_all_datasets(self.cache_dir, SERVER, '{}_{}'.format(self.XNAT_TEST_PROJECT, self.name), overwrite=False) with self._connect() as mbi_xnat: project = mbi_xnat.projects[self.PROJECT] subject = mbi_xnat.classes.SubjectData(label='{}_{}'.format( self.PROJECT, self.SUBJECT), parent=project) session = mbi_xnat.classes.MrSessionData( label=self.session_label(), parent=subject) for fname in os.listdir(self.cache_dir): if fname == FIELDS_FNAME: continue name, ext = split_extension(fname) dataset = mbi_xnat.classes.MrScanData(type=name, parent=session) resource = dataset.create_resource( DataFormat.by_ext(ext).name.upper()) resource.upload(os.path.join(self.cache_dir, fname), fname)
def _gen_grad_filename(self, comp): filename = getattr(self.inputs, comp + 's_file') if not isdefined(filename): base, _ = split_extension(os.path.basename(self.inputs.in_file)) filename = os.path.join( os.getcwd(), "{base}_{comp}s.{comp}".format(base=base, comp=comp)) return filename
def _gen_outfilename(self): if isdefined(self.inputs.out_file): fpath = self.inputs.out_file else: fname = ( split_extension(os.path.basename(self.inputs.in_file))[0] + '_dicom') fpath = os.path.join(os.getcwd(), fname) return fpath
def _gen_outfilename(self): if isdefined(self.inputs.out_file): out_name = self.inputs.out_file else: base, orig_ext = split_extension( os.path.basename(self.inputs.in_file)) ext = (self.inputs.out_ext if isdefined(self.inputs.out_ext) else orig_ext) out_name = os.path.join(os.getcwd(), "{}_conv{}".format(base, ext)) return out_name
def _gen_outfilename(self): if isdefined(self.inputs.out_file): filename = self.inputs.out_file else: base, ext = split_extension(os.path.basename(self.inputs.in_file)) if isdefined(self.inputs.bzero): suffix = 'b0' else: suffix = 'dw' filename = os.path.join(os.getcwd(), "{}_{}{}".format(base, suffix, ext)) return filename
def download_dataset(cls, tmp_dir, xresource, xdataset, dataset, session_label, cache_path): # Download resource to zip file zip_path = os.path.join(tmp_dir, 'download.zip') with open(zip_path, 'w') as f: xresource.xnat_session.download_stream(xresource.uri + '/files', f, format='zip', verbose=True) digests = cls.get_digests(xresource) # Extract downloaded zip file expanded_dir = os.path.join(tmp_dir, 'expanded') try: with ZipFile(zip_path) as zip_file: zip_file.extractall(expanded_dir) except BadZipfile as e: raise NiAnalysisError("Could not unzip file '{}' ({})".format( xresource.id, e)) data_path = os.path.join( expanded_dir, session_label, 'scans', (xdataset.id + '-' + special_char_re.sub('_', xdataset.type)), 'resources', dataset.format.xnat_resource_name, 'files') if not dataset.format.directory: # If the dataformat is not a directory (e.g. DICOM), # attempt to locate a single file within the resource # directory with the appropriate filename and add that # to be the complete data path. fnames = os.listdir(data_path) match_fnames = [ f for f in fnames if (lower(split_extension(f)[-1]) == lower( dataset.format.extension)) ] if len(match_fnames) == 1: data_path = os.path.join(data_path, match_fnames[0]) else: raise NiAnalysisMissingDataException( "Did not find single file with extension '{}' " "(found '{}') in resource '{}'".format( dataset.format.extension, "', '".join(fnames), data_path)) shutil.move(data_path, cache_path) with open(cache_path + XnatArchive.MD5_SUFFIX, 'w') as f: json.dump(digests, f) shutil.rmtree(tmp_dir)
def _list_outputs(self): if (not isdefined(self.inputs.compression) or (self.inputs.compression == 'y' or self.inputs.compression == 'i')): im_ext = '.nii.gz' else: im_ext = '.nii' outputs = self._outputs().get() # As Dcm2niix sometimes prepends a prefix onto the filenames to avoid # name clashes with multiple echos, we need to check the output folder # for all filenames that end with the "generated filename". out_dir = self._gen_filename('out_dir') fname = self._gen_filename('filename') + im_ext base, ext = split_extension(fname) match_re = re.compile(r'(_e\d+)?{}(_(?:e|c)\d+)?{}'.format( base, ext if ext is not None else '')) products = [ os.path.join(out_dir, f) for f in os.listdir(out_dir) if match_re.match(f) is not None ] if len(products) == 1: converted = products[0] elif len(products) > 1 and self.inputs.multifile_concat: ex_file = nib.load(products[0]) data = ex_file.get_data() merged_file = np.zeros( (data.shape[0], data.shape[1], data.shape[2], len(products))) for i, el in enumerate(products): f = nib.load(el) merged_file[:, :, :, i] = f.get_data() im2save = nib.Nifti1Image(merged_file, ex_file.affine) nib.save(im2save, out_dir + fname) converted = out_dir + fname elif len(products) > 1 and not self.inputs.multifile_concat: converted = products[-1] else: raise NiAnalysisError( "No products produced by dcm2niix ({})".format(', '.join( os.listdir(out_dir)))) outputs['converted'] = converted return outputs
def _list_outputs(self): """Execute this module. """ # Initiate output outputs = self._base_outputs() out_files = [] missing_files = [] # Open XNAT session sess_kwargs = {} if 'user' in self.inputs.trait_names(): # Because InputSpec is dynamic sess_kwargs['user'] = self.inputs.user if 'password' in self.inputs.trait_names(): sess_kwargs['password'] = self.inputs.password logger.debug("Session kwargs: {}".format(sess_kwargs)) with xnat.connect(server=self.inputs.server, **sess_kwargs) as xnat_login: # Add session for derived scans if not present session, cache_dir = self._get_session(xnat_login) # Make session cache dir if not os.path.exists(cache_dir): os.makedirs(cache_dir, stat.S_IRWXU | stat.S_IRWXG) # Loop through datasets connected to the sink and copy them to the # cache directory and upload to daris. for dataset in self.datasets: assert dataset.frequency == self.frequency assert dataset.derived, ( "{} (format: {}, freq: {}) isn't derived".format( dataset.name, dataset.format_name, dataset.frequency)) filename = getattr(self.inputs, dataset.name + PATH_SUFFIX) if not isdefined(filename): missing_files.append(dataset.name) continue # skip the upload for this file ext = dataset.format.extension assert split_extension(filename)[1] == ext, ( "Mismatching extension '{}' for format '{}' ('{}')".format( split_extension(filename)[1], dataset.format.name, dataset.format.extension)) src_path = os.path.abspath(filename) out_fname = dataset.fname() # Copy to local cache dst_path = os.path.join(cache_dir, out_fname) out_files.append(dst_path) shutil.copyfile(src_path, dst_path) # Create md5 digest with open(dst_path) as f: digests = {out_fname: hashlib.md5(f.read()).hexdigest()} with open(dst_path + XnatArchive.MD5_SUFFIX, 'w') as f: json.dump(digests, f) # Upload to XNAT xdataset = xnat_login.classes.MrScanData( type=dataset.basename(), parent=session) # Delete existing resource # TODO: probably should have check to see if we want to # override it try: xresource = xdataset.resources[dataset.format.name.upper()] xresource.delete() except KeyError: pass xresource = xdataset.create_resource( dataset.format.name.upper()) xresource.upload(dst_path, out_fname) for field in self.fields: assert field.frequency == self.frequency assert field.derived, ("{} isn't derived".format(field)) session.fields[field.prefixed_name] = getattr( self.inputs, field.name + FIELD_SUFFIX) if missing_files: # FIXME: Not sure if this should be an exception or not, # indicates a problem but stopping now would throw # away the datasets that were created logger.warning("Missing output datasets '{}' in XnatSink".format( "', '".join(str(f) for f in missing_files))) # Return cache file paths outputs['out_files'] = out_files return outputs
def _list_outputs(self): """Execute this module. """ # Initiate outputs outputs = self._base_outputs() out_files = [] missing_files = [] # Get output dir from base ArchiveSink class (will change depending on # whether it is per session/subject/visit/project) out_path = self._get_output_path() out_dir = os.path.abspath(os.path.join(*out_path)) # Make session dir if not os.path.exists(out_dir): os.makedirs(out_dir, stat.S_IRWXU | stat.S_IRWXG) # Loop through datasets connected to the sink and copy them to archive # directory for spec in self.datasets: assert spec.derived, ( "Should only be sinking derived datasets, not '{}'".format( spec.name)) filename = getattr(self.inputs, spec.name + PATH_SUFFIX) ext = spec.format.extension if not isdefined(filename): missing_files.append(spec.name) continue # skip the upload for this file if lower(split_extension(filename)[1]) != lower(ext): raise NiAnalysisError( "Mismatching extension '{}' for format '{}' ('{}')".format( split_extension(filename)[1], spec.format, ext)) assert spec.frequency == self.frequency # Copy to local system src_path = os.path.abspath(filename) out_fname = spec.fname() dst_path = os.path.join(out_dir, out_fname) out_files.append(dst_path) if os.path.isfile(src_path): shutil.copyfile(src_path, dst_path) elif os.path.isdir(src_path): shutil.copytree(src_path, dst_path) else: assert False if missing_files: # FIXME: Not sure if this should be an exception or not, # indicates a problem but stopping now would throw # away the datasets that were created logger.warning("Missing input datasets '{}' in LocalSink".format( "', '".join(missing_files))) # Return cache file paths outputs['out_files'] = out_files # Loop through fields connected to the sink and save them in the # fields JSON file out_fields = [] fpath = self.fields_path(self.frequency) # Open fields JSON, locking to prevent other processes # reading or writing if self.fields: with InterProcessLock(fpath + LOCK, logger=logger): try: with open(fpath, 'rb') as f: fields = json.load(f) except IOError as e: if e.errno == errno.ENOENT: fields = {} else: raise # Update fields JSON and write back to file. for spec in self.fields: value = getattr(self.inputs, spec.name + FIELD_SUFFIX) qual_name = self.prefix_study_name(spec.name) if spec.dtype is str: assert isinstance(value, basestring) else: assert isinstance(value, spec.dtype) fields[qual_name] = value out_fields.append((qual_name, value)) with open(fpath, 'wb') as f: json.dump(fields, f) outputs['out_fields'] = out_fields return outputs