def run(cls, args): if not args.quiet: set_loggers(args.logger) if args.scratch is not None: scratch_dir = args.scratch else: scratch_dir = op.join(op.expanduser('~'), 'banana-scratch') # Ensure scratch dir exists os.makedirs(scratch_dir, exist_ok=True) work_dir = op.join(scratch_dir, 'work') proc_args = {'reprocess': args.reprocess} if args.processor[0] == 'single': processor = SingleProc(work_dir, **proc_args) elif args.processor[0] == 'multi': if len(args.processor) > 1: num_processes = args.processor[1] elif len(args.processor) > 2: raise BananaUsageError( "Unrecognised arguments passed to '--processor' option " "({}) expected at most 1 additional argument for 'multi' " "type processor (NUM_PROCS)".format(args.processor)) else: num_processes = cpu_count() processor = MultiProc(work_dir, num_processes=num_processes, **proc_args) else: raise BananaUsageError( "Unrecognised processor type provided as first argument to " "'--processor' option ({})".format(args.processor[0])) if args.environment == 'static': environment = StaticEnv() else: environment = ModulesEnv() parts = args.test_class.split('.') sys.path.insert(0, op.join(args.test_root, parts[:-2])) module = import_module(parts[-2]) sys.path.pop(0) test_cls = getattr(module, parts[-1]) test_cls().generate_reference_data(*args.specs, processor=processor, environment=environment)
def init_dataset(dataset_path, dataset_type, option_str, dataset_args, create_root=False, **kwargs): if dataset_type == 'bids': if create_root: os.makedirs(dataset_path, exist_ok=True) dataset = BidsDataset(dataset_path, **kwargs) elif dataset_type == 'basic': if len(dataset_args) != 1: raise BananaUsageError( "Unrecognised arguments passed to '--{}' option " "({}) exactly 1 additional argument is required for " "'basic' type dataset (DEPTH)".format( option_str, dataset_args)) if create_root: os.makedirs(dataset_path, exist_ok=True) dataset = Dataset(dataset_path, depth=int(dataset_args[0]), **kwargs) elif dataset_type == 'xnat': nargs = len(dataset_args) if nargs < 1: raise BananaUsageError( "Not enough arguments passed to '--{}' option " "({}), at least 1 additional argument is required for " "'xnat' type dataset (SERVER)".format( option_str, dataset_args)) elif nargs > 3: raise BananaUsageError( "Unrecognised arguments passed to '--{}' option " "({}), at most 3 additional arguments are accepted for" " 'xnat' type dataset (SERVER, USER, PASSWORD)".format( option_str, dataset_args)) dataset = XnatRepo( server=dataset_args[0], user=(dataset_args[1] if nargs > 2 else None), password=(dataset_args[2] if nargs > 3 else None), cache_dir=op.join(scratch_dir, 'cache')).dataset( dataset_path, **kwargs) else: raise BananaUsageError( "Unrecognised dataset type provided as first argument " "to '--{}' option ({})".format(option_str, dataset_args[0])) return dataset
def series_coreg_pipeline(self, **name_maps): pipeline = self.new_pipeline( 'series_coreg', desc="Applies coregistration transform to DW series", citations=[], name_maps=name_maps) if self.provided('coreg_ref'): coreg_ref = 'coreg_ref' elif self.provided('coreg_ref_brain'): coreg_ref = 'coreg_ref_brain' else: raise BananaUsageError( "Cannot coregister DW series as reference ('coreg_ref' or " "'coreg_ref_brain') has not been provided to {}".format(self)) # Apply co-registration transformation to DW series pipeline.add( 'mask_transform', fsl.ApplyXFM( output_type='NIFTI_GZ', apply_xfm=True), inputs={ 'in_matrix_file': ('coreg_fsl_mat', text_matrix_format), 'in_file': ('series_preproc', nifti_gz_format), 'reference': (coreg_ref, nifti_gz_format)}, outputs={ 'series_coreg': ('out_file', nifti_gz_format)}, requirements=[fsl_req.v('5.0.10')], wall_time=10) return pipeline
def resolve_class(class_str, prefixes=(DEFAULT_STUDY_CLASS_PATH, )): """ Resolves a class from the '.' delimted module + class name string """ parts = class_str.split('.') module_name = '.'.join(parts[:-1]) class_name = parts[-1] cls = None for prefix in [None] + list(prefixes): if prefix is not None: mod_name = prefix + '.' + module_name else: mod_name = module_name if not mod_name: continue mod_name = mod_name.strip('.') try: module = import_module(mod_name) except ModuleNotFoundError: continue else: try: cls = getattr(module, class_name) except AttributeError: continue else: break if cls is None: raise BananaUsageError("Did not find class '{}'".format(class_str)) return cls
def segmentation_pipeline(self, img_type=2, **name_maps): pipeline = self.new_pipeline( name='FAST_segmentation', name_maps=name_maps, desc="White matter segmentation of the reference image", citations=[fsl_cite]) fast = pipeline.add('fast', fsl.FAST(img_type=img_type, segments=True, out_basename='Reference_segmentation', output_type='NIFTI_GZ'), inputs={'in_files': ('brain', nifti_gz_format)}, requirements=[fsl_req.v('5.0.9')]) # Determine output field of split to use if img_type == 1: split_output = 'out3' elif img_type == 2: split_output = 'out2' else: raise BananaUsageError( "'img_type' parameter can either be 1 or 2 (not {})".format( img_type)) pipeline.add('split', Split(splits=[1, 1, 1], squeeze=True), inputs={'inlist': (fast, 'tissue_class_files')}, outputs={'wm_seg': (split_output, nifti_gz_format)}) return pipeline
def detect_format(path, aux_files): ext = split_extension(path)[1] aux_names = set(aux_files.keys()) for frmt in BIDS_FORMATS: if frmt.extension == ext and set(frmt.aux_files.keys()) == aux_names: return frmt raise BananaUsageError( "No matching BIDS format matches provided path ({}) and aux files ({})" .format(path, aux_files))
def header_extraction_pipeline(self, **name_maps): pipeline = self.new_pipeline( name='header_extraction', name_maps=name_maps, desc=("Pipeline to extract the most important scan " "information from the image header"), citations=[]) input_format = self.input(self.header_image_spec_name).format if input_format == dicom_format: pipeline.add( 'hd_info_extraction', DicomHeaderInfoExtraction( multivol=False), inputs={ 'dicom_folder': (self.header_image_spec_name, dicom_format)}, outputs={ 'tr': ('tr', float), 'start_time': ('start_time', str), 'total_duration': ('total_duration', str), 'real_duration': ('real_duration', str), 'ped': ('ped', str), 'pe_angle': ('pe_angle', str), 'echo_times': ('echo_times', float), 'voxel_sizes': ('voxel_sizes', float), 'main_field_strength': ('B0', float), 'main_field_orient': ('H', float)}) elif input_format == nifti_gz_x_format: pipeline.add( 'hd_info_extraction', NiftixHeaderInfoExtraction(), inputs={ 'in_file': (self.header_image_spec_name, nifti_gz_x_format)}, outputs={ 'tr': ('tr', float), 'start_time': ('start_time', str), 'total_duration': ('total_duration', str), 'real_duration': ('real_duration', str), 'ped': ('ped', str), 'pe_angle': ('pe_angle', str), 'echo_times': ('echo_times', float), 'voxel_sizes': ('voxel_sizes', float), 'main_field_strength': ('B0', float), 'main_field_orient': ('H', float)}) else: raise BananaUsageError( "Can only extract header info if 'magnitude' fileset " "is provided in DICOM or extended NIfTI format (provided {})" .format(self.input(self.header_image_spec_name).format)) return pipeline
def init_repo(repo_path, repo_type, option_str, *repo_args, create_root=False): if repo_type == 'bids': if create_root: os.makedirs(repo_path, exist_ok=True) repo = BidsRepo(repo_path) elif repo_type == 'basic': if len(repo_args) != 1: raise BananaUsageError( "Unrecognised arguments passed to '--{}' option " "({}) exactly 1 additional argument is required for " "'basic' type repository (DEPTH)".format( option_str, args.respository)) if create_root: os.makedirs(repo_path, exist_ok=True) repo = BasicRepo(repo_path, depth=repo_args[0]) elif repo_type == 'xnat': nargs = len(repo_args) if nargs < 1: raise BananaUsageError( "Not enough arguments passed to '--{}' option " "({}), at least 1 additional argument is required for " "'xnat' type repository (SERVER)".format( option_str, args.respository)) elif nargs > 3: raise BananaUsageError( "Unrecognised arguments passed to '--{}' option " "({}), at most 3 additional arguments are accepted for" " 'xnat' type repository (SERVER, USER, PASSWORD)". format(option_str, args.respository)) repo = XnatRepo(project_id=repo_path, server=repo_args[0], user=(repo_args[1] if nargs > 2 else None), password=(repo_args[2] if nargs > 3 else None), cache_dir=op.join(scratch_dir, 'cache')) else: raise BananaUsageError( "Unrecognised repository type provided as first argument " "to '--{}' option ({})".format(option_str, repo_args[0])) return repo
def image_centre(cls, array, offset=None): """ Returns the centre point of the non-zero voxels in the image """ nonzeros = np.argwhere(array) min_ind = nonzeros.min(axis=0) max_ind = nonzeros.max(axis=0) centre = ((max_ind - min_ind) // 2) + min_ind if offset is not None: centre += np.array(offset, dtype=int) if np.any(centre < 0) or np.any(centre > array.shape): raise BananaUsageError( "Specified offset ({}) is larger than the " "dimension of the image / 2 ({})".format( offset, array.shape // 2)) return centre
def sinogram_unlisting_pipeline(self, **kwargs): pipeline = self.new_pipeline( name='prepare_sinogram', desc=('Unlist pet listmode data into several sinograms and ' 'perform ssrb compression to prepare data for motion ' 'detection using PCA pipeline.'), citations=[], **kwargs) if not self.provided('list_mode'): raise BananaUsageError( "'list_mode' was not provided as an input to the study " "so cannot perform sinogram unlisting") prepare_inputs = pipeline.add('prepare_inputs', PrepareUnlistingInputs(), inputs={ 'list_mode': ('list_mode', list_mode_format), 'time_offset': ('time_offset', int), 'num_frames': ('num_frames', int), 'temporal_len': ('temporal_length', float) }) unlisting = pipeline.add( 'unlisting', PETListModeUnlisting(), inputs={'list_inputs': (prepare_inputs, 'out')}, iterfield=['list_inputs']) ssrb = pipeline.add( 'ssrb', SSRB(), inputs={'unlisted_sinogram': (unlisting, 'pet_sinogram')}, requirements=[stir_req.v('3.0')]) pipeline.add( 'merge_sinograms', MergeUnlistingOutputs(), inputs={'sinograms': (ssrb, 'ssrb_sinograms')}, ouputs={'ssrb_sinograms': ('sinogram_folder', directory_format)}, joinsource='unlisting', joinfield=['sinograms']) return pipeline
def __init__( self, spec_name, primary, association, format=None, # @ReservedAssignment @IgnorePep8 fieldmap_order=0, **kwargs): FilesetSelector.__init__(self, spec_name, format, frequency='per_session', **kwargs) self._primary = primary if association not in self.VALID_ASSOCIATIONS: raise BananaUsageError( "Invalid association '{}' passed to BidsAssociatedSelector, " "can be one of '{}'".format( association, "', '".join(self.VALID_ASSOCIATIONS))) self._association = association self._fieldmap_order = fieldmap_order
def _list_outputs(self): outputs = self._outputs().get() mag_img = nib.load(self.inputs.magnitude) tissue_phase_img = nib.load(self.inputs.tissue_phase) mask_img = nib.load(self.inputs.mask) mag = mag_img.get_fdata() tissue_phase = tissue_phase_img.get_fdata() mask = mask_img.get_fdata() if mag.shape != tissue_phase.shape: raise BananaUsageError( "Dimensions of provided magnitude and phase images " "differ ({} and {})".format(mag.shape, tissue_phase.shape)) pos_mask = np.where(tissue_phase > 0) * mask # Positive phase mask rho = np.ones(tissue_phase.shape) rho[pos_mask] = np.max(0, np.pi - (tissue_phase[pos_mask] / np.pi)) swi = mag * (rho**self.inputs.alpha) # Set filenames in output spec outputs['out_file'] = self._gen_filename('out_file') out_file_img = nib.Nifti1Image(swi, mag_img.affine, mag_img.header) nib.save(out_file_img, outputs['out_file']) return outputs
def coreg_fsl_mat_pipeline(self, **name_maps): if self.branch('coreg_method', 'flirt'): pipeline = self._coreg_mat_pipeline(**name_maps) elif self.branch('coreg_method', 'ants'): # Convert ANTS transform to FSL transform pipeline = self.new_pipeline( name='convert_ants_to_fsl_coreg_mat', name_maps=name_maps) if self.provided('coreg_ref'): source = 'mag_preproc' ref = 'coreg_ref' elif self.provided('coreg_ref_brain'): source = 'brain' ref = 'coreg_ref_brain' else: raise BananaUsageError( "Either 'coreg_ref' or 'coreg_ref_brain' needs to be " "provided in order to derive brain_coreg or brain_coreg_" "mask") pipeline.add( 'transform_conv', ANTs2FSLMatrixConversion( ras2fsl=True), inputs={ 'itk_file': ('coreg_ants_mat', text_matrix_format), 'source_file': (source, nifti_gz_format), 'reference_file': (ref, nifti_gz_format)}, outputs={ 'coreg_fsl_mat': ('fsl_matrix', text_matrix_format)}, requirements=[c3d_req.v('1.0')]) else: self.unhandled_branch('coreg_method') return pipeline
def preprocess_pipeline(self, **name_maps): """ Performs a series of FSL preprocessing steps, including Eddy and Topup Parameters ---------- phase_dir : str{AP|LR|IS} The phase encode direction """ # Determine whether we can correct for distortion, i.e. if reference # scans are provided # Include all references references = [fsl_cite, eddy_cite, topup_cite, distort_correct_cite, n4_cite] if self.branch('preproc_denoise'): references.extend(dwidenoise_cites) pipeline = self.new_pipeline( name='preprocess', name_maps=name_maps, desc=( "Preprocess dMRI studies using distortion correction"), citations=references) # Create nodes to gradients to FSL format if self.input('series').format == dicom_format: extract_grad = pipeline.add( "extract_grad", ExtractFSLGradients(), inputs={ 'in_file': ('series', dicom_format)}, outputs={ 'grad_dirs': ('bvecs_file', fsl_bvecs_format), 'bvalues': ('bvals_file', fsl_bvals_format)}, requirements=[mrtrix_req.v('3.0rc3')]) grad_fsl_inputs = {'in1': (extract_grad, 'bvecs_file'), 'in2': (extract_grad, 'bvals_file')} elif self.provided('grad_dirs') and self.provided('bvalues'): grad_fsl_inputs = {'in1': ('grad_dirs', fsl_bvecs_format), 'in2': ('bvalues', fsl_bvals_format)} else: raise BananaUsageError( "Either input 'magnitude' image needs to be in DICOM format " "or gradient directions and b-values need to be explicitly " "provided to {}".format(self)) # Gradient merge node grad_fsl = pipeline.add( "grad_fsl", MergeTuple(2), inputs=grad_fsl_inputs) gradients = (grad_fsl, 'out') # Create node to reorient preproc out_file if self.branch('reorient2std'): reorient = pipeline.add( 'fslreorient2std', fsl.utils.Reorient2Std( output_type='NIFTI_GZ'), inputs={ 'in_file': ('series', nifti_gz_format)}, requirements=[fsl_req.v('5.0.9')]) reoriented = (reorient, 'out_file') else: reoriented = ('series', nifti_gz_format) # Denoise the dwi-scan if self.branch('preproc_denoise'): # Run denoising denoise = pipeline.add( 'denoise', DWIDenoise(), inputs={ 'in_file': reoriented}, requirements=[mrtrix_req.v('3.0rc3')]) # Calculate residual noise subtract_operands = pipeline.add( 'subtract_operands', Merge(2), inputs={ 'in1': reoriented, 'in2': (denoise, 'noise')}) pipeline.add( 'subtract', MRCalc( operation='subtract'), inputs={ 'operands': (subtract_operands, 'out')}, outputs={ 'noise_residual': ('out_file', mrtrix_image_format)}, requirements=[mrtrix_req.v('3.0rc3')]) denoised = (denoise, 'out_file') else: denoised = reoriented # Preproc kwargs preproc_kwargs = {} preproc_inputs = {'in_file': denoised, 'grad_fsl': gradients} if self.provided('reverse_phase'): if self.provided('magnitude', default_okay=False): dwi_reference = ('magnitude', mrtrix_image_format) else: # Extract b=0 volumes dwiextract = pipeline.add( 'dwiextract', ExtractDWIorB0( bzero=True, out_ext='.nii.gz'), inputs={ 'in_file': denoised, 'fslgrad': gradients}, requirements=[mrtrix_req.v('3.0rc3')]) # Get first b=0 from dwi b=0 volumes extract_first_b0 = pipeline.add( "extract_first_vol", MRConvert( coord=(3, 0)), inputs={ 'in_file': (dwiextract, 'out_file')}, requirements=[mrtrix_req.v('3.0rc3')]) dwi_reference = (extract_first_b0, 'out_file') # Concatenate extracted forward rpe with reverse rpe combined_images = pipeline.add( 'combined_images', MRCat(), inputs={ 'first_scan': dwi_reference, 'second_scan': ('reverse_phase', mrtrix_image_format)}, requirements=[mrtrix_req.v('3.0rc3')]) # Create node to assign the right PED to the diffusion prep_dwi = pipeline.add( 'prepare_dwi', PrepareDWI(), inputs={ 'pe_dir': ('ped', float), 'ped_polarity': ('pe_angle', float)}) preproc_kwargs['rpe_pair'] = True distortion_correction = True preproc_inputs['se_epi'] = (combined_images, 'out_file') else: distortion_correction = False preproc_kwargs['rpe_none'] = True if self.parameter('preproc_pe_dir') is not None: preproc_kwargs['pe_dir'] = self.parameter('preproc_pe_dir') preproc = pipeline.add( 'dwipreproc', DWIPreproc( no_clean_up=True, out_file_ext='.nii.gz', # FIXME: Need to determine this programmatically # eddy_parameters = '--data_is_shelled ' temp_dir='dwipreproc_tempdir', **preproc_kwargs), inputs=preproc_inputs, outputs={ 'eddy_par': ('eddy_parameters', eddy_par_format)}, requirements=[mrtrix_req.v('3.0rc3'), fsl_req.v('5.0.10')], wall_time=60) if distortion_correction: pipeline.connect(prep_dwi, 'pe', preproc, 'pe_dir') mask = pipeline.add( 'dwi2mask', BrainMask( out_file='brainmask.nii.gz'), inputs={ 'in_file': (preproc, 'out_file'), 'grad_fsl': gradients}, requirements=[mrtrix_req.v('3.0rc3')]) # Create bias correct node pipeline.add( "bias_correct", DWIBiasCorrect( method='ants'), inputs={ 'grad_fsl': gradients, # internal 'in_file': (preproc, 'out_file'), 'mask': (mask, 'out_file')}, outputs={ 'series_preproc': ('out_file', nifti_gz_format)}, requirements=[mrtrix_req.v('3.0rc3'), ants_req.v('2.0')]) return pipeline
def arlo(te, y): """ Used in calculating the R2* signal Parameters ---------- te : list(float) array containing te values(in s) y : 4-d array a multi-echo data set of arbitrary dimension echo should be the last dimension Outputs ------- r2 : 3-d array r2-map(in Hz) empty when only one echo is provided If you use this function please cite Pei M, Nguyen TD, Thimmappa ND, Salustri C, Dong F, Cooper MA, Li J, Prince MR, Wang Y. Algorithm for fast monoexponential fitting based on Auto-Regression on Linear Operations(ARLO) of data. Magn Reson Med. 2015 Feb (2): 843-50. doi: 10.1002/mrm.25137. Epub 2014 Mar 24. PubMed PMID: 24664497 PubMed Central PMCID: PMC4175304. """ num_echos = len(te) if num_echos < 2: return [] if y.shape[-1] != num_echos: raise BananaUsageError( 'Last dimension of y has size {}, expected {}'.format( y.shape[-1], num_echos)) yy = np.zeros(y.shape[:3]) yx = np.zeros(y.shape[:3]) beta_yx = np.zeros(y.shape[:3]) beta_xx = np.zeros(y.shape[:3]) for j in range(num_echos - 2): alpha = ((te[j + 2] - te[j]) * (te[j + 2] - te[j]) / 2) / (te[j + 1] - te[j]) tmp = (2 * te[j + 2] * te[j + 2] - te[j] * te[j + 2] - te[j] * te[j] + 3 * te[j] * te[j + 1] - 3 * te[j + 1] * te[j + 2]) / 6 beta = tmp / (te[j + 2] - te[j + 1]) gamma = tmp / (te[j + 1] - te[j]) echo0 = y[:, :, :, j] echo1 = y[:, :, :, j + 1] echo2 = y[:, :, :, j + 2] # [te[j+2]-te[j]-alpha+gamma alpha-beta-gamma beta]/((te[2]-te[1])/3) y1 = (echo0 * (te[j + 2] - te[j] - alpha + gamma) + echo1 * (alpha - beta - gamma) + echo2 * beta) x1 = echo0 - echo2 yy = yy + y1 * y1 yx = yx + y1 * x1 beta_yx = beta_yx + beta * y1 * x1 beta_xx = beta_xx + beta * x1 * x1 r2 = (yx + beta_xx) / (beta_yx + yy) # Set NaN and inf values to 0.0 r2[~np.isfinite(r2)] = 0.0 return r2
def brain_coreg_pipeline(self, **name_maps): """ Coregistered + brain-extracted images can be derived in 2-ways. If an explicit brain-extracted reference is provided to 'coreg_ref_brain' then that is used to coregister a brain extracted image against. Alternatively, if only a skull-included reference is provided then the registration is performed with skulls-included and then brain extraction is performed after """ if self.provided('coreg_ref_brain'): # If a reference brain extracted image is provided we coregister # the brain extracted image to that pipeline = self.coreg_pipeline( name='brain_coreg', name_maps=dict( input_map={ 'mag_preproc': 'brain', 'coreg_ref': 'coreg_ref_brain'}, output_map={ 'mag_coreg': 'brain_coreg'}, name_maps=name_maps)) # Apply coregistration transform to brain mask if self.branch('coreg_method', 'flirt'): pipeline.add( 'mask_transform', ApplyXFM( output_type='NIFTI_GZ', apply_xfm=True), inputs={ 'in_matrix_file': (pipeline.node('flirt'), 'out_matrix_file'), 'in_file': ('brain_mask', nifti_gz_format), 'reference': ('coreg_ref_brain', nifti_gz_format)}, outputs={ 'brain_mask_coreg': ('out_file', nifti_gz_format)}, requirements=[fsl_req.v('5.0.10')], wall_time=10) elif self.branch('coreg_method', 'ants'): # Convert ANTs transform matrix to FSL format if we have used # Ants registration so we can apply the transform using # ApplyXFM pipeline.add( 'mask_transform', ants.resampling.ApplyTransforms( interpolation='Linear', input_image_type=3, invert_transform_flags=[True, True, False]), inputs={ 'input_image': ('brain_mask', nifti_gz_format), 'reference_image': ('coreg_ref_brain', nifti_gz_format), 'transforms': (pipeline.node('ants_reg'), 'forward_transforms')}, requirements=[ants_req.v('1.9')], mem_gb=16, wall_time=30) else: self.unhandled_branch('coreg_method') elif self.provided('coreg_ref'): # If coreg_ref is provided then we co-register the non-brain # extracted images and then brain extract the co-registered image pipeline = self.brain_extraction_pipeline( name='bet_coreg', input_map={'mag_preproc': 'mag_coreg'}, output_map={'brain': 'brain_coreg', 'brain_mask': 'brain_mask_coreg'}, name_maps=name_maps) else: raise BananaUsageError( "Either 'coreg_ref' or 'coreg_ref_brain' needs to be provided " "in order to derive brain_coreg or brain_mask_coreg") return pipeline
def generate_test_data(cls, study_class, in_repo, out_repo, in_server=None, out_server=None, work_dir=None, parameters=(), include=None, skip=(), include_bases=(), reprocess=False, repo_depth=0, modules_env=False, clean_work_dir=True, loggers=('nipype.workflow', 'arcana', 'banana')): """ Generates reference data for a pipeline tester unittests given a study class and set of parameters Parameters ---------- study_class : type(Study) The path to the study class to test, e.g. banana.study.MriStudy in_repo : str The path to repository that houses the input data out_repo : str If the 'xnat_server' argument is provided then out is interpreted as the project ID to use the XNAT server (the project must exist already). Otherwise it is interpreted as the path to a basic repository in_server : str | None The server to download the input data from out_server : str | None The server to upload the reference data to work_dir : str The work directory parameters : dict[str, *] Parameter to set when initialising the study include : list[str] | None Spec names to include in the output repository. If None all names except those listed in 'skip' are included skip : list[str] Spec names to skip in the generation process. Only valid if 'include' is None include_bases : list[type(Study)] List of base classes in which all entries in their data specification are added to the list to include reprocess : bool Whether to reprocess the generated datasets repo_depth : int The depth of the input repository modules_env : bool Whether to use modules environment or not clean_work_dir : bool Whether to clean the Nipype work directory or not """ for logger_name in loggers: logger = logging.getLogger(logger_name) logger.setLevel(logging.INFO) handler = logging.StreamHandler() formatter = logging.Formatter("%(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) if work_dir is None: work_dir = tempfile.mkdtemp() else: work_dir = work_dir if study_class.__name__.endswith('Study'): study_name = study_class.__name__[:-len('Study')] else: study_name = study_class.__name__ # Get output repository to write the data to if in_server is not None: in_repo = XnatRepo(project_id=in_repo, server=in_server, cache_dir=op.join(work_dir, 'xnat-cache')) else: in_repo = BasicRepo(in_repo, depth=repo_depth) temp_repo_root = op.join(work_dir, 'temp-repo') if os.path.exists(temp_repo_root) and reprocess: shutil.rmtree(temp_repo_root) os.makedirs(temp_repo_root, exist_ok=True) temp_repo = BasicRepo(temp_repo_root, depth=repo_depth) inputs = None for session in in_repo.tree().sessions: session_inputs = [] for item in chain(session.filesets, session.fields): if isinstance(item, Fileset): inpt = InputFilesets(item.basename, item.basename, item.format, repository=in_repo) else: inpt = InputFields(item.name, item.name, item.dtype, repository=in_repo) try: spec = study_class.data_spec(inpt) except ArcanaNameError: print( "Skipping {} as it doesn't match a spec in {}".format( item, study_class)) else: session_inputs.append(inpt) session_inputs = sorted(session_inputs) if inputs is not None and session_inputs != inputs: raise BananaUsageError( "Inconsistent inputs ({} and {}) found in sessions of {}". format(inputs, session_inputs, in_repo)) else: inputs = session_inputs if modules_env: env = ModulesEnv() else: env = StaticEnv() study = study_class( study_name, repository=temp_repo, processor=SingleProc( work_dir, reprocess=reprocess, clean_work_dir_between_runs=clean_work_dir, prov_ignore=( SingleProc.DEFAULT_PROV_IGNORE + ['.*/pkg_version', 'workflow/nodes/.*/requirements/.*'])), environment=env, inputs=inputs, parameters=parameters, subject_ids=in_repo.tree().subject_ids, visit_ids=in_repo.tree().visit_ids, fill_tree=True) if include is None: # Get set of methods that could override pipeline getters in # base classes that are not included potentially_overridden = set() for cls in chain(include_bases, [study_class]): potentially_overridden.update(cls.__dict__.keys()) include = set() for base in study_class.__mro__: if not hasattr(base, 'add_data_specs'): continue for spec in base.add_data_specs: if isinstance(spec, BaseInputSpecMixin) or spec.name in skip: continue if (base is study_class or base in include_bases or spec.pipeline_getter in potentially_overridden): include.add(spec.name) # Generate all derived data for spec_name in sorted(include): study.data(spec_name) # Get output repository to write the data to if out_server is not None: out_repo = XnatRepo(project_id=out_repo, server=out_server, cache_dir=op.join(work_dir, 'xnat-cache')) else: out_repo = BasicRepo(out_repo, depth=repo_depth) # Upload data to repository for spec in study.data_specs(): try: data = study.data(spec.name, generate=False) except ArcanaMissingDataException: continue for item in data: if not item.exists: logger.info("Skipping upload of non-existant {}".format( item.name)) continue if skip is not None and item.name in skip: logger.info("Forced skip of {}".format(item.name)) continue if item.is_fileset: item_cpy = Fileset(name=item.name, format=item.format, frequency=item.frequency, path=item.path, aux_files=copy(item.aux_files), subject_id=item.subject_id, visit_id=item.visit_id, repository=out_repo, exists=True) else: item_cpy = Field(name=item.name, value=item.value, dtype=item.dtype, frequency=item.frequency, array=item.array, subject_id=item.subject_id, visit_id=item.visit_id, repository=out_repo, exists=True) logger.info("Uploading {}".format(item_cpy)) item_cpy.put() logger.info("Uploaded {}".format(item_cpy)) logger.info( "Finished generating and uploading test data for {}".format( study_class))
def _list_outputs(self): outputs = self._outputs().get() mag_fname = outputs['magnitude'] = self._gen_filename('magnitude') phase_fname = outputs['phase'] = self._gen_filename('phase') q_fname = outputs['q'] = self._gen_filename('q') mag_paths = defaultdict(dict) phase_paths = defaultdict(dict) # Compile regular expression for extracting channel, echo and # complex axis indices from input file names fname_re = re.compile(self.inputs.in_fname_re) for dpath, dct in ((self.inputs.magnitudes_dir, mag_paths), (self.inputs.phases_dir, phase_paths)): for fname in os.listdir(dpath): match = fname_re.match(fname) if match is None: logger.warning("Skipping '{}' file in '{}' as it doesn't " "match expected filename pattern for raw " "channel files ('{}')".format( fname, dpath, self.inputs.in_fname_re)) continue dct[match.group('channel')][match.group('echo')] = op.join( dpath, fname) if len(mag_paths) != len(phase_paths): raise BananaUsageError( "Mismatching number of channels between magnitude and phase " "channels") hip = None for chann_i in mag_paths: if len(mag_paths[chann_i]) != 2: raise BananaUsageError( "Expected exactly two echos for channel magnitude {}, " "found {}".format(chann_i, len(mag_paths[chann_i]))) if len(phase_paths[chann_i]) != 2: raise BananaUsageError( "Expected exactly two echos for channel magnitude {}, " "found {}".format(chann_i, len(phase_paths[chann_i]))) mag1 = nib.load(mag_paths[chann_i][0]) phase1 = nib.load(phase_paths[chann_i][0]) mag2 = nib.load(mag_paths[chann_i][1]) phase2 = nib.load(phase_paths[chann_i][1]) # Get array data mag1_array = mag1.get_fdata() phase1_array = phase1.get_fdata() mag2_array = mag2.get_fdata() phase2_array = phase2.get_fdata() if hip is None: hip = np.zeros(mag1_array.shape) sum_mag = np.zeros(mag1_array.shape) hip += mag1_array * mag2_array * np.exp( -1j * (phase1_array - phase2_array)) sum_mag += mag1_array * mag2_array # Get magnitude and phase phase = np.angle(hip) mag = np.abs(hip) q = mag / sum_mag # Create NIfTI images phase_img = nib.Nifti1Image(phase, phase1.affine, phase1.header) mag_img = nib.Nifti1Image(mag, mag1.affine, mag1.header) q_img = nib.Nifti1Image(q, mag1.affine, mag1.header) # Save NIfTIs nib.save(phase_img, phase_fname) nib.save(mag_img, mag_fname) nib.save(q_img, q_fname) return outputs
def _list_outputs(self): outputs = self._outputs().get() hip = None for fname in os.listdir(self.inputs.channels_dir): img = nib.load(op.join(self.inputs.channels_dir, fname)) img_data = img.get_fdata() if hip is None: hip = np.zeros(img_data.shape[:3], dtype=complex) sum_mag = np.zeros(img_data.shape[:3]) r2star = np.zeros(img_data.shape[:3]) mag = np.zeros(img_data.shape[:3]) num_echos = img_data.shape[3] if len(self.inputs.echo_times) != num_echos: raise BananaUsageError( "Number of echos differs from provided dataset ({}) and " "echo times ({})".format(num_echos, self.inputs.echo_times)) if num_echos < 2: raise BananaUsageError( "At least two echos required for channel magnitude {}, " "found {}".format(fname, num_echos)) cmplx_coil = np.squeeze(img_data[:, :, :, :, 0] + 1j * img_data[:, :, :, :, 1]) phase_coil = np.angle(cmplx_coil) mag_coil = np.abs(cmplx_coil) for i, j in zip(range(0, num_echos - 1), range(1, num_echos)): # Get successive echos mag_a = mag_coil[:, :, :, i] mag_b = mag_coil[:, :, :, j] phase_a = phase_coil[:, :, :, i] phase_b = phase_coil[:, :, :, j] # Combine HIP and sum and total magnitude hip += mag_a * mag_b * np.exp(-1j * (phase_a - phase_b)) sum_mag += mag_a * mag_b # Calculate R2* sum_echo_mags = np.sum(mag_coil, axis=3) r2star += sum_echo_mags * arlo(self.inputs.echo_times, mag_coil) mag += sum_echo_mags if hip is None: raise BananaUsageError( "No channels loaded from channels directory {}".format( self.inputs.channels_dir)) # Get magnitude and phase phase = np.angle(hip) mag = np.abs(hip) q = mag / sum_mag # Set filenames in output spec outputs['phase'] = self._gen_filename('phase') outputs['magnitude'] = self._gen_filename('magnitude') outputs['q'] = self._gen_filename('q') outputs['r2star'] = self._gen_filename('r2star') # Create NIfTI images phase_img = nib.Nifti1Image(phase, img.affine, img.header) mag_img = nib.Nifti1Image(mag, img.affine, img.header) q_img = nib.Nifti1Image(q, img.affine, img.header) r2star_img = nib.Nifti1Image(r2star, img.affine, img.header) # Save NIfTIs nib.save(phase_img, outputs['phase']) nib.save(mag_img, outputs['magnitude']) nib.save(q_img, outputs['q']) nib.save(r2star_img, outputs['r2star']) return outputs
def run(cls, args): set_loggers(args.logger) study_class = resolve_class(args.study_class) if args.scratch is not None: scratch_dir = args.scratch else: scratch_dir = op.join(op.expanduser('~'), 'banana-scratch') # Ensure scratch dir exists os.makedirs(scratch_dir, exist_ok=True) work_dir = op.join(scratch_dir, 'work') if args.repository is None: if args.input: repository_type = 'basic' else: repository_type = 'bids' else: repository_type = args.repository[0] # Load subject_ids from file if single value is provided with # a '/' in the string if (args.subject_ids is not None and len(args.subject_ids) and '/' in args.subject_ids[0]): with open(args.subject_ids[0]) as f: subject_ids = f.read().split() else: subject_ids = args.subject_ids # Load visit_ids from file if single value is provided with # a '/' in the string if (args.visit_ids is not None and len(args.visit_ids) and '/' in args.visit_ids[0]): with open(args.visit_ids[0]) as f: visit_ids = f.read().split() else: visit_ids = args.visit_ids def init_repo(repo_path, repo_type, option_str, *repo_args, create_root=False): if repo_type == 'bids': if create_root: os.makedirs(repo_path, exist_ok=True) repo = BidsRepo(repo_path) elif repo_type == 'basic': if len(repo_args) != 1: raise BananaUsageError( "Unrecognised arguments passed to '--{}' option " "({}) exactly 1 additional argument is required for " "'basic' type repository (DEPTH)".format( option_str, args.respository)) if create_root: os.makedirs(repo_path, exist_ok=True) repo = BasicRepo(repo_path, depth=repo_args[0]) elif repo_type == 'xnat': nargs = len(repo_args) if nargs < 1: raise BananaUsageError( "Not enough arguments passed to '--{}' option " "({}), at least 1 additional argument is required for " "'xnat' type repository (SERVER)".format( option_str, args.respository)) elif nargs > 3: raise BananaUsageError( "Unrecognised arguments passed to '--{}' option " "({}), at most 3 additional arguments are accepted for" " 'xnat' type repository (SERVER, USER, PASSWORD)". format(option_str, args.respository)) repo = XnatRepo(project_id=repo_path, server=repo_args[0], user=(repo_args[1] if nargs > 2 else None), password=(repo_args[2] if nargs > 3 else None), cache_dir=op.join(scratch_dir, 'cache')) else: raise BananaUsageError( "Unrecognised repository type provided as first argument " "to '--{}' option ({})".format(option_str, repo_args[0])) return repo repository = init_repo(args.repository_path, repository_type, 'repository', *args.repository) if args.output_repository is not None: input_repository = repository tree = repository.cached_tree() if subject_ids is None: subject_ids = list(tree.subject_ids) if visit_ids is None: visit_ids = list(tree.visit_ids) fill_tree = True nargs = len(args.output_repository) if nargs == 1: repo_type = 'basic' out_path = args.output_repository[0] out_repo_args = [input_repository.depth] else: repo_type = args.output_repository[0] out_path = args.output_repository[1] out_repo_args = args.output_repository[2:] repository = init_repo(out_path, repo_type, 'output_repository', *out_repo_args, create_root=True) else: input_repository = None fill_tree = False if args.email is not None: email = args.email else: try: email = os.environ['EMAIL'] except KeyError: email = None proc_args = {'reprocess': args.reprocess} if args.processor[0] == 'single': processor = SingleProc(work_dir, **proc_args) elif args.processor[0] == 'multi': if len(args.processor) > 1: num_processes = args.processor[1] elif len(args.processor) > 2: raise BananaUsageError( "Unrecognised arguments passed to '--processor' option " "({}) expected at most 1 additional argument for 'multi' " "type processor (NUM_PROCS)".format(args.processor)) else: num_processes = cpu_count() processor = MultiProc(work_dir, num_processes=num_processes, **proc_args) elif args.processor[0] == 'slurm': if email is None: raise BananaUsageError( "Email needs to be provided either via '--email' argument " "or set in 'EMAIL' environment variable for SLURM " "processor") nargs = len(args.processor) if nargs > 3: raise BananaUsageError( "Unrecognised arguments passed to '--processor' option " "with 'slurm' type ({}), expected at most 2 additional " "arguments [ACCOUNT, PARTITION]".format(args.processor)) processor = SlurmProc( work_dir, account=(args.processor[1] if nargs >= 2 else None), partition=(args.processor[2] if nargs >= 3 else None), email=email, mail_on=('FAIL', ), **proc_args) else: raise BananaUsageError( "Unrecognised processor type provided as first argument to " "'--processor' option ({})".format(args.processor[0])) if args.environment == 'static': environment = StaticEnv() else: environment = ModulesEnv() parameters = {} for name, value in args.parameter: parameters[name] = parse_value( value, dtype=study_class.param_spec(name).dtype) if input_repository is not None and input_repository.type == 'bids': inputs = study_class.get_bids_inputs(args.bids_task, repository=input_repository) else: inputs = {} for name, pattern in args.input: spec = study_class.data_spec(name) if spec.is_fileset: inpt_cls = InputFilesets else: inpt_cls = InputFields inputs[name] = inpt_cls(name, pattern=pattern, is_regex=True, repository=input_repository) study = study_class(name=args.study_name, repository=repository, processor=processor, environment=environment, inputs=inputs, parameters=parameters, subject_ids=subject_ids, visit_ids=visit_ids, enforce_inputs=args.enforce_inputs, fill_tree=fill_tree, bids_task=args.bids_task) for spec_name in args.cache: spec = study.bound_spec(spec_name) if not isinstance(spec, InputFilesets): raise BananaUsageError( "Cannot cache non-input fileset '{}'".format(spec_name)) spec.cache() # Generate data study.data(args.derivatives) logger.info("Generated derivatives for '{}'".format(args.derivatives))
def display_slice_panel(self, filesets, img_size=5, row_kwargs=None, offset=None, **kwargs): """ Displays an image in a Nx3 panel axial, coronal and sagittal slices for the filesets correspdong to each of the data names provided. Parameters ---------- data_names : List[str] List of image names to plot as rows of a panel size : Tuple(2)[int] Size of the figure to plot row_kargs : List[Dict[str, *]] A list of row-specific kwargs to passed on to _display_mid_slices offset : Tuple(3)[int] An array of integers with which to offset the slices displayed """ n_rows = len(filesets) if row_kwargs is None: row_kwargs = repeat({}) elif not n_rows == len(row_kwargs): raise BananaUsageError("Length of row_kwargs ({}) needs to " "match length of filesets ({})".format( len(row_kwargs), n_rows)) # Set up figure gs = GridSpec(n_rows, 3) gs.update(wspace=0.0, hspace=0.0) fig = plt.figure(figsize=(3 * img_size, n_rows * img_size)) # Loop through derivatives and generate image for i, (fileset, rkwargs) in enumerate(zip(filesets, row_kwargs)): array = fileset.get_array() header = fileset.get_header() if fileset.format in (nifti_format, nifti_gz_format): vox = header['pixdim'][1:4] elif fileset.format == dicom_format: vox = [float(v) for v in header.PixelSpacing] vox.append(float(header.SliceThickness)) else: raise BananaUsageError( "'{}' format images are not supported for display slice ". format(fileset.format)) rkwargs = copy(rkwargs) rkwargs.update(kwargs) try: self._display_mid_slices(array, vox, fig, gs, i, offset=offset, **rkwargs) except BananaUsageError as e: raise BananaUsageError( str(e) + " displaying {}".format(fileset.path)) # Remove space around figure plt.tight_layout(0.0)