def test_local_execution(self): """ Test local execution. """ logfile = tempfile.NamedTemporaryFile(suffix=".log").name hopla(self.script, d=[self.demodir, self.demodir], l=[2, 3], fbreak=[False, True], verbose=0, hopla_iterative_kwargs=["d", "fbreak"], hopla_logfile=logfile, hopla_cpus=4, hopla_optional=["fbreak", "verbose"]) os.remove(logfile)
def test_cluster_execution(self): """ Test cluster execution. """ logfile = tempfile.NamedTemporaryFile(suffix=".log").name logdir = tempfile.mkdtemp() hopla(self.script, d=[self.demodir, self.demodir], l=[2, 3], fbreak=[False, True], verbose=0, hopla_iterative_kwargs=["d", "fbreak"], hopla_logfile=logfile, hopla_cpus=4, hopla_optional=["fbreak", "verbose"], hopla_cluster=True, hopla_cluster_logdir=logdir, hopla_cluster_queue="DUMMY") shutil.rmtree(logdir) os.remove(logfile)
def test_normal_execution(self, mock_scheduler): """ Test normal execution. """ # Local execution for fbreak in (True, False): hopla(self.script, d=["dir1"], l=[2, 3], fbreak=fbreak, verbose=[0], hopla_iterative_kwargs=["d", "verbose"], hopla_optional=["fbreak", "verbose"]) print(mock_scheduler.call_args_list[-1][1]["commands"]) generated_commands = mock_scheduler.call_args_list[-1][1][ "commands"] expected_commands = [ [self.script, "-d", "dir1", "--verbose", "0", "-l", "2", "3"]] if fbreak: expected_commands[0].insert(5, "--fbreak") self.assertEqual(sorted(generated_commands), sorted(expected_commands)) for optional in (None, "some_string"): hopla(self.script, d=["dir1"], l=[2, 3], o=optional, verbose=[0], hopla_iterative_kwargs=["d", "verbose"], hopla_optional=["fbreak", "verbose"]) generated_commands = mock_scheduler.call_args_list[-1][1][ "commands"] expected_commands = [ [self.script, "-d", "dir1", "--verbose", "0", "-l", "2", "3"]] if optional is not None: expected_commands[0].extend(["-o", optional]) self.assertEqual(sorted(generated_commands), sorted(expected_commands)) # Local execution with boolean iter for fbreak in (True, False): hopla(self.script, d=["dir1"], l=[2, 3], fbreak=[fbreak], verbose=0, hopla_iterative_kwargs=["d", "fbreak"], hopla_optional=["fbreak", "verbose"]) generated_commands = mock_scheduler.call_args_list[-1][1][ "commands"] expected_commands = [ [self.script, "-d", "dir1", "-l", "2", "3", "--verbose", "0"]] if fbreak: expected_commands[0].insert(3, "--fbreak") self.assertEqual(generated_commands, expected_commands) for optional in (None, "some_string"): hopla(self.script, d=["dir1"], l=[2, 3], o=[optional], verbose=0, hopla_iterative_kwargs=["d", "o"], hopla_optional=["fbreak", "verbose"]) generated_commands = mock_scheduler.call_args_list[-1][1][ "commands"] expected_commands = [ [self.script, "-d", "dir1", "-l", "2", "3", "--verbose", "0"]] if optional is not None: expected_commands[0].insert(3, "-o") expected_commands[0].insert(4, optional) self.assertEqual(sorted(generated_commands), sorted(expected_commands))
from pprint import pprint # Hopla import import hopla as root from hopla.converter import hopla # Define script parameters apath = os.path.join(os.path.abspath(os.path.dirname(root.__file__)), "demo") script = os.path.join(apath, "my_ls_script.py") cluster_logdir = os.path.join(apath, "pbs_logs") # Local execution print("-------", "local execution") status, exitcodes = hopla( script, d=[apath, apath, apath], b=False, v=0, l=[1, 2], hopla_iterative_kwargs=["d"], hopla_verbose=1, hopla_cpus=10) pprint(exitcodes) # Local execution with optional arguments print("-------", "local execution with optional arguments") status, exitcodes = hopla( script, dir=[apath, apath, apath], b=False, verbose=1, mylist=[1, 2], hopla_iterative_kwargs=["dir"], hopla_verbose=1, hopla_cpus=10, hopla_optional=["dir", "verbose", "mylist"]) pprint(exitcodes) # Local execution with boolean iter print("-------", "local execution with boolean iter") status, exitcodes = hopla( script, dir=[apath, apath, apath], b=[False, True, False], verbose=1,
def preproc_multi(bidsdir, template, jipdir, fslconfig, auto, resample, anatorient="RAS", funcorient="RAS", njobs=1, simage=None, shome=None, sbinds=None, verbose=0): """ Perform the FMRI preprocessing on a BIDS organized directory in parallel (without FUGUE or TOPUP). This function can be called with a singularity image that contains all the required software. If a 'jip_trf' directory is available in the session directory, the code will use the available JIP transformation. Parameters bidsdir: str the BIDS organized directory. template: str the path to the template in RAS coordiante system. jipdir: str the jip software binary path. fslconfig: str the FSL configuration script. auto: bool control the JIP window with the script. resample: bool if set resample the input template to fit the anatomical image. anatorient: str, default "RAS" the input anatomical image orientation. funcorient: str, default "RAS" the input functional image orientation. njobs: int, default 1 the number of parallel jobs. simage: simg, default None a singularity image. shome: str, default None a home directory for the singularity image. sbinds: str, default None shared directories for the singularity image. verbose: int, default 0 control the verbosity level. """ # TODO: remove when all controls available in pypipe if not isinstance(auto, bool): auto = eval(auto) resample = eval(resample) sbinds = eval(sbinds) # Parse the BIDS directory desc_file = os.path.join(bidsdir, "dataset_description.json") if not os.path.isfile(desc_file): raise ValueError( "Expect '{0}' in a BIDS organized directory.".format(desc_file)) with open(desc_file, "rt") as of: desc = json.load(of) name = desc["Name"] if verbose > 0: print("Processing dataset '{0}'...".format(name)) dataset = {name: {}} funcfiles, anatfiles, subjects, outputs, trs, encdirs, normfiles, coregfiles = [ [] for _ in range(8) ] outdir = os.path.join(bidsdir, "derivatives", "preproc") for sesdir in glob.glob(os.path.join(bidsdir, "sub-*", "ses-*")): split = sesdir.split(os.sep) sid, ses = split[-2:] _anatfiles = glob.glob(os.path.join(sesdir, "anat", "sub-*T1w.nii.gz")) _funcfiles = glob.glob(os.path.join(sesdir, "func", "sub-*bold.nii.gz")) if len(_anatfiles) != 1: if verbose > 0: print("Skip session '{0}': no valid anat.".format(sesdir)) continue if len(_funcfiles) == 0: if verbose > 0: print("Skip session '{0}': no valid func.".format(sesdir)) continue descs = [] for path in _funcfiles: runs = re.findall(".*_(run-[0-9]*)_.*", path) if len(runs) != 1: if verbose > 0: print("Skip path '{0}': not valid name.".format(path)) continue sesdir = os.path.join(outdir, sid, ses) rundir = os.path.join(sesdir, runs[0]) if not os.path.isdir(rundir): os.makedirs(rundir) desc_file = path.replace(".nii.gz", ".json") if not os.path.isfile(desc_file): break with open(desc_file, "rt") as of: _desc = json.load(of) if _desc["PhaseEncodingDirection"].startswith("i"): warp_restrict = [1, 0, 0] elif _desc["PhaseEncodingDirection"].startswith("j"): warp_restrict = [0, 1, 0] elif _desc["PhaseEncodingDirection"].startswith("k"): warp_restrict = [0, 0, 1] else: raise ValueError( "Unknown encode phase direction : {0}...".format( _desc["PhaseEncodingDirection"])) jip_normalization = os.path.join(sesdir, "jip_trf", "Normalization", "align.com") if not os.path.isfile(jip_normalization): if verbose > 0: print("No JIP normalization align.com file found " "in {0}.".format(sesdir)) jip_normalization = None jip_coregistration = os.path.join(sesdir, "jip_trf", "Coregistration", "align.com") if not os.path.isfile(jip_coregistration): if verbose > 0: print("No JIP coregistration align.com file found " "in {0}.".format(sesdir)) jip_coregistration = None desc = { "tr": _desc["RepetitionTime"], "warp_restrict": warp_restrict, "output": rundir } descs.append(desc) funcfiles.append(path) anatfiles.append(_anatfiles[0]) subjects.append(sid) outputs.append(rundir) trs.append(_desc["RepetitionTime"]) encdirs.append(warp_restrict) normfiles.append(jip_normalization) coregfiles.append(jip_coregistration) if len(_funcfiles) != len(descs): if verbose > 0: print("Skip session '{0}': no valid desc.".format(sesdir)) continue if sid not in dataset[name]: dataset[name][sid] = {} dataset[name][sid][ses] = { "anat": _anatfiles[0], "func": _funcfiles, "desc": descs } if verbose > 0: pprint(dataset) # Preprare inputs expected_size = len(subjects) if expected_size == 0: if verbose > 0: print("no data to process.") return None for name, item in (("outputs", outputs), ("funcfiles", funcfiles), ("anatfiles", anatfiles), ("subjects", subjects), ("trs", trs), ("encdirs", encdirs), ("normfiles", normfiles), ("coregfiles", coregfiles)): if item is None: continue if verbose > 0: print("[{0}] {1} : {2} ... {3}".format(name.upper(), len(item), item[0], item[-1])) if len(item) != expected_size: raise ValueError("BIDS dataset not aligned.") # Run preproc scriptdir = os.path.join(os.path.dirname(pypreclin.__file__), "scripts") script = os.path.join(scriptdir, "pypreclin_preproc_fmri") python_cmd = "python3" if not os.path.isfile(script): script = "pypreclin_preproc_fmri" python_cmd = None if simage is not None: script = "singularity run --home {0} ".format(shome or tempfile.gettempdir()) sbinds = sbinds or [] for path in sbinds: script += "--bind {0} ".format(path) script += simage python_cmd = None logdir = os.path.join(bidsdir, "derivatives", "logs") if not os.path.isdir(logdir): os.makedirs(logdir) logfile = os.path.join( logdir, "pypreclin_{0}.log".format(datetime.now().isoformat())) status, exitcodes = hopla( script, hopla_iterative_kwargs=["f", "a", "s", "o", "r", "WR", "N", "M"], f=funcfiles, a=anatfiles, s=subjects, o=outputs, r=trs, t=template, j=jipdir, resample=resample, W=True, WN=1, WR=encdirs, NA=anatorient, NF=funcorient, C=fslconfig, N=normfiles, M=coregfiles, A=auto, hopla_python_cmd=python_cmd, hopla_use_subprocess=True, hopla_verbose=1, hopla_cpus=njobs, hopla_logfile=logfile) return {"logfile": logfile}
# get infiles and filter them infiles = [] refiles = [] for s in subjects: infiles.extend(glob(os.path.join(args.basedir, '{}'.format(s), args.infile, '{}.nii.gz'.format(args.infile)))) refiles.extend(glob(os.path.join(args.basedir, '{}'.format(s), args.refile, '{}.nii.gz'.format(args.refile)))) outfiles = [i.replace(args.basedir, os.path.join(args.basedir, 'preprocess')) for i in infiles] outfiles = [i.replace(args.infile,'',1) for i in outfiles] outfiles = [os.path.splitext(os.path.splitext(i)[0])[0] for i in outfiles] outfiles = [os.path.join('{}'.format(os.path.dirname(i)), 'r{}.nii.gz'.format(os.path.basename(i))) for i in outfiles] if args.process: logfile = "{}/preprocess/log.txt".format(args.basedir) if not os.path.isdir(os.path.dirname(logfile)): os.makedirs(os.path.dirname(logfile)) # status, exitcodes = hopla( os.path.join('/volatile','frouin','radiomique_radiogenomique', 'register_pitie.py'), i=infiles, r=refiles, o=outfiles, hopla_iterative_kwargs=["i", "r", "o"], hopla_cpus=2, hopla_logfile=logfile, hopla_verbose=1)
for i, (hat, m2n) in enumerate(zip(hatlist, m2nlist)): hat_scan = ni.load(hat).get_affine() m2n_trans = np.asarray(pd.read_csv(m2n, sep=' ', header=None)) inion = np.asarray([100, 20, 80, 1]) nasion = np.asarray([100, 190, 80, 1]) # performe the xform in real (mm) world inion = m2n_trans.dot(hat_scan.dot(inion)) nasion = m2n_trans.dot(hat_scan.dot(nasion)) if not ((inion[1] < 0) and (nasion[1] > 0)): imagekeeps.append(imagefiles[i]) destkeeps.append(destfiles[i]) print imagekeeps, destkeeps # if args.process and len(imagekeeps) > 0: # rescue processing logfile = "{}/log/MniSegLog/rescuelog.txt".format(args.destdir) if not os.path.isdir(os.path.dirname(logfile)): os.makedirs(os.path.dirname(logfile)) cmd = os.path.join(os.getenv('HOME'), 'gits', 'scripts', '2017_rr', 'metastasis', 'm03_mni.py') status, exitcodes = hopla(cmd, i=imagekeeps, d=destkeeps, r=True, hopla_iterative_kwargs=["i", "d"], hopla_cpus=3, hopla_logfile=logfile, hopla_verbose=args.verbose)
os.makedirs(d) print "infiles (", len(infiles), ") [", infiles[0], ",...,", infiles[-1] print "infiles2 (", len(infiles), ") [", infiles2[0], ",...,", infiles2[-1] print "destfiles (", len( destfiles), ") [", destfiles[0], ",...,", destfiles[-1] print "maskfiles (", len( maskfiles), ") [", maskfiles[0], ",...,", maskfiles[-1] print "pvefiles (", len(pvefiles), ") [", pvefiles[0], ",...,", pvefiles[-1] if args.process: logfile = "{}/WSlog/WSlog.txt".format(args.destdir) if not os.path.isdir(os.path.dirname(logfile)): os.makedirs(os.path.dirname(logfile)) # cmd = os.path.join(os.getenv('HOME'), 'gits', 'scripts', '2017_rr', 'metastasis', 'm04_ws_std.py') status, exitcodes = hopla( cmd, #i1=infiles, i2=infiles2, t="FLAIR", d=destfiles, m=maskfiles, p=pvefiles, hopla_iterative_kwargs=["i2", "d", "m", "p"], hopla_cpus=3, hopla_logfile=logfile, hopla_verbose=args.verbose)
if __name__ == "__main__": # parsing parser = define_parser() args = parser.parse_args() logfile = os.path.join(args.outdir, 'logfile.txt') indirs = glob(os.path.join(args.indir, '*', 'Subj*')) if len(indirs) > 0: print("indirs {0} [{1} ... {2}])".format(len(indirs), indirs[0], indirs[-1])) else: print("no data") sys.exit(0) if args.process: # status, exitcodes = hopla(os.path.join(os.getenv('HOME'), "gits", "scripts", "2017_rr", "lpsnc", "import_lpsnc.py"), i=indirs, o=args.outdir, t=args.transcoding, s='c', hopla_iterative_kwargs=["i"], hopla_cpus=1, hopla_logfile=logfile, hopla_verbose=1)
def timeserie_to_reference(tfile, outdir, rindex=None, restrict_deformation=(1, 1, 1), rfile=None, njobs=1, clean_tmp=True): """ Register all the fMRI volumes to a reference volume identified by his index in the timeserie. The registration used is a non-linear regisration. Parameters ---------- tfile: str the file that containes the timeserie. outdir: str the destination folder. rindex: int, default None the reference volume index in the timeserie. restrict_deformation: 3-uplet restrict the deformation in the given axis. rfile: str, default None the reference volume. njobs: int, default 1 the desired number of parallel job during the registration. clean_tmp: bool, default True if set, clean the temporary results. Returns ------- resfile: str the registration result. """ # Check input index im = nibabel.load(tfile) array = im.get_data() if array.ndim != 4: raise ValueError("A timeserie (4d volume) is expected.") reference_image = None if rindex is not None: if rindex < 0 or rindex > array.shape[3]: raise ValueError( "Index '{0}' is out of bound considering the last dimension " "as the time dimension '{1}'.".format(rindex, array.shape)) elif rfile is not None: reference_image = rfile else: raise ValueError("You need to specify a reference file or index.") # Split the timeserie tmpdir = os.path.join(outdir, "tmp") if not os.path.isdir(tmpdir): os.mkdir(tmpdir) moving_images = [] outdirs = [] for i in range(array.shape[3]): _im = nibabel.Nifti1Image(array[..., i], im.affine) _outfile = os.path.join(tmpdir, str(i).zfill(4) + ".nii.gz") outdirs.append(os.path.join(tmpdir, str(i).zfill(4))) if not os.path.isdir(outdirs[-1]): os.mkdir(outdirs[-1]) nibabel.save(_im, _outfile) moving_images.append(_outfile) if reference_image is None and i == rindex: reference_image = _outfile # Start volume to volume non rigid registration scriptdir = os.path.join(os.path.dirname(pyconnectome.__file__), "scripts") script = os.path.join(scriptdir, "pyconnectome_ants_register") python_cmd = "python3" if not os.path.isfile(script): script = "pyconnectome_ants_register" python_cmd = None logfile = os.path.join(tmpdir, "log") if os.path.isfile(logfile): os.remove(logfile) status, exitcodes = hopla( script, b="/usr/lib/ants", o=outdirs, i=moving_images, r=reference_image, w=1, D=3, G=0.2, J=1, N=True, B=True, R=list(restrict_deformation), V=2, hopla_iterative_kwargs=["o", "i"], hopla_cpus=njobs, hopla_logfile=logfile, hopla_python_cmd=python_cmd, hopla_use_subprocess=True, hopla_verbose=1) if not (numpy.asarray(list(exitcodes.values())) == 0).all(): raise ValueError("The registration failed, check the log " "'{0}'.".format(logfile)) # Start timeserie concatenation timeserie = [] affine = None for path in outdirs: sid = os.path.basename(path) _im = nibabel.load(os.path.join( path, "ants_2WarpToTemplate_{0}.nii.gz".format(sid))) if affine is None: affine = _im.affine elif not numpy.allclose(affine, im.affine, atol=1e-3): raise ValueError("Affine matrices must be the same: {0} - " "{1}.".format(outdirs[0], path)) data = _im.get_data() data.shape += (1, ) timeserie.append(data) registered_array = numpy.concatenate(timeserie, axis=3) _im = nibabel.Nifti1Image(registered_array, affine) resfile = os.path.join(outdir, "ants_WarpToTemplate.nii.gz") nibabel.save(_im, resfile) # Clean temporary files if requested if clean_tmp: shutil.rmtree(tmpdir, ignore_errors=True) return resfile
if args.process: logfile = "/volatile/frouin/radiomique_radiogenomique/base2/log.txt" # sids = [] protocols = [] dcmdirs = [] for s in dcm2niiparameter: for i, o, p in zip(dcm2niiparameter[s]['inpath'], dcm2niiparameter[s]['outpath'], dcm2niiparameter[s]['proto']): sids.append(s) protocols.append(p) dcmdirs.append( os.path.join('/volatile/frouin/radiomique_radiogenomique/', i)) # status, exitcodes = hopla( os.path.join(os.getenv('HOME'), "gits", "pydcmio", "pydcmio", "scripts", "pydcmio_dicom2nifti"), s=sids, p=protocols, d=dcmdirs, o="/volatile/frouin/radiomique_radiogenomique/base2/raw", t=True, r="/volatile/frouin/radiomique_radiogenomique/base2/transcoding.json", x=True, hopla_iterative_kwargs=["s", "p", "d"], hopla_cpus=2, hopla_logfile=logfile, hopla_verbose=1)
if not os.path.isdir(os.path.dirname(logfile)): os.makedirs(os.path.dirname(logfile)) # # cmd = os.path.join(os.getenv('HOME'), 'gits', # 'scripts', '2017_rr', 'metastasis', # 'm50_radiomic_feat_extraction.py') # status, exitcodes = hopla(cmd, # '--param', PARAM_FILE, # '--format', 'json', # '--habitat', 'edema', # '--out', destdirs, # '--image', imafiles, # '--segment', maskfiles, # hopla_iterative_kwargs=["i", "s", "-o"], # hopla_cpu=args.cpus, # hopla_logfile=logfile, # hopla_verbose=args.verbose) status, exitcodes = hopla(os.path.join(os.getenv('HOME'), 'gits', 'scripts', '2017_rr', 'metastasis', 'm50_radiomic_feat_extraction.py'), p=PARAM_FILE, f='json', a='all', o=destdirs, i=imafiles, s=maskfiles, hopla_iterative_kwargs=["i", "s", "o"], hopla_cpus=args.cpus, hopla_logfile=logfile, hopla_verbose=args.verbose)
log_msg = [] indir = glob(os.path.join(args.dcmdir, '*', 'Subj*')) outdir = [] for i in indir: h, s = os.path.split(i) _, n = os.path.split(h) outdir.append(os.path.join(args.outdir, n, s)) if not args.process: os.makedirs(outdir[0]) split_series(indir[0], outdir[0]) else: if args.process: logfile = os.path.join(os.path.realpath(args.outdir), "log.txt") # for i in outdir: os.makedirs(i) # status, exitcodes = hopla(os.path.join(os.getenv('HOME'), "gits", "pydcmio", "pydcmio", "scripts", "pydcmio_splitseries"), i=indir, o=outdir, v=0, hopla_iterative_kwargs=["i", "o"], hopla_cpus=2, hopla_logfile=logfile, hopla_verbose=1)