def test_local_execution(self):
""" Test local execution.
"""
logfile = tempfile.NamedTemporaryFile(suffix=".log").name
hopla(self.script, d=[self.demodir, self.demodir], l=[2, 3],
fbreak=[False, True], verbose=0,
hopla_iterative_kwargs=["d", "fbreak"], hopla_logfile=logfile,
hopla_cpus=4, hopla_optional=["fbreak", "verbose"])
os.remove(logfile)
def test_cluster_execution(self):
""" Test cluster execution.
"""
logfile = tempfile.NamedTemporaryFile(suffix=".log").name
logdir = tempfile.mkdtemp()
hopla(self.script, d=[self.demodir, self.demodir], l=[2, 3],
fbreak=[False, True], verbose=0,
hopla_iterative_kwargs=["d", "fbreak"], hopla_logfile=logfile,
hopla_cpus=4, hopla_optional=["fbreak", "verbose"],
hopla_cluster=True, hopla_cluster_logdir=logdir,
hopla_cluster_queue="DUMMY")
shutil.rmtree(logdir)
os.remove(logfile)
def test_normal_execution(self, mock_scheduler):
""" Test normal execution.
"""
# Local execution
for fbreak in (True, False):
hopla(self.script, d=["dir1"], l=[2, 3], fbreak=fbreak,
verbose=[0], hopla_iterative_kwargs=["d", "verbose"],
hopla_optional=["fbreak", "verbose"])
print(mock_scheduler.call_args_list[-1][1]["commands"])
generated_commands = mock_scheduler.call_args_list[-1][1][
"commands"]
expected_commands = [
[self.script, "-d", "dir1", "--verbose", "0", "-l", "2", "3"]]
if fbreak:
expected_commands[0].insert(5, "--fbreak")
self.assertEqual(sorted(generated_commands),
sorted(expected_commands))
for optional in (None, "some_string"):
hopla(self.script, d=["dir1"], l=[2, 3], o=optional,
verbose=[0], hopla_iterative_kwargs=["d", "verbose"],
hopla_optional=["fbreak", "verbose"])
generated_commands = mock_scheduler.call_args_list[-1][1][
"commands"]
expected_commands = [
[self.script, "-d", "dir1", "--verbose", "0", "-l", "2", "3"]]
if optional is not None:
expected_commands[0].extend(["-o", optional])
self.assertEqual(sorted(generated_commands),
sorted(expected_commands))
# Local execution with boolean iter
for fbreak in (True, False):
hopla(self.script, d=["dir1"], l=[2, 3], fbreak=[fbreak],
verbose=0, hopla_iterative_kwargs=["d", "fbreak"],
hopla_optional=["fbreak", "verbose"])
generated_commands = mock_scheduler.call_args_list[-1][1][
"commands"]
expected_commands = [
[self.script, "-d", "dir1", "-l", "2", "3", "--verbose", "0"]]
if fbreak:
expected_commands[0].insert(3, "--fbreak")
self.assertEqual(generated_commands, expected_commands)
for optional in (None, "some_string"):
hopla(self.script, d=["dir1"], l=[2, 3], o=[optional],
verbose=0, hopla_iterative_kwargs=["d", "o"],
hopla_optional=["fbreak", "verbose"])
generated_commands = mock_scheduler.call_args_list[-1][1][
"commands"]
expected_commands = [
[self.script, "-d", "dir1", "-l", "2", "3", "--verbose", "0"]]
if optional is not None:
expected_commands[0].insert(3, "-o")
expected_commands[0].insert(4, optional)
self.assertEqual(sorted(generated_commands),
sorted(expected_commands))
from pprint import pprint
# Hopla import
import hopla as root
from hopla.converter import hopla
# Define script parameters
apath = os.path.join(os.path.abspath(os.path.dirname(root.__file__)), "demo")
script = os.path.join(apath, "my_ls_script.py")
cluster_logdir = os.path.join(apath, "pbs_logs")
# Local execution
print("-------", "local execution")
status, exitcodes = hopla(
script, d=[apath, apath, apath], b=False, v=0, l=[1, 2],
hopla_iterative_kwargs=["d"], hopla_verbose=1, hopla_cpus=10)
pprint(exitcodes)
# Local execution with optional arguments
print("-------", "local execution with optional arguments")
status, exitcodes = hopla(
script, dir=[apath, apath, apath], b=False, verbose=1, mylist=[1, 2],
hopla_iterative_kwargs=["dir"], hopla_verbose=1, hopla_cpus=10,
hopla_optional=["dir", "verbose", "mylist"])
pprint(exitcodes)
# Local execution with boolean iter
print("-------", "local execution with boolean iter")
status, exitcodes = hopla(
script, dir=[apath, apath, apath], b=[False, True, False], verbose=1,
def preproc_multi(bidsdir,
template,
jipdir,
fslconfig,
auto,
resample,
anatorient="RAS",
funcorient="RAS",
njobs=1,
simage=None,
shome=None,
sbinds=None,
verbose=0):
""" Perform the FMRI preprocessing on a BIDS organized directory in
parallel (without FUGUE or TOPUP).
This function can be called with a singularity image that contains all the
required software.
If a 'jip_trf' directory is available in the session directory, the code
will use the available JIP transformation.
Parameters
bidsdir: str
the BIDS organized directory.
template: str
the path to the template in RAS coordiante system.
jipdir: str
the jip software binary path.
fslconfig: str
the FSL configuration script.
auto: bool
control the JIP window with the script.
resample: bool
if set resample the input template to fit the anatomical image.
anatorient: str, default "RAS"
the input anatomical image orientation.
funcorient: str, default "RAS"
the input functional image orientation.
njobs: int, default 1
the number of parallel jobs.
simage: simg, default None
a singularity image.
shome: str, default None
a home directory for the singularity image.
sbinds: str, default None
shared directories for the singularity image.
verbose: int, default 0
control the verbosity level.
"""
# TODO: remove when all controls available in pypipe
if not isinstance(auto, bool):
auto = eval(auto)
resample = eval(resample)
sbinds = eval(sbinds)
# Parse the BIDS directory
desc_file = os.path.join(bidsdir, "dataset_description.json")
if not os.path.isfile(desc_file):
raise ValueError(
"Expect '{0}' in a BIDS organized directory.".format(desc_file))
with open(desc_file, "rt") as of:
desc = json.load(of)
name = desc["Name"]
if verbose > 0:
print("Processing dataset '{0}'...".format(name))
dataset = {name: {}}
funcfiles, anatfiles, subjects, outputs, trs, encdirs, normfiles, coregfiles = [
[] for _ in range(8)
]
outdir = os.path.join(bidsdir, "derivatives", "preproc")
for sesdir in glob.glob(os.path.join(bidsdir, "sub-*", "ses-*")):
split = sesdir.split(os.sep)
sid, ses = split[-2:]
_anatfiles = glob.glob(os.path.join(sesdir, "anat", "sub-*T1w.nii.gz"))
_funcfiles = glob.glob(os.path.join(sesdir, "func",
"sub-*bold.nii.gz"))
if len(_anatfiles) != 1:
if verbose > 0:
print("Skip session '{0}': no valid anat.".format(sesdir))
continue
if len(_funcfiles) == 0:
if verbose > 0:
print("Skip session '{0}': no valid func.".format(sesdir))
continue
descs = []
for path in _funcfiles:
runs = re.findall(".*_(run-[0-9]*)_.*", path)
if len(runs) != 1:
if verbose > 0:
print("Skip path '{0}': not valid name.".format(path))
continue
sesdir = os.path.join(outdir, sid, ses)
rundir = os.path.join(sesdir, runs[0])
if not os.path.isdir(rundir):
os.makedirs(rundir)
desc_file = path.replace(".nii.gz", ".json")
if not os.path.isfile(desc_file):
break
with open(desc_file, "rt") as of:
_desc = json.load(of)
if _desc["PhaseEncodingDirection"].startswith("i"):
warp_restrict = [1, 0, 0]
elif _desc["PhaseEncodingDirection"].startswith("j"):
warp_restrict = [0, 1, 0]
elif _desc["PhaseEncodingDirection"].startswith("k"):
warp_restrict = [0, 0, 1]
else:
raise ValueError(
"Unknown encode phase direction : {0}...".format(
_desc["PhaseEncodingDirection"]))
jip_normalization = os.path.join(sesdir, "jip_trf",
"Normalization", "align.com")
if not os.path.isfile(jip_normalization):
if verbose > 0:
print("No JIP normalization align.com file found "
"in {0}.".format(sesdir))
jip_normalization = None
jip_coregistration = os.path.join(sesdir, "jip_trf",
"Coregistration", "align.com")
if not os.path.isfile(jip_coregistration):
if verbose > 0:
print("No JIP coregistration align.com file found "
"in {0}.".format(sesdir))
jip_coregistration = None
desc = {
"tr": _desc["RepetitionTime"],
"warp_restrict": warp_restrict,
"output": rundir
}
descs.append(desc)
funcfiles.append(path)
anatfiles.append(_anatfiles[0])
subjects.append(sid)
outputs.append(rundir)
trs.append(_desc["RepetitionTime"])
encdirs.append(warp_restrict)
normfiles.append(jip_normalization)
coregfiles.append(jip_coregistration)
if len(_funcfiles) != len(descs):
if verbose > 0:
print("Skip session '{0}': no valid desc.".format(sesdir))
continue
if sid not in dataset[name]:
dataset[name][sid] = {}
dataset[name][sid][ses] = {
"anat": _anatfiles[0],
"func": _funcfiles,
"desc": descs
}
if verbose > 0:
pprint(dataset)
# Preprare inputs
expected_size = len(subjects)
if expected_size == 0:
if verbose > 0:
print("no data to process.")
return None
for name, item in (("outputs", outputs), ("funcfiles", funcfiles),
("anatfiles", anatfiles), ("subjects", subjects),
("trs", trs), ("encdirs", encdirs),
("normfiles", normfiles), ("coregfiles", coregfiles)):
if item is None:
continue
if verbose > 0:
print("[{0}] {1} : {2} ... {3}".format(name.upper(), len(item),
item[0], item[-1]))
if len(item) != expected_size:
raise ValueError("BIDS dataset not aligned.")
# Run preproc
scriptdir = os.path.join(os.path.dirname(pypreclin.__file__), "scripts")
script = os.path.join(scriptdir, "pypreclin_preproc_fmri")
python_cmd = "python3"
if not os.path.isfile(script):
script = "pypreclin_preproc_fmri"
python_cmd = None
if simage is not None:
script = "singularity run --home {0} ".format(shome
or tempfile.gettempdir())
sbinds = sbinds or []
for path in sbinds:
script += "--bind {0} ".format(path)
script += simage
python_cmd = None
logdir = os.path.join(bidsdir, "derivatives", "logs")
if not os.path.isdir(logdir):
os.makedirs(logdir)
logfile = os.path.join(
logdir, "pypreclin_{0}.log".format(datetime.now().isoformat()))
status, exitcodes = hopla(
script,
hopla_iterative_kwargs=["f", "a", "s", "o", "r", "WR", "N", "M"],
f=funcfiles,
a=anatfiles,
s=subjects,
o=outputs,
r=trs,
t=template,
j=jipdir,
resample=resample,
W=True,
WN=1,
WR=encdirs,
NA=anatorient,
NF=funcorient,
C=fslconfig,
N=normfiles,
M=coregfiles,
A=auto,
hopla_python_cmd=python_cmd,
hopla_use_subprocess=True,
hopla_verbose=1,
hopla_cpus=njobs,
hopla_logfile=logfile)
return {"logfile": logfile}
# get infiles and filter them
infiles = []
refiles = []
for s in subjects:
infiles.extend(glob(os.path.join(args.basedir, '{}'.format(s),
args.infile, '{}.nii.gz'.format(args.infile))))
refiles.extend(glob(os.path.join(args.basedir, '{}'.format(s),
args.refile, '{}.nii.gz'.format(args.refile))))
outfiles = [i.replace(args.basedir, os.path.join(args.basedir, 'preprocess')) for i in infiles]
outfiles = [i.replace(args.infile,'',1) for i in outfiles]
outfiles = [os.path.splitext(os.path.splitext(i)[0])[0] for i in outfiles]
outfiles = [os.path.join('{}'.format(os.path.dirname(i)),
'r{}.nii.gz'.format(os.path.basename(i)))
for i in outfiles]
if args.process:
logfile = "{}/preprocess/log.txt".format(args.basedir)
if not os.path.isdir(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
#
status, exitcodes = hopla(
os.path.join('/volatile','frouin','radiomique_radiogenomique',
'register_pitie.py'),
i=infiles,
r=refiles,
o=outfiles,
hopla_iterative_kwargs=["i", "r", "o"],
hopla_cpus=2,
hopla_logfile=logfile,
hopla_verbose=1)
for i, (hat, m2n) in enumerate(zip(hatlist, m2nlist)):
hat_scan = ni.load(hat).get_affine()
m2n_trans = np.asarray(pd.read_csv(m2n, sep=' ', header=None))
inion = np.asarray([100, 20, 80, 1])
nasion = np.asarray([100, 190, 80, 1])
# performe the xform in real (mm) world
inion = m2n_trans.dot(hat_scan.dot(inion))
nasion = m2n_trans.dot(hat_scan.dot(nasion))
if not ((inion[1] < 0) and (nasion[1] > 0)):
imagekeeps.append(imagefiles[i])
destkeeps.append(destfiles[i])
print imagekeeps, destkeeps
#
if args.process and len(imagekeeps) > 0:
# rescue processing
logfile = "{}/log/MniSegLog/rescuelog.txt".format(args.destdir)
if not os.path.isdir(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
cmd = os.path.join(os.getenv('HOME'), 'gits', 'scripts', '2017_rr',
'metastasis', 'm03_mni.py')
status, exitcodes = hopla(cmd,
i=imagekeeps,
d=destkeeps,
r=True,
hopla_iterative_kwargs=["i", "d"],
hopla_cpus=3,
hopla_logfile=logfile,
hopla_verbose=args.verbose)
os.makedirs(d)
print "infiles (", len(infiles), ") [", infiles[0], ",...,", infiles[-1]
print "infiles2 (", len(infiles), ") [", infiles2[0], ",...,", infiles2[-1]
print "destfiles (", len(
destfiles), ") [", destfiles[0], ",...,", destfiles[-1]
print "maskfiles (", len(
maskfiles), ") [", maskfiles[0], ",...,", maskfiles[-1]
print "pvefiles (", len(pvefiles), ") [", pvefiles[0], ",...,", pvefiles[-1]
if args.process:
logfile = "{}/WSlog/WSlog.txt".format(args.destdir)
if not os.path.isdir(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
#
cmd = os.path.join(os.getenv('HOME'), 'gits', 'scripts', '2017_rr',
'metastasis', 'm04_ws_std.py')
status, exitcodes = hopla(
cmd,
#i1=infiles,
i2=infiles2,
t="FLAIR",
d=destfiles,
m=maskfiles,
p=pvefiles,
hopla_iterative_kwargs=["i2", "d", "m", "p"],
hopla_cpus=3,
hopla_logfile=logfile,
hopla_verbose=args.verbose)
if __name__ == "__main__":
# parsing
parser = define_parser()
args = parser.parse_args()
logfile = os.path.join(args.outdir, 'logfile.txt')
indirs = glob(os.path.join(args.indir, '*', 'Subj*'))
if len(indirs) > 0:
print("indirs {0} [{1} ... {2}])".format(len(indirs), indirs[0],
indirs[-1]))
else:
print("no data")
sys.exit(0)
if args.process:
#
status, exitcodes = hopla(os.path.join(os.getenv('HOME'), "gits",
"scripts", "2017_rr", "lpsnc",
"import_lpsnc.py"),
i=indirs,
o=args.outdir,
t=args.transcoding,
s='c',
hopla_iterative_kwargs=["i"],
hopla_cpus=1,
hopla_logfile=logfile,
hopla_verbose=1)
def timeserie_to_reference(tfile, outdir, rindex=None,
restrict_deformation=(1, 1, 1), rfile=None, njobs=1,
clean_tmp=True):
""" Register all the fMRI volumes to a reference volume identified by his
index in the timeserie.
The registration used is a non-linear regisration.
Parameters
----------
tfile: str
the file that containes the timeserie.
outdir: str
the destination folder.
rindex: int, default None
the reference volume index in the timeserie.
restrict_deformation: 3-uplet
restrict the deformation in the given axis.
rfile: str, default None
the reference volume.
njobs: int, default 1
the desired number of parallel job during the registration.
clean_tmp: bool, default True
if set, clean the temporary results.
Returns
-------
resfile: str
the registration result.
"""
# Check input index
im = nibabel.load(tfile)
array = im.get_data()
if array.ndim != 4:
raise ValueError("A timeserie (4d volume) is expected.")
reference_image = None
if rindex is not None:
if rindex < 0 or rindex > array.shape[3]:
raise ValueError(
"Index '{0}' is out of bound considering the last dimension "
"as the time dimension '{1}'.".format(rindex, array.shape))
elif rfile is not None:
reference_image = rfile
else:
raise ValueError("You need to specify a reference file or index.")
# Split the timeserie
tmpdir = os.path.join(outdir, "tmp")
if not os.path.isdir(tmpdir):
os.mkdir(tmpdir)
moving_images = []
outdirs = []
for i in range(array.shape[3]):
_im = nibabel.Nifti1Image(array[..., i], im.affine)
_outfile = os.path.join(tmpdir, str(i).zfill(4) + ".nii.gz")
outdirs.append(os.path.join(tmpdir, str(i).zfill(4)))
if not os.path.isdir(outdirs[-1]):
os.mkdir(outdirs[-1])
nibabel.save(_im, _outfile)
moving_images.append(_outfile)
if reference_image is None and i == rindex:
reference_image = _outfile
# Start volume to volume non rigid registration
scriptdir = os.path.join(os.path.dirname(pyconnectome.__file__), "scripts")
script = os.path.join(scriptdir, "pyconnectome_ants_register")
python_cmd = "python3"
if not os.path.isfile(script):
script = "pyconnectome_ants_register"
python_cmd = None
logfile = os.path.join(tmpdir, "log")
if os.path.isfile(logfile):
os.remove(logfile)
status, exitcodes = hopla(
script,
b="/usr/lib/ants",
o=outdirs,
i=moving_images,
r=reference_image,
w=1,
D=3,
G=0.2,
J=1,
N=True,
B=True,
R=list(restrict_deformation),
V=2,
hopla_iterative_kwargs=["o", "i"],
hopla_cpus=njobs,
hopla_logfile=logfile,
hopla_python_cmd=python_cmd,
hopla_use_subprocess=True,
hopla_verbose=1)
if not (numpy.asarray(list(exitcodes.values())) == 0).all():
raise ValueError("The registration failed, check the log "
"'{0}'.".format(logfile))
# Start timeserie concatenation
timeserie = []
affine = None
for path in outdirs:
sid = os.path.basename(path)
_im = nibabel.load(os.path.join(
path, "ants_2WarpToTemplate_{0}.nii.gz".format(sid)))
if affine is None:
affine = _im.affine
elif not numpy.allclose(affine, im.affine, atol=1e-3):
raise ValueError("Affine matrices must be the same: {0} - "
"{1}.".format(outdirs[0], path))
data = _im.get_data()
data.shape += (1, )
timeserie.append(data)
registered_array = numpy.concatenate(timeserie, axis=3)
_im = nibabel.Nifti1Image(registered_array, affine)
resfile = os.path.join(outdir, "ants_WarpToTemplate.nii.gz")
nibabel.save(_im, resfile)
# Clean temporary files if requested
if clean_tmp:
shutil.rmtree(tmpdir, ignore_errors=True)
return resfile
if args.process:
logfile = "/volatile/frouin/radiomique_radiogenomique/base2/log.txt"
#
sids = []
protocols = []
dcmdirs = []
for s in dcm2niiparameter:
for i, o, p in zip(dcm2niiparameter[s]['inpath'],
dcm2niiparameter[s]['outpath'],
dcm2niiparameter[s]['proto']):
sids.append(s)
protocols.append(p)
dcmdirs.append(
os.path.join('/volatile/frouin/radiomique_radiogenomique/', i))
#
status, exitcodes = hopla(
os.path.join(os.getenv('HOME'), "gits", "pydcmio", "pydcmio",
"scripts", "pydcmio_dicom2nifti"),
s=sids,
p=protocols,
d=dcmdirs,
o="/volatile/frouin/radiomique_radiogenomique/base2/raw",
t=True,
r="/volatile/frouin/radiomique_radiogenomique/base2/transcoding.json",
x=True,
hopla_iterative_kwargs=["s", "p", "d"],
hopla_cpus=2,
hopla_logfile=logfile,
hopla_verbose=1)
if not os.path.isdir(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
#
# cmd = os.path.join(os.getenv('HOME'), 'gits',
# 'scripts', '2017_rr', 'metastasis',
# 'm50_radiomic_feat_extraction.py')
# status, exitcodes = hopla(cmd,
# '--param', PARAM_FILE,
# '--format', 'json',
# '--habitat', 'edema',
# '--out', destdirs,
# '--image', imafiles,
# '--segment', maskfiles,
# hopla_iterative_kwargs=["i", "s", "-o"],
# hopla_cpu=args.cpus,
# hopla_logfile=logfile,
# hopla_verbose=args.verbose)
status, exitcodes = hopla(os.path.join(os.getenv('HOME'), 'gits',
'scripts', '2017_rr', 'metastasis',
'm50_radiomic_feat_extraction.py'),
p=PARAM_FILE,
f='json',
a='all',
o=destdirs,
i=imafiles,
s=maskfiles,
hopla_iterative_kwargs=["i", "s", "o"],
hopla_cpus=args.cpus,
hopla_logfile=logfile,
hopla_verbose=args.verbose)
log_msg = []
indir = glob(os.path.join(args.dcmdir, '*', 'Subj*'))
outdir = []
for i in indir:
h, s = os.path.split(i)
_, n = os.path.split(h)
outdir.append(os.path.join(args.outdir, n, s))
if not args.process:
os.makedirs(outdir[0])
split_series(indir[0], outdir[0])
else:
if args.process:
logfile = os.path.join(os.path.realpath(args.outdir), "log.txt")
#
for i in outdir:
os.makedirs(i)
#
status, exitcodes = hopla(os.path.join(os.getenv('HOME'), "gits",
"pydcmio", "pydcmio",
"scripts",
"pydcmio_splitseries"),
i=indir,
o=outdir,
v=0,
hopla_iterative_kwargs=["i", "o"],
hopla_cpus=2,
hopla_logfile=logfile,
hopla_verbose=1)
