def test_custom_call_fmt(path, local_file):
ds = Dataset(path).create()
subds = ds.create('sub')
# plug in a proper singularity image
subds.containers_add(
'mycontainer',
url=get_local_file_url(op.join(local_file, 'some_container.img')),
image='righthere',
call_fmt='echo image={img} cmd={cmd} img_dspath={img_dspath} '
# and environment variable being set/propagated by default
'name=$DATALAD_CONTAINER_NAME'
)
ds.save() # record the effect in super-dataset
# Running should work fine either withing sub or within super
with swallow_outputs() as cmo:
subds.containers_run('XXX', container_name='mycontainer')
assert_in('image=righthere cmd=XXX img_dspath=. name=mycontainer', cmo.out)
with swallow_outputs() as cmo:
ds.containers_run('XXX', container_name='sub/mycontainer')
assert_in('image=sub/righthere cmd=XXX img_dspath=sub', cmo.out)
# Test within subdirectory of the super-dataset
subdir = op.join(ds.path, 'subdir')
os.mkdir(subdir)
with chpwd(subdir):
with swallow_outputs() as cmo:
containers_run('XXX', container_name='sub/mycontainer')
assert_in('image=../sub/righthere cmd=XXX img_dspath=../sub', cmo.out)
def test_containers_run(self, path):
if self.image_existed:
raise SkipTest(
"Not pulling with containers-run due to existing image: {}".
format(self.image_name))
from datalad.api import Dataset
ds = Dataset(path).create(force=True)
ds.save(path="foo")
ds.containers_add("bb", url="dhub://" + self.image_name)
with swallow_outputs() as out:
ds.containers_run(["cat", "foo"], container_name="bb")
assert_in("content", out.out)
def test_docker(path): # Singularity's "docker://" scheme.
ds = Dataset(path).create()
ds.containers_add(
"bb",
url=("docker://busybox@sha256:"
"7964ad52e396a6e045c39b5a44438424ac52e12e4d5a25d94895f2058cb863a0"
))
img = op.join(ds.path, ".datalad", "environments", "bb", "image")
assert_result_count(ds.containers_list(), 1, path=img, name="bb")
ok_clean_git(path)
with swallow_outputs():
ds.containers_run(["ls", "/singularity"])
def test_container_files(path):
ds = Dataset(path).create()
# plug in a proper singularity image
ds.containers_add(
'mycontainer',
url=testimg_url,
image='righthere',
# the next one is auto-guessed
#call_fmt='singularity exec {img} {cmd}'
)
assert_result_count(
ds.containers_list(), 1,
path=op.join(ds.path, 'righthere'),
name='mycontainer',
updateurl=testimg_url)
ok_clean_git(path)
# now we can run stuff in the container
# and because there is just one, we don't even have to name the container
res = ds.containers_run(['dir'] if on_windows else ['ls'])
# container becomes an 'input' for `run` -> get request, but "notneeded"
assert_result_count(
res, 1, action='get', status='notneeded',
path=op.join(ds.path, 'righthere'), type='file')
# this command changed nothing
assert_result_count(
res, 1, action='add', status='notneeded', path=ds.path, type='dataset')
def test_run_unknown_cmdexec_placeholder(path):
ds = Dataset(path).create(force=True)
ds.containers_add("i", image="i.img", call_fmt="{youdontknowme}")
assert_result_count(ds.containers_run("doesn't matter",
on_failure="ignore"),
1,
path=ds.path,
action="run",
status="error")
def test_run_mispecified(path):
ds = Dataset(path).create(force=True)
ds.save(path=["dummy0.img", "dummy1.img"])
ok_clean_git(path)
# Abort if no containers exist.
with assert_raises(ValueError) as cm:
ds.containers_run("doesn't matter")
assert_in("No known containers", text_type(cm.exception))
# Abort if more than one container exists but no container name is
# specified.
ds.containers_add("d0", image="dummy0.img")
ds.containers_add("d1", image="dummy0.img")
with assert_raises(ValueError) as cm:
ds.containers_run("doesn't matter")
assert_in("explicitly specify container", text_type(cm.exception))
# Abort if unknown container is specified.
with assert_raises(ValueError) as cm:
ds.containers_run("doesn't matter", container_name="ghost")
assert_in("Container selection impossible", text_type(cm.exception))
dataset.containers_run(
[
'heudiconv',
# XXX absolute path will make rerun on other
# system impossible -- hard to avoid
# TODO: from toolbox? config?
'-f',
heuristic_path,
# leaves identifying info in run record
'-s',
subject,
'-c',
'dcm2niix',
# TODO decide on the fate of .heudiconv/
# but ATM we need to (re)move it:
# https://github.com/nipy/heudiconv/issues/196
'-o',
rel_trash_path,
'-b',
'-a',
'{dspath}',
'-l',
'',
# avoid glory details provided by dcmstack,
# we have them in the aggregated DICOM
# metadata already
'--minmeta',
'--files',
location
],
sidecar=anonymize,
# TODO: This doesn't work! ... Well, it does. What was the problem?
container_name=op.relpath(
op.join(op.dirname(op.realpath(__file__)), "heudiconv.simg"),
dataset.path),
explicit=True,
expand="both",
inputs=inputs,
# Note: Outputs determination isn't good yet. We need a way to
# figure what exactly heudiconv produced. This is different from
# other toolbox-procedures due to our "injection heuristic".
outputs=outputs,
message="[HIRNI] Convert DICOM data for subject {}"
"".format(subject),
)
def test_container_files(path, super_path):
raise SkipTest('SingularityHub is gone for now')
ds = Dataset(path).create()
cmd = ['dir'] if on_windows else ['ls']
# plug in a proper singularity image
ds.containers_add(
'mycontainer',
url=testimg_url,
image='righthere',
# the next one is auto-guessed
#call_fmt='singularity exec {img} {cmd}'
)
assert_result_count(ds.containers_list(),
1,
path=op.join(ds.path, 'righthere'),
name='mycontainer')
ok_clean_git(path)
def assert_no_change(res, path):
# this command changed nothing
#
# Avoid specifying the action because it will change from "add" to
# "save" in DataLad v0.12.
assert_result_count(res,
1,
status='notneeded',
path=path,
type='dataset')
# now we can run stuff in the container
# and because there is just one, we don't even have to name the container
res = ds.containers_run(cmd)
# container becomes an 'input' for `run` -> get request, but "notneeded"
assert_result_count(res,
1,
action='get',
status='notneeded',
path=op.join(ds.path, 'righthere'),
type='file')
assert_no_change(res, ds.path)
# same thing as we specify the container by its name:
res = ds.containers_run(cmd, container_name='mycontainer')
# container becomes an 'input' for `run` -> get request, but "notneeded"
assert_result_count(res,
1,
action='get',
status='notneeded',
path=op.join(ds.path, 'righthere'),
type='file')
assert_no_change(res, ds.path)
# we can also specify the container by its path:
res = ds.containers_run(cmd, container_name=op.join(ds.path, 'righthere'))
# container becomes an 'input' for `run` -> get request, but "notneeded"
assert_result_count(res,
1,
action='get',
status='notneeded',
path=op.join(ds.path, 'righthere'),
type='file')
assert_no_change(res, ds.path)
# Now, test the same thing, but with this dataset being a subdataset of
# another one:
super_ds = Dataset(super_path).create()
super_ds.install("sub", source=path)
# When running, we don't discover containers in subdatasets
with assert_raises(ValueError) as cm:
super_ds.containers_run(cmd)
assert_in("No known containers", text_type(cm.exception))
# ... unless we need to specify the name
res = super_ds.containers_run(cmd, container_name="sub/mycontainer")
# container becomes an 'input' for `run` -> get request (needed this time)
assert_result_count(res,
1,
action='get',
status='ok',
path=op.join(super_ds.path, 'sub', 'righthere'),
type='file')
assert_no_change(res, super_ds.path)
# Print all available help.
#
# --version
# Print version information and exit.
# TODO: Expose all the parameters (and prob. set up argparse for that)
if __name__ == '__main__':
import sys
import os.path as op
from datalad.api import Dataset
from datalad.config import anything2bool
dataset = Dataset(sys.argv[1])
anonymize = anything2bool(sys.argv[2])
files = sys.argv[3:]
dataset.containers_run(
['mridefacer', '--apply'] + files,
sidecar=anonymize,
container_name=op.relpath(
op.join(op.dirname(op.realpath(__file__)), "mridefacer.simg"),
dataset.path),
inputs=files,
outputs=[dataset.path],
message="[HIRNI] Deface MRI data"
)
import os.path as op
from datalad.api import Dataset
dataset = Dataset(sys.argv[1])
params = []
delimiter = None
for i, p in enumerate(sys.argv[2:]):
if p == '--':
delimiter = i
break
else:
params.append(p)
if not delimiter:
print("Missing '-- [FILES]'")
sys.exit(1)
files = sys.argv[delimiter + 3:]
# TODO: How to get those into a single commit? Script in container or
# call a loop directly?
for f in files:
dataset.containers_run(
['fslroi', f, f] + params,
sidecar=False,
container_name=op.relpath(
op.join(op.dirname(op.realpath(__file__)), "fsl.simg"),
dataset.path),
inputs=[f],
outputs=[f],
)
