def test_run_no_explicit_dataset(path):
raise SkipTest('SingularityHub is gone for now')
ds = Dataset(path).create(force=True)
ds.save()
ds.containers_add("deb",
url=testimg_url,
call_fmt="singularity exec {img} {cmd}")
# When no explicit dataset is given, paths are interpreted as relative to
# the current working directory.
# From top-level directory.
with chpwd(path):
containers_run("cat {inputs[0]} {inputs[0]} >doubled",
inputs=[op.join("subdir", "in")],
outputs=["doubled"])
ok_file_has_content(op.join(path, "doubled"), "innardsinnards")
# From under a subdirectory.
subdir = op.join(ds.path, "subdir")
with chpwd(subdir):
containers_run("cat {inputs[0]} {inputs[0]} >doubled",
inputs=["in"],
outputs=["doubled"])
ok_file_has_content(op.join(subdir, "doubled"), "innardsinnards")
def test_container_files(path):
ds = Dataset(path).create()
# plug in a proper singularity image
ds.containers_add(
'mycontainer',
url=testimg_url,
image='righthere',
# the next one is auto-guessed
#call_fmt='singularity exec {img} {cmd}'
)
assert_result_count(
ds.containers_list(), 1,
path=op.join(ds.path, 'righthere'),
name='mycontainer',
updateurl=testimg_url)
ok_clean_git(path)
# now we can run stuff in the container
# and because there is just one, we don't even have to name the container
res = ds.containers_run(['dir'] if on_windows else ['ls'])
# container becomes an 'input' for `run` -> get request, but "notneeded"
assert_result_count(
res, 1, action='get', status='notneeded',
path=op.join(ds.path, 'righthere'), type='file')
# this command changed nothing
assert_result_count(
res, 1, action='add', status='notneeded', path=ds.path, type='dataset')
def test_find_containers(path):
ds = Dataset(path).create(force=True)
ds.save(path=[op.join('sub', 'i.img')], message="dummy container")
ds.containers_add("i", image=op.join('sub', 'i.img'))
ok_clean_git(path)
# find the only one
res = find_container(ds)
assert_is_instance(res, dict)
assert_result_count([res],
1,
status="ok",
path=op.join(ds.path, "sub", "i.img"))
# find by name
res = find_container(ds, "i")
assert_is_instance(res, dict)
assert_result_count([res],
1,
status="ok",
path=op.join(ds.path, "sub", "i.img"))
# find by path
res = find_container(ds, op.join("sub", "i.img"))
assert_is_instance(res, dict)
assert_result_count([res],
1,
status="ok",
path=op.join(ds.path, "sub", "i.img"))
# don't find another thing
assert_raises(ValueError, find_container, ds, "nothere")
def test_run_unknown_cmdexec_placeholder(path):
ds = Dataset(path).create(force=True)
ds.containers_add("i", image="i.img", call_fmt="{youdontknowme}")
assert_result_count(ds.containers_run("doesn't matter",
on_failure="ignore"),
1,
path=ds.path,
action="run",
status="error")
def test_containers_run(self, path):
if self.image_existed:
raise SkipTest(
"Not pulling with containers-run due to existing image: {}".
format(self.image_name))
from datalad.api import Dataset
ds = Dataset(path).create(force=True)
ds.save(path="foo")
ds.containers_add("bb", url="dhub://" + self.image_name)
with swallow_outputs() as out:
ds.containers_run(["cat", "foo"], container_name="bb")
assert_in("content", out.out)
def test_docker(path): # Singularity's "docker://" scheme.
ds = Dataset(path).create()
ds.containers_add(
"bb",
url=("docker://busybox@sha256:"
"7964ad52e396a6e045c39b5a44438424ac52e12e4d5a25d94895f2058cb863a0"
))
img = op.join(ds.path, ".datalad", "environments", "bb", "image")
assert_result_count(ds.containers_list(), 1, path=img, name="bb")
ok_clean_git(path)
with swallow_outputs():
ds.containers_run(["ls", "/singularity"])
def test_docker(path): # Singularity's "docker://" scheme.
ds = Dataset(path).create()
ds.containers_add(
"bb",
url=("docker://busybox@sha256:"
"7964ad52e396a6e045c39b5a44438424ac52e12e4d5a25d94895f2058cb863a0"
))
img = op.join(ds.path, ".datalad", "environments", "bb", "image")
assert_result_count(ds.containers_list(), 1, path=img, name="bb")
ok_clean_git(path)
WitlessRunner(cwd=ds.path).run(
["datalad", "containers-run", "ls", "/singularity"],
protocol=StdOutCapture)
def test_containers_run(self, path):
if self.image_existed:
raise SkipTest(
"Not pulling with containers-run due to existing image: {}".
format(self.image_name))
from datalad.api import Dataset
ds = Dataset(path).create(force=True)
ds.save(path="foo")
ds.containers_add("bb", url="dhub://" + self.image_name)
out = WitlessRunner(cwd=ds.path).run(
["datalad", "containers-run", "-n", "bb", "cat foo"],
protocol=StdOutCapture)
assert_in("content", out["stdout"])
def test_add_local_path(path, local_file):
ds = Dataset(path).create()
res = ds.containers_add(name="foobert", url=op.join(local_file, "foo.img"))
foo_target = op.join(path, ".datalad", "environments", "foobert", "image")
assert_result_count(res,
1,
status="ok",
type="file",
path=foo_target,
action="containers_add")
# We've just copied and added the file.
assert_not_in(ds.repo.WEB_UUID, ds.repo.whereis(foo_target))
# We can force the URL to be added. (Note: This works because datalad
# overrides 'annex.security.allowed-url-schemes' in its tests.)
ds.containers_add(name="barry",
url=get_local_file_url(op.join(local_file, "bar.img")))
bar_target = op.join(path, ".datalad", "environments", "barry", "image")
assert_in(ds.repo.WEB_UUID, ds.repo.whereis(bar_target))
def test_add_noop(path):
ds = Dataset(path).create()
ok_clean_git(ds.path)
assert_raises(TypeError, ds.containers_add)
# fails when there is no image
assert_status('error', ds.containers_add('name', on_failure='ignore'))
# no config change
ok_clean_git(ds.path)
# place a dummy "image" file
with open(op.join(ds.path, 'dummy'), 'w') as f:
f.write('some')
ds.save('dummy')
ok_clean_git(ds.path)
# config will be added, as long as there is a file, even when URL access
# fails
res = ds.containers_add('broken',
url='bogus-protocol://bogus-server',
image='dummy',
on_failure='ignore')
assert_status('ok', res)
assert_result_count(res, 1, action='save', status='ok')
def test_container_files(ds_path, local_file, url):
# setup things to add
#
# Note: Since "adding" as a container doesn't actually call anything or use
# the container in some way, but simply registers it, for testing any file
# is sufficient.
local_file = get_local_file_url(op.join(local_file, 'some_container.img'))
# prepare dataset:
ds = Dataset(ds_path).create()
# non-default location:
ds.config.add("datalad.containers.location",
value=op.join(".datalad", "test-environments"),
where='dataset')
ds.save(message="Configure container mountpoint")
# no containers yet:
res = ds.containers_list(**RAW_KWDS)
assert_result_count(res, 0)
# add first "image": must end up at the configured default location
target_path = op.join(ds.path, ".datalad", "test-environments", "first",
"image")
res = ds.containers_add(name="first", url=local_file)
ok_clean_git(ds.repo)
assert_result_count(res,
1,
status="ok",
type="file",
path=target_path,
action="containers_add")
ok_(op.lexists(target_path))
res = ds.containers_list(**RAW_KWDS)
assert_result_count(res, 1)
assert_result_count(res,
1,
name='first',
type='file',
action='containers',
status='ok',
path=target_path)
# and kill it again
# but needs name
assert_raises(TypeError, ds.containers_remove)
res = ds.containers_remove('first', remove_image=True)
assert_status('ok', res)
assert_result_count(ds.containers_list(**RAW_KWDS), 0)
# image removed
assert (not op.lexists(target_path))
def test_run_mispecified(path):
ds = Dataset(path).create(force=True)
ds.save(path=["dummy0.img", "dummy1.img"])
ok_clean_git(path)
# Abort if no containers exist.
with assert_raises(ValueError) as cm:
ds.containers_run("doesn't matter")
assert_in("No known containers", text_type(cm.exception))
# Abort if more than one container exists but no container name is
# specified.
ds.containers_add("d0", image="dummy0.img")
ds.containers_add("d1", image="dummy0.img")
with assert_raises(ValueError) as cm:
ds.containers_run("doesn't matter")
assert_in("explicitly specify container", text_type(cm.exception))
# Abort if unknown container is specified.
with assert_raises(ValueError) as cm:
ds.containers_run("doesn't matter", container_name="ghost")
assert_in("Container selection impossible", text_type(cm.exception))
def test_container_update(ds_path, local_file, url):
url_foo = get_local_file_url(op.join(local_file, 'foo.img'))
url_bar = get_local_file_url(op.join(local_file, 'bar.img'))
img = op.join(".datalad", "environments", "foo", "image")
ds = Dataset(ds_path).create()
ds.containers_add(name="foo", call_fmt="call-fmt1", url=url_foo)
# Abort without --update flag.
res = ds.containers_add(name="foo", on_failure="ignore")
assert_result_count(res, 1, action="containers_add", status="impossible")
# Abort if nothing to update is specified.
res = ds.containers_add(name="foo", update=True, on_failure="ignore")
assert_result_count(res,
1,
action="containers_add",
status="impossible",
message="No values to update specified")
# Update call format.
ds.containers_add(name="foo", update=True, call_fmt="call-fmt2")
assert_equal(ds.config.get("datalad.containers.foo.cmdexec"), "call-fmt2")
ok_file_has_content(op.join(ds.path, img), "foo")
# Update URL/image.
ds.drop(img) # Make sure it works even with absent content.
res = ds.containers_add(name="foo", update=True, url=url_bar)
assert_result_count(res, 1, action="remove", status="ok", path=img)
assert_result_count(res, 1, action="save", status="ok")
ok_file_has_content(op.join(ds.path, img), "bar")
# Test commit message
# In the above case it was updating existing image so should have "Update "
get_commit_msg = lambda *args: ds.repo.format_commit('%B')
assert_in("Update ", get_commit_msg())
# If we add a new image with update=True should say Configure
res = ds.containers_add(name="foo2", update=True, url=url_bar)
assert_in("Configure ", get_commit_msg())
def test_container_files(path, super_path):
raise SkipTest('SingularityHub is gone for now')
ds = Dataset(path).create()
cmd = ['dir'] if on_windows else ['ls']
# plug in a proper singularity image
ds.containers_add(
'mycontainer',
url=testimg_url,
image='righthere',
# the next one is auto-guessed
#call_fmt='singularity exec {img} {cmd}'
)
assert_result_count(ds.containers_list(),
1,
path=op.join(ds.path, 'righthere'),
name='mycontainer')
ok_clean_git(path)
def assert_no_change(res, path):
# this command changed nothing
#
# Avoid specifying the action because it will change from "add" to
# "save" in DataLad v0.12.
assert_result_count(res,
1,
status='notneeded',
path=path,
type='dataset')
# now we can run stuff in the container
# and because there is just one, we don't even have to name the container
res = ds.containers_run(cmd)
# container becomes an 'input' for `run` -> get request, but "notneeded"
assert_result_count(res,
1,
action='get',
status='notneeded',
path=op.join(ds.path, 'righthere'),
type='file')
assert_no_change(res, ds.path)
# same thing as we specify the container by its name:
res = ds.containers_run(cmd, container_name='mycontainer')
# container becomes an 'input' for `run` -> get request, but "notneeded"
assert_result_count(res,
1,
action='get',
status='notneeded',
path=op.join(ds.path, 'righthere'),
type='file')
assert_no_change(res, ds.path)
# we can also specify the container by its path:
res = ds.containers_run(cmd, container_name=op.join(ds.path, 'righthere'))
# container becomes an 'input' for `run` -> get request, but "notneeded"
assert_result_count(res,
1,
action='get',
status='notneeded',
path=op.join(ds.path, 'righthere'),
type='file')
assert_no_change(res, ds.path)
# Now, test the same thing, but with this dataset being a subdataset of
# another one:
super_ds = Dataset(super_path).create()
super_ds.install("sub", source=path)
# When running, we don't discover containers in subdatasets
with assert_raises(ValueError) as cm:
super_ds.containers_run(cmd)
assert_in("No known containers", text_type(cm.exception))
# ... unless we need to specify the name
res = super_ds.containers_run(cmd, container_name="sub/mycontainer")
# container becomes an 'input' for `run` -> get request (needed this time)
assert_result_count(res,
1,
action='get',
status='ok',
path=op.join(super_ds.path, 'sub', 'righthere'),
type='file')
assert_no_change(res, super_ds.path)
def test_container_from_subdataset(ds_path, src_subds_path, local_file):
# prepare a to-be subdataset with a registered container
src_subds = Dataset(src_subds_path).create()
src_subds.containers_add(name="first",
url=get_local_file_url(
op.join(local_file, 'some_container.img')))
# add it as subdataset to a super ds:
ds = Dataset(ds_path).create()
subds = ds.install("sub", source=src_subds_path)
# add it again one level down to see actual recursion:
subds.install("subsub", source=src_subds_path)
# We come up empty without recursive:
res = ds.containers_list(recursive=False, **RAW_KWDS)
assert_result_count(res, 0)
# query available containers from within super:
res = ds.containers_list(recursive=True, **RAW_KWDS)
assert_result_count(res, 2)
assert_in_results(res, action="containers", refds=ds.path)
# default location within the subdataset:
target_path = op.join(subds.path, '.datalad', 'environments', 'first',
'image')
assert_result_count(res,
1,
name='sub/first',
type='file',
action='containers',
status='ok',
path=target_path,
parentds=subds.path)
# not installed subdataset doesn't pose an issue:
sub2 = ds.create("sub2")
assert_result_count(ds.subdatasets(), 2, type="dataset")
ds.uninstall("sub2")
from datalad.tests.utils import assert_false
assert_false(sub2.is_installed())
# same results as before, not crashing or somehow confused by a not present
# subds:
res = ds.containers_list(recursive=True, **RAW_KWDS)
assert_result_count(res, 2)
assert_result_count(res,
1,
name='sub/first',
type='file',
action='containers',
status='ok',
path=target_path,
parentds=subds.path)
# The default renderer includes the image names.
with swallow_outputs() as out:
ds.containers_list(recursive=True)
lines = out.out.splitlines()
assert_re_in("sub/first", lines)
assert_re_in("sub/subsub/first", lines)
# But we are careful not to render partial names from subdataset traversals
# (i.e. we recurse with containers_list(..., result_renderer=None)).
with assert_raises(AssertionError):
assert_re_in("subsub/first", lines)
def test_demo_repro_analysis(bids_path, ana_path, toolbox_url):
import glob
localizer_ds = Dataset(bids_path).create()
localizer_ds.run_procedure('cfg_bids')
# TODO: decorator
# TODO: with config patch for toolbox ? -> overwrite?
# localizer_ds.install(source="https://github.com/psychoinformatics-de/hirni-demo",
# path="sourcedata",
# recursive=True)
with patch.dict('os.environ', {'DATALAD_HIRNI_TOOLBOX_URL': toolbox_url}):
install_demo_dataset(localizer_ds, "sourcedata", recursive=True)
assert_repo_status(localizer_ds.repo)
subs = localizer_ds.subdatasets(recursive=True)
assert_result_count(subs, 4)
assert_result_count(subs, 1, path=op.join(localizer_ds.path, 'sourcedata'))
assert_result_count(subs,
1,
path=op.join(localizer_ds.path, 'sourcedata', 'code',
'hirni-toolbox'))
assert_result_count(subs,
1,
path=op.join(localizer_ds.path, 'sourcedata', 'acq1',
'dicoms'))
assert_result_count(subs,
1,
path=op.join(localizer_ds.path, 'sourcedata', 'acq2',
'dicoms'))
localizer_ds.hirni_spec2bids(
[op.join(localizer_ds.path, 'sourcedata', 'studyspec.json')] +
glob.glob(
op.join(localizer_ds.path, 'sourcedata', '*', 'studyspec.json')),
anonymize=True)
for f in [
'sub-001',
'task-oneback_bold.json',
'participants.tsv',
op.join('sub-001', 'sub-001_scans.tsv'),
op.join('sub-001', 'anat'),
op.join('sub-001', 'anat', 'sub-001_run-1_T1w.json'),
op.join('sub-001', 'anat', 'sub-001_run-1_T1w.nii.gz'),
op.join('sub-001', 'func'),
op.join('sub-001', 'func',
'sub-001_task-oneback_run-01_bold.json'),
op.join('sub-001', 'func',
'sub-001_task-oneback_run-01_bold.nii.gz'),
op.join('sub-001', 'func',
'sub-001_task-oneback_run-01_events.tsv'),
]:
assert_true(op.lexists(op.join(localizer_ds.path, f)))
analysis_ds = Dataset(ana_path).create()
analysis_ds.install(source=localizer_ds.path,
path=op.join('inputs', 'rawdata'))
analysis_ds.run_procedure('cfg_yoda')
# download-url expects the target dir to exist
(analysis_ds.pathobj / 'code').mkdir(exist_ok=True)
analysis_ds.download_url(
path=op.join(analysis_ds.path, 'code') + op.
sep, # TODO: File issue. relative path via python API bound method doesn't work
urls=[
'https://raw.githubusercontent.com/myyoda/ohbm2018-training/master/section23/scripts/events2ev3.sh',
'https://raw.githubusercontent.com/myyoda/ohbm2018-training/master/section23/scripts/ffa_design.fsf'
])
assert_repo_status(analysis_ds.repo)
ok_file_under_git(op.join(analysis_ds.path, 'code'),
'events2ev3.sh',
annexed=False)
ok_file_under_git(op.join(analysis_ds.path, 'code'),
'ffa_design.fsf',
annexed=False)
analysis_ds.run(inputs=[
op.join('inputs', 'rawdata', 'sub-001', 'func',
'sub-001_task-oneback_run-01_events.tsv')
],
outputs=[op.join('sub-001', 'onsets')],
cmd='bash code/events2ev3.sh sub-001 {inputs}',
message="Build FSL EV3 design files")
raise SkipTest("Solve datalad-containers #115")
analysis_ds.containers_add('fsl',
url="shub://ReproNim/ohbm2018-training:fsln")
# % datalad containers-list
analysis_ds.save(version_tag="ready4analysis")
assert_repo_status(analysis_ds.repo)
#
analysis_ds.run(
outputs=[op.join('sub-001', '1stlvl_design.fsf')],
cmd=
"bash -c 'sed -e \"s,##BASEPATH##,{pwd},g\" -e \"s,##SUB##,sub-001,g\" code/ffa_design.fsf > {outputs}'",
message="FSL FEAT analysis config script")
assert_repo_status(analysis_ds.repo)
def test_container_files(ds_path, local_file, url):
# setup things to add
#
# Note: Since "adding" as a container doesn't actually call anything or use
# the container in some way, but simply registers it, for testing any file
# is sufficient.
local_file = get_local_file_url(op.join(local_file, 'some_container.img'))
remote_file = urljoin(url, 'some_container.img')
# prepare dataset:
ds = Dataset(ds_path).create()
# non-default location:
ds.config.add("datalad.containers.location",
value=op.join(".datalad", "test-environments"),
where='dataset')
ds.save(message="Configure container mountpoint")
# no containers yet:
res = ds.containers_list()
assert_result_count(res, 0)
# add first "image":
res = ds.containers_add(name="first", url=local_file)
ok_clean_git(ds.repo)
target_path = op.join(ds.path, ".datalad", "test-environments", "first")
assert_result_count(res,
1,
status="ok",
type="file",
path=target_path,
action="containers_add")
ok_(op.lexists(target_path))
eq_(local_file, ds.config.get("datalad.containers.first.url"))
# add a "remote" one:
# don't provide url in the call, but in a config:
ds.config.add("datalad.containers.second.url",
value=remote_file,
where='dataset')
ds.save(message="Configure URL for container 'second'")
res = ds.containers_add(name="second")
ok_clean_git(ds.repo)
target_path = op.join(ds.path, ".datalad", "test-environments", "second")
assert_result_count(res,
1,
status="ok",
type="file",
path=target_path,
action="containers_add")
ok_(op.lexists(target_path))
# config wasn't changed:
eq_(remote_file, ds.config.get("datalad.containers.second.url"))
res = ds.containers_list()
assert_result_count(res,
2,
status='ok',
type='file',
action='containers_list')
