def test_custom_call_fmt(path, local_file): ds = Dataset(path).create() subds = ds.create('sub') # plug in a proper singularity image subds.containers_add( 'mycontainer', url=get_local_file_url(op.join(local_file, 'some_container.img')), image='righthere', call_fmt='echo image={img} cmd={cmd} img_dspath={img_dspath} ' # and environment variable being set/propagated by default 'name=$DATALAD_CONTAINER_NAME' ) ds.save() # record the effect in super-dataset # Running should work fine either withing sub or within super with swallow_outputs() as cmo: subds.containers_run('XXX', container_name='mycontainer') assert_in('image=righthere cmd=XXX img_dspath=. name=mycontainer', cmo.out) with swallow_outputs() as cmo: ds.containers_run('XXX', container_name='sub/mycontainer') assert_in('image=sub/righthere cmd=XXX img_dspath=sub', cmo.out) # Test within subdirectory of the super-dataset subdir = op.join(ds.path, 'subdir') os.mkdir(subdir) with chpwd(subdir): with swallow_outputs() as cmo: containers_run('XXX', container_name='sub/mycontainer') assert_in('image=../sub/righthere cmd=XXX img_dspath=../sub', cmo.out)
def test_containers_run(self, path): if self.image_existed: raise SkipTest( "Not pulling with containers-run due to existing image: {}". format(self.image_name)) from datalad.api import Dataset ds = Dataset(path).create(force=True) ds.save(path="foo") ds.containers_add("bb", url="dhub://" + self.image_name) with swallow_outputs() as out: ds.containers_run(["cat", "foo"], container_name="bb") assert_in("content", out.out)
def test_docker(path): # Singularity's "docker://" scheme. ds = Dataset(path).create() ds.containers_add( "bb", url=("docker://busybox@sha256:" "7964ad52e396a6e045c39b5a44438424ac52e12e4d5a25d94895f2058cb863a0" )) img = op.join(ds.path, ".datalad", "environments", "bb", "image") assert_result_count(ds.containers_list(), 1, path=img, name="bb") ok_clean_git(path) with swallow_outputs(): ds.containers_run(["ls", "/singularity"])
def test_container_files(path): ds = Dataset(path).create() # plug in a proper singularity image ds.containers_add( 'mycontainer', url=testimg_url, image='righthere', # the next one is auto-guessed #call_fmt='singularity exec {img} {cmd}' ) assert_result_count( ds.containers_list(), 1, path=op.join(ds.path, 'righthere'), name='mycontainer', updateurl=testimg_url) ok_clean_git(path) # now we can run stuff in the container # and because there is just one, we don't even have to name the container res = ds.containers_run(['dir'] if on_windows else ['ls']) # container becomes an 'input' for `run` -> get request, but "notneeded" assert_result_count( res, 1, action='get', status='notneeded', path=op.join(ds.path, 'righthere'), type='file') # this command changed nothing assert_result_count( res, 1, action='add', status='notneeded', path=ds.path, type='dataset')
def test_run_unknown_cmdexec_placeholder(path): ds = Dataset(path).create(force=True) ds.containers_add("i", image="i.img", call_fmt="{youdontknowme}") assert_result_count(ds.containers_run("doesn't matter", on_failure="ignore"), 1, path=ds.path, action="run", status="error")
def test_run_mispecified(path): ds = Dataset(path).create(force=True) ds.save(path=["dummy0.img", "dummy1.img"]) ok_clean_git(path) # Abort if no containers exist. with assert_raises(ValueError) as cm: ds.containers_run("doesn't matter") assert_in("No known containers", text_type(cm.exception)) # Abort if more than one container exists but no container name is # specified. ds.containers_add("d0", image="dummy0.img") ds.containers_add("d1", image="dummy0.img") with assert_raises(ValueError) as cm: ds.containers_run("doesn't matter") assert_in("explicitly specify container", text_type(cm.exception)) # Abort if unknown container is specified. with assert_raises(ValueError) as cm: ds.containers_run("doesn't matter", container_name="ghost") assert_in("Container selection impossible", text_type(cm.exception))
dataset.containers_run( [ 'heudiconv', # XXX absolute path will make rerun on other # system impossible -- hard to avoid # TODO: from toolbox? config? '-f', heuristic_path, # leaves identifying info in run record '-s', subject, '-c', 'dcm2niix', # TODO decide on the fate of .heudiconv/ # but ATM we need to (re)move it: # https://github.com/nipy/heudiconv/issues/196 '-o', rel_trash_path, '-b', '-a', '{dspath}', '-l', '', # avoid glory details provided by dcmstack, # we have them in the aggregated DICOM # metadata already '--minmeta', '--files', location ], sidecar=anonymize, # TODO: This doesn't work! ... Well, it does. What was the problem? container_name=op.relpath( op.join(op.dirname(op.realpath(__file__)), "heudiconv.simg"), dataset.path), explicit=True, expand="both", inputs=inputs, # Note: Outputs determination isn't good yet. We need a way to # figure what exactly heudiconv produced. This is different from # other toolbox-procedures due to our "injection heuristic". outputs=outputs, message="[HIRNI] Convert DICOM data for subject {}" "".format(subject), )
def test_container_files(path, super_path): raise SkipTest('SingularityHub is gone for now') ds = Dataset(path).create() cmd = ['dir'] if on_windows else ['ls'] # plug in a proper singularity image ds.containers_add( 'mycontainer', url=testimg_url, image='righthere', # the next one is auto-guessed #call_fmt='singularity exec {img} {cmd}' ) assert_result_count(ds.containers_list(), 1, path=op.join(ds.path, 'righthere'), name='mycontainer') ok_clean_git(path) def assert_no_change(res, path): # this command changed nothing # # Avoid specifying the action because it will change from "add" to # "save" in DataLad v0.12. assert_result_count(res, 1, status='notneeded', path=path, type='dataset') # now we can run stuff in the container # and because there is just one, we don't even have to name the container res = ds.containers_run(cmd) # container becomes an 'input' for `run` -> get request, but "notneeded" assert_result_count(res, 1, action='get', status='notneeded', path=op.join(ds.path, 'righthere'), type='file') assert_no_change(res, ds.path) # same thing as we specify the container by its name: res = ds.containers_run(cmd, container_name='mycontainer') # container becomes an 'input' for `run` -> get request, but "notneeded" assert_result_count(res, 1, action='get', status='notneeded', path=op.join(ds.path, 'righthere'), type='file') assert_no_change(res, ds.path) # we can also specify the container by its path: res = ds.containers_run(cmd, container_name=op.join(ds.path, 'righthere')) # container becomes an 'input' for `run` -> get request, but "notneeded" assert_result_count(res, 1, action='get', status='notneeded', path=op.join(ds.path, 'righthere'), type='file') assert_no_change(res, ds.path) # Now, test the same thing, but with this dataset being a subdataset of # another one: super_ds = Dataset(super_path).create() super_ds.install("sub", source=path) # When running, we don't discover containers in subdatasets with assert_raises(ValueError) as cm: super_ds.containers_run(cmd) assert_in("No known containers", text_type(cm.exception)) # ... unless we need to specify the name res = super_ds.containers_run(cmd, container_name="sub/mycontainer") # container becomes an 'input' for `run` -> get request (needed this time) assert_result_count(res, 1, action='get', status='ok', path=op.join(super_ds.path, 'sub', 'righthere'), type='file') assert_no_change(res, super_ds.path)
# Print all available help. # # --version # Print version information and exit. # TODO: Expose all the parameters (and prob. set up argparse for that) if __name__ == '__main__': import sys import os.path as op from datalad.api import Dataset from datalad.config import anything2bool dataset = Dataset(sys.argv[1]) anonymize = anything2bool(sys.argv[2]) files = sys.argv[3:] dataset.containers_run( ['mridefacer', '--apply'] + files, sidecar=anonymize, container_name=op.relpath( op.join(op.dirname(op.realpath(__file__)), "mridefacer.simg"), dataset.path), inputs=files, outputs=[dataset.path], message="[HIRNI] Deface MRI data" )
import os.path as op from datalad.api import Dataset dataset = Dataset(sys.argv[1]) params = [] delimiter = None for i, p in enumerate(sys.argv[2:]): if p == '--': delimiter = i break else: params.append(p) if not delimiter: print("Missing '-- [FILES]'") sys.exit(1) files = sys.argv[delimiter + 3:] # TODO: How to get those into a single commit? Script in container or # call a loop directly? for f in files: dataset.containers_run( ['fslroi', f, f] + params, sidecar=False, container_name=op.relpath( op.join(op.dirname(op.realpath(__file__)), "fsl.simg"), dataset.path), inputs=[f], outputs=[f], )