def test_dx_docker_pull_hash_or_not(self):
run("dx-docker pull dnanexus/testdocker")
self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'dnanexus%2Ftestdocker.aci')))
repo = "dnanexus/testdocker@sha256:4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb"
run("dx-docker pull {}".format(repo))
sanit='dnanexus%2Ftestdocker%40sha256%3A4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb.aci'
self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, sanit)))
def test_dx_docker_volume(self):
os.makedirs('dxdtestdata')
run("dx-docker run -v dxdtestdata:/data-host ubuntu:14.04 touch /data-host/newfile.txt"
)
self.assertTrue(
os.path.isfile(os.path.join('dxdtestdata', 'newfile.txt')))
shutil.rmtree('dxdtestdata')
def test_dx_docker_pull(self):
run("dx-docker pull ubuntu:14.04")
self.assertTrue(
os.path.isfile(os.path.join(CACHE_DIR, 'ubuntu%3A14.04.aci')))
run("dx-docker pull ubuntu:15.04")
self.assertTrue(
os.path.isfile(os.path.join(CACHE_DIR, 'ubuntu%3A15.04.aci')))
def test_dx_docker_pull_hash_or_not(self):
run("dx-docker pull dnanexus/testdocker")
self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'dnanexus%2Ftestdocker.aci')))
repo = "dnanexus/testdocker@sha256:4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb"
run("dx-docker pull {}".format(repo))
sanit='dnanexus%2Ftestdocker%40sha256%3A4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb.aci'
self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, sanit)))
def test_file_arrays(self):
# Create file with junk content
dxfile = dxpy.upload_string("xxyyzz",
project=self.project,
wait_on_close=True,
name="bubbles")
# write python code into code.py file
tmp_path = tempfile.mkdtemp()
code_path = os.path.join(tmp_path, 'code.py')
with open(code_path, write_mode) as f:
f.write("@dxpy.entry_point('main')\n")
f.write("def main(**job_inputs):\n")
f.write("\toutput = {}\n")
f.write("\toutput['plant'] = job_inputs['plant']\n")
f.write("\treturn output\n")
f.write("\n")
f.write("dxpy.run()\n")
with open(code_path, 'r') as f:
code = f.read()
# write arguments table. These ara arrays with a single element.
arg_table = os.path.join(tmp_path, 'table.csv')
with open(arg_table, write_mode) as csvfile:
writer = csv.writer(csvfile, delimiter=delimiter)
header = ["batch ID", "plant", "plant ID"]
writer.writerow(header)
writer.writerow(
["SRR_1", "[bubbles]", "[" + dxfile.get_id() + "]"])
applet = dxpy.api.applet_new({
"name":
"ident_file_array",
"project":
self.project,
"dxapi":
"1.0.0",
"inputSpec": [{
"name": "plant",
"class": "array:file"
}],
"outputSpec": [{
"name": "plant",
"class": "array:file"
}],
"runSpec": {
"interpreter": "python2.7",
"code": code,
"distribution": "Ubuntu",
"release": "14.04"
}
})
job_id = run("dx run {} --batch-tsv={} --yes --brief".format(
applet["id"], arg_table)).strip()
job_desc = dxpy.api.job_describe(job_id)
self.assertEqual(job_desc["executableName"], 'ident_file_array')
self.assertEqual(job_desc["input"],
{"plant": [{
"$dnanexus_link": dxfile.get_id()
}]})
def test_dx_docker_create_asset(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
run("dx-docker create-asset ubuntu:14.04")
self.assertEqual(run("dx ls ubuntu\\\\:14.04").strip(), 'ubuntu:14.04')
create_folder_in_project(test_projectid, '/testfolder')
run("dx-docker create-asset busybox -o testfolder")
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, 'busybox')
ls_out = run("dx ls testfolder\\/busybox.tar.gz").strip()
self.assertEqual(ls_out, 'busybox.tar.gz')
def test_dx_docker_create_asset(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
run("dx-docker create-asset ubuntu:14.04")
self.assertEqual(run("dx ls ubuntu\\\\:14.04").strip(), 'ubuntu:14.04')
create_folder_in_project(test_projectid, '/testfolder')
run("dx-docker create-asset busybox -o testfolder")
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, 'busybox')
ls_out = run("dx ls testfolder\\/busybox.tar.gz").strip()
self.assertEqual(ls_out, 'busybox.tar.gz')
def test_dx_docker_create_asset_with_image_digest(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
create_folder_in_project(test_projectid, '/testfolder')
image_digest = run("docker inspect ubuntu:14.04 | jq -r '.[] | .RepoDigests[0]'").strip()
run("dx-docker create-asset {image_digest} -o testfolder".format(image_digest=image_digest))
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, image_digest)
def test_dx_docker_create_asset_with_long_imageid(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
long_id = run("docker images --no-trunc -q ubuntu:14.04").strip()
create_folder_in_project(test_projectid, '/testfolder')
run("dx-docker create-asset {long_id} -o testfolder".format(long_id=long_id))
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, long_id)
def test_dx_docker_create_asset_with_long_imageid(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
long_id = run("docker images --no-trunc -q ubuntu:14.04").strip()
create_folder_in_project(test_projectid, '/testfolder')
run("dx-docker create-asset {long_id} -o testfolder".format(long_id=long_id))
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, long_id)
def test_dx_docker_create_asset_with_image_digest(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
create_folder_in_project(test_projectid, '/testfolder')
image_digest = run("docker inspect ubuntu:14.04 | jq -r '.[] | .RepoDigests[0]'").strip()
run("dx-docker create-asset {image_digest} -o testfolder".format(image_digest=image_digest))
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, image_digest)
def test_file_arrays(self):
# Create file with junk content
dxfile = dxpy.upload_string("xxyyzz", project=self.project,
wait_on_close=True, name="bubbles")
# write python code into code.py file
tmp_path = tempfile.mkdtemp()
code_path = os.path.join(tmp_path, 'code.py')
with open(code_path, write_mode) as f:
f.write("@dxpy.entry_point('main')\n")
f.write("def main(**job_inputs):\n")
f.write("\toutput = {}\n")
f.write("\toutput['plant'] = job_inputs['plant']\n")
f.write("\treturn output\n")
f.write("\n")
f.write("dxpy.run()\n")
with open(code_path, 'r') as f:
code = f.read()
# write arguments table. These ara arrays with a single element.
arg_table = os.path.join(tmp_path, 'table.csv')
with open(arg_table, write_mode) as csvfile:
writer = csv.writer(csvfile, delimiter=delimiter)
header = ["batch ID", "plant", "plant ID"]
writer.writerow(header)
writer.writerow(["SRR_1",
"[bubbles]",
"[" + dxfile.get_id() + "]"
])
applet = dxpy.api.applet_new({
"name": "ident_file_array",
"project": self.project,
"dxapi": "1.0.0",
"inputSpec": [ { "name": "plant", "class": "array:file" } ],
"outputSpec": [ { "name": "plant", "class": "array:file" } ],
"runSpec": { "interpreter": "python2.7",
"code": code,
"distribution": "Ubuntu",
"release": "14.04" }
})
job_id = run("dx run {} --batch-tsv={} --yes --brief"
.format(applet["id"], arg_table)).strip()
job_desc = dxpy.api.job_describe(job_id)
self.assertEqual(job_desc["executableName"], 'ident_file_array')
self.assertEqual(job_desc["input"],
{ "plant":
[{ "$dnanexus_link": dxfile.get_id() }]
})
def test_dx_docker_add_to_applet(self):
os.makedirs('tmpapp')
run("docker pull busybox")
with self.assertSubprocessFailure(exit_code=1, stderr_regexp='does not appear to have a dxapp.json that parses'):
run("dx-docker add-to-applet busybox tmpapp")
with open('tmpapp/dxapp.json', 'w') as dxapp:
dxapp.write("[]")
run("dx-docker add-to-applet busybox tmpapp")
self.assertTrue(os.path.isfile(os.path.join('tmpapp', 'resources/tmp/dx-docker-cache/busybox.aci')))
shutil.rmtree('tmpapp')
def test_dx_docker_add_to_applet(self):
os.makedirs('tmpapp')
run("docker pull busybox")
with self.assertSubprocessFailure(exit_code=1, stderr_regexp='does not appear to have a dxapp.json that parses'):
run("dx-docker add-to-applet busybox tmpapp")
with open('tmpapp/dxapp.json', 'w') as dxapp:
dxapp.write("[]")
run("dx-docker add-to-applet busybox tmpapp")
self.assertTrue(os.path.isfile(os.path.join('tmpapp', 'resources/tmp/dx-docker-cache/busybox.aci')))
shutil.rmtree('tmpapp')
def test_symlinks(self):
dxfile1 = self.download_url_create_symlink("https://s3.amazonaws.com/1000genomes/CHANGELOG",
"sym1")
dxfile2 = self.download_url_create_symlink("https://documentation.dnanexus.com/",
"sym2")
# download to PWD
run("dx download {}:/{} -o {}".format(self.proj_id, "sym1", "localfile"))
# absolute path
run("dx download {}:/{} -f -o {}".format(self.proj_id, "sym1", "/tmp/localfile"))
# relative path
run("dx download {}:/{} -f -o {}".format(self.proj_id, "sym2", "../localfile"))
def test_symlinks(self):
dxfile1 = self.download_url_create_symlink(
"https://s3.amazonaws.com/1000genomes/CHANGELOG", "sym1")
dxfile2 = self.download_url_create_symlink(
"https://en.wikipedia.org/wiki/Poitevin_horse", "sym2")
# download to PWD
run("dx download {}:/{} -o {}".format(self.proj_id, "sym1",
"localfile"))
# absolute path
run("dx download {}:/{} -f -o {}".format(self.proj_id, "sym1",
"/tmp/localfile"))
# relative path
run("dx download {}:/{} -f -o {}".format(self.proj_id, "sym2",
"../localfile"))
def test_dx_docker_volume(self):
os.makedirs('dxdtestdata')
run("dx-docker run -v dxdtestdata:/data-host ubuntu:14.04 touch /data-host/newfile.txt")
self.assertTrue(os.path.isfile(os.path.join('dxdtestdata', 'newfile.txt')))
shutil.rmtree('dxdtestdata')
def test_dx_docker_pull_failure(self):
with self.assertSubprocessFailure(
exit_code=1, stderr_regexp='Failed to obtain image'):
run("dx-docker pull busyboxasdf")
def test_dx_docker_additional_container(self):
run("dx-docker run busybox ls")
def test_complex_quote(self):
run('dx-docker run python:2-slim /bin/sh -c "echo \'{"foo": {"location": "file:///"}}\' > /dev/stdout"'
)
def test_dx_docker_entrypoint_cmd(self):
docker_out = run("docker run dnanexus/testdocker /bin")
dx_docker_out = run("dx-docker run -q dnanexus/testdocker /bin")
self.assertEqual(docker_out, dx_docker_out)
def test_dx_docker_run_rm(self):
run("dx-docker run --rm ubuntu ls")
def test_dx_docker_basic_commands(self):
run("dx-docker run ubuntu:14.04 ls --color")
run("dx-docker run ubuntu:15.04 ls")
def test_dx_docker_run_error_codes(self):
with self.assertSubprocessFailure(exit_code=1):
run("dx-docker run ubuntu:14.04 false")
run("dx-docker run ubuntu:14.04 true")
def test_dx_docker_basic_commands(self):
run("dx-docker run ubuntu:14.04 ls --color")
run("dx-docker run ubuntu:15.04 ls")
def setUpClass(cls):
run("docker pull ubuntu:14.04")
run("docker pull busybox")
def test_dx_docker_pull_failure(self):
with self.assertSubprocessFailure(exit_code=1, stderr_regexp='Failed to obtain image'):
run("dx-docker pull busyboxasdf")
def test_dx_docker_working_dir_override(self):
run("dx-docker run -v $PWD:/tmp -w /tmp quay.io/ucsc_cgl/samtools faidx test.fa")
def test_dx_docker_pull_quay(self):
run("dx-docker pull quay.io/ucsc_cgl/samtools")
self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'quay.io%2Fucsc_cgl%2Fsamtools.aci')))
def test_dx_docker_pull_silent(self):
dx_docker_out = run("dx-docker pull -q busybox").strip()
self.assertEqual(dx_docker_out, '')
def test_dx_docker_pull(self):
run("dx-docker pull ubuntu:14.04")
self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'ubuntu%3A14.04.aci')))
run("dx-docker pull ubuntu:15.04")
self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'ubuntu%3A15.04.aci')))
def test_basic(self):
# write python code into code.py file
tmp_path = tempfile.mkdtemp()
code_path = os.path.join(tmp_path, 'code.py')
with open(code_path, write_mode) as f:
f.write("@dxpy.entry_point('main')\n")
f.write("def main(**job_inputs):\n")
f.write("\toutput = {}\n")
f.write("\toutput['thresholds'] = job_inputs['thresholds']\n")
f.write("\toutput['pie'] = job_inputs['pie'] + 1\n")
f.write("\toutput['misc'] = {'n': 'non', 'y': 'oui'}\n")
f.write("\treturn output\n")
f.write("\n")
f.write("dxpy.run()\n")
with open(code_path, 'r') as f:
code = f.read()
# write arguments table
arg_table = os.path.join(tmp_path, 'table.csv')
with open(arg_table, write_mode) as csvfile:
writer = csv.writer(csvfile, delimiter=delimiter)
header = ["batch ID", "thresholds", "pie", "misc"]
writer.writerow(header)
writer.writerow(["SRR_1", "[10,81]", "3.12", "{}"])
applet = dxpy.api.applet_new({
"name":
"copy_all",
"project":
self.project,
"dxapi":
"1.0.0",
"inputSpec": [{
"name": "thresholds",
"class": "array:int"
}, {
"name": "pie",
"class": "float"
}, {
"name": "misc",
"class": "hash"
}],
"outputSpec": [{
"name": "thresholds",
"class": "array:int"
}, {
"name": "pie",
"class": "float"
}, {
"name": "misc",
"class": "hash"
}],
"runSpec": {
"interpreter": "python2.7",
"code": code,
"distribution": "Ubuntu",
"release": "14.04"
}
})
# run in batch mode
job_id = run("dx run {} --batch-tsv={} --yes --brief".format(
applet["id"], arg_table)).strip()
job_desc = dxpy.api.job_describe(job_id)
self.assertEqual(job_desc["executableName"], 'copy_all')
self.assertEqual(job_desc["input"], {
"thresholds": [10, 81],
"misc": {},
"pie": 3.12
})
# run in batch mode with --batch-folders
job_id = run(
"dx run {} --batch-tsv={} --batch-folders --yes --brief".format(
applet["id"], arg_table)).strip()
job_desc = dxpy.api.job_describe(job_id)
self.assertEqual(job_desc["folder"], "/SRR_1")
# run in batch mode with --batch-folders and --destination
job_id = run(
"dx run {} --batch-tsv={} --batch-folders --destination={}:/run_01 --yes --brief"
.format(applet["id"], arg_table, self.project)).strip()
job_desc = dxpy.api.job_describe(job_id)
self.assertEqual(job_desc["folder"], "/run_01/SRR_1")
def test_dx_docker_run_canonical(self):
run("dx-docker run quay.io/ucsc_cgl/samtools --help")
def test_dx_docker_run_from_hash(self):
repo = "dnanexus/testdocker@sha256:4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb"
run("dx-docker run {}".format(repo))
def test_dx_docker_additional_container(self):
run("dx-docker run busybox ls")
def test_dx_docker_entrypoint_cmd(self):
docker_out = run("docker run dnanexus/testdocker /bin")
dx_docker_out = run("dx-docker run -q dnanexus/testdocker /bin")
self.assertEqual(docker_out, dx_docker_out)
def test_dx_docker_run_error_codes(self):
with self.assertSubprocessFailure(exit_code=1):
run("dx-docker run ubuntu:14.04 false")
run("dx-docker run ubuntu:14.04 true")
def test_dx_docker_home_dir(self):
run("dx-docker run julia:0.5.0 julia -E 'println(\"hello world\")'")
def test_dx_docker_run_canonical(self):
run("dx-docker run quay.io/ucsc_cgl/samtools --help")
def test_dx_docker_set_env(self):
dx_docker_out = run(
"dx-docker run --env HOME=/somethingelse busybox env")
self.assertTrue(dx_docker_out.find("HOME=/somethingelse") != -1)
def test_dx_docker_set_env(self):
dx_docker_out = run("dx-docker run --env HOME=/somethingelse busybox env")
self.assertTrue(dx_docker_out.find("HOME=/somethingelse") != -1)
def test_dx_docker_working_dir_override(self):
run("dx-docker run -v $PWD:/tmp -w /tmp busybox ls")
def test_dx_docker_working_dir_override(self):
run("dx-docker run -v $PWD:/tmp -w /tmp quay.io/ucsc_cgl/samtools faidx test.fa")
def setUpClass(cls):
run("docker pull ubuntu:14.04")
run("docker pull busybox")
def test_dx_docker_pull_quay(self):
run("dx-docker pull quay.io/ucsc_cgl/samtools")
self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'quay.io%2Fucsc_cgl%2Fsamtools.aci')))
def test_dx_docker_pull_silent(self):
dx_docker_out = run("dx-docker pull -q busybox").strip()
self.assertEqual(dx_docker_out, '')
def test_dx_docker_run_rm(self):
run("dx-docker run --rm ubuntu ls")
def test_dx_docker_home_dir(self):
run("dx-docker run julia:0.5.0 julia -E 'println(\"hello world\")'")
def test_complex_quote(self):
run('dx-docker run python:2-slim /bin/sh -c "echo \'{"foo": {"location": "file:///"}}\' > /dev/stdout"')
def test_basic(self):
# write python code into code.py file
tmp_path = tempfile.mkdtemp()
code_path = os.path.join(tmp_path, 'code.py')
with open(code_path, write_mode) as f:
f.write("@dxpy.entry_point('main')\n")
f.write("def main(**job_inputs):\n")
f.write("\toutput = {}\n")
f.write("\toutput['thresholds'] = job_inputs['thresholds']\n")
f.write("\toutput['pie'] = job_inputs['pie'] + 1\n")
f.write("\toutput['misc'] = {'n': 'non', 'y': 'oui'}\n")
f.write("\treturn output\n")
f.write("\n")
f.write("dxpy.run()\n")
with open(code_path, 'r') as f:
code = f.read()
# write arguments table
arg_table = os.path.join(tmp_path, 'table.csv')
with open(arg_table, write_mode) as csvfile:
writer = csv.writer(csvfile, delimiter=delimiter)
header = ["batch ID", "thresholds", "pie", "misc"]
writer.writerow(header)
writer.writerow(["SRR_1", "[10,81]", "3.12", "{}"])
applet = dxpy.api.applet_new({
"name": "copy_all",
"project": self.project,
"dxapi": "1.0.0",
"inputSpec": [ { "name": "thresholds", "class": "array:int"},
{ "name": "pie", "class": "float" },
{ "name": "misc", "class": "hash" } ],
"outputSpec": [ { "name": "thresholds", "class": "array:int" },
{ "name": "pie", "class": "float" },
{ "name": "misc", "class": "hash" } ],
"runSpec": { "interpreter": "python2.7",
"code": code,
"distribution": "Ubuntu",
"release": "14.04" }
})
# run in batch mode
job_id = run("dx run {} --batch-tsv={} --yes --brief"
.format(applet["id"], arg_table)).strip()
job_desc = dxpy.api.job_describe(job_id)
self.assertEqual(job_desc["executableName"], 'copy_all')
self.assertEqual(job_desc["input"], { "thresholds": [10,81],
"misc": {},
"pie": 3.12 })
# run in batch mode with --batch-folders
job_id = run("dx run {} --batch-tsv={} --batch-folders --yes --brief"
.format(applet["id"], arg_table)).strip()
job_desc = dxpy.api.job_describe(job_id)
self.assertEqual(job_desc["folder"], "/SRR_1")
# run in batch mode with --batch-folders and --destination
job_id = run("dx run {} --batch-tsv={} --batch-folders --destination={}:/run_01 --yes --brief"
.format(applet["id"], arg_table, self.project)).strip()
job_desc = dxpy.api.job_describe(job_id)
self.assertEqual(job_desc["folder"], "/run_01/SRR_1")
def test_dx_docker_run_from_hash(self):
repo = "dnanexus/testdocker@sha256:4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb"
run("dx-docker run {}".format(repo))
