def test_dx_docker_pull_hash_or_not(self): run("dx-docker pull dnanexus/testdocker") self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'dnanexus%2Ftestdocker.aci'))) repo = "dnanexus/testdocker@sha256:4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb" run("dx-docker pull {}".format(repo)) sanit='dnanexus%2Ftestdocker%40sha256%3A4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb.aci' self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, sanit)))
def test_dx_docker_volume(self): os.makedirs('dxdtestdata') run("dx-docker run -v dxdtestdata:/data-host ubuntu:14.04 touch /data-host/newfile.txt" ) self.assertTrue( os.path.isfile(os.path.join('dxdtestdata', 'newfile.txt'))) shutil.rmtree('dxdtestdata')
def test_dx_docker_pull(self): run("dx-docker pull ubuntu:14.04") self.assertTrue( os.path.isfile(os.path.join(CACHE_DIR, 'ubuntu%3A14.04.aci'))) run("dx-docker pull ubuntu:15.04") self.assertTrue( os.path.isfile(os.path.join(CACHE_DIR, 'ubuntu%3A15.04.aci')))
def test_file_arrays(self): # Create file with junk content dxfile = dxpy.upload_string("xxyyzz", project=self.project, wait_on_close=True, name="bubbles") # write python code into code.py file tmp_path = tempfile.mkdtemp() code_path = os.path.join(tmp_path, 'code.py') with open(code_path, write_mode) as f: f.write("@dxpy.entry_point('main')\n") f.write("def main(**job_inputs):\n") f.write("\toutput = {}\n") f.write("\toutput['plant'] = job_inputs['plant']\n") f.write("\treturn output\n") f.write("\n") f.write("dxpy.run()\n") with open(code_path, 'r') as f: code = f.read() # write arguments table. These ara arrays with a single element. arg_table = os.path.join(tmp_path, 'table.csv') with open(arg_table, write_mode) as csvfile: writer = csv.writer(csvfile, delimiter=delimiter) header = ["batch ID", "plant", "plant ID"] writer.writerow(header) writer.writerow( ["SRR_1", "[bubbles]", "[" + dxfile.get_id() + "]"]) applet = dxpy.api.applet_new({ "name": "ident_file_array", "project": self.project, "dxapi": "1.0.0", "inputSpec": [{ "name": "plant", "class": "array:file" }], "outputSpec": [{ "name": "plant", "class": "array:file" }], "runSpec": { "interpreter": "python2.7", "code": code, "distribution": "Ubuntu", "release": "14.04" } }) job_id = run("dx run {} --batch-tsv={} --yes --brief".format( applet["id"], arg_table)).strip() job_desc = dxpy.api.job_describe(job_id) self.assertEqual(job_desc["executableName"], 'ident_file_array') self.assertEqual(job_desc["input"], {"plant": [{ "$dnanexus_link": dxfile.get_id() }]})
def test_dx_docker_create_asset(self): with temporary_project(select=True) as temp_project: test_projectid = temp_project.get_id() run("docker pull ubuntu:14.04") run("dx-docker create-asset ubuntu:14.04") self.assertEqual(run("dx ls ubuntu\\\\:14.04").strip(), 'ubuntu:14.04') create_folder_in_project(test_projectid, '/testfolder') run("dx-docker create-asset busybox -o testfolder") ls_out = run("dx ls /testfolder").strip() self.assertEqual(ls_out, 'busybox') ls_out = run("dx ls testfolder\\/busybox.tar.gz").strip() self.assertEqual(ls_out, 'busybox.tar.gz')
def test_dx_docker_create_asset_with_image_digest(self): with temporary_project(select=True) as temp_project: test_projectid = temp_project.get_id() run("docker pull ubuntu:14.04") create_folder_in_project(test_projectid, '/testfolder') image_digest = run("docker inspect ubuntu:14.04 | jq -r '.[] | .RepoDigests[0]'").strip() run("dx-docker create-asset {image_digest} -o testfolder".format(image_digest=image_digest)) ls_out = run("dx ls /testfolder").strip() self.assertEqual(ls_out, image_digest)
def test_dx_docker_create_asset_with_long_imageid(self): with temporary_project(select=True) as temp_project: test_projectid = temp_project.get_id() run("docker pull ubuntu:14.04") long_id = run("docker images --no-trunc -q ubuntu:14.04").strip() create_folder_in_project(test_projectid, '/testfolder') run("dx-docker create-asset {long_id} -o testfolder".format(long_id=long_id)) ls_out = run("dx ls /testfolder").strip() self.assertEqual(ls_out, long_id)
def test_file_arrays(self): # Create file with junk content dxfile = dxpy.upload_string("xxyyzz", project=self.project, wait_on_close=True, name="bubbles") # write python code into code.py file tmp_path = tempfile.mkdtemp() code_path = os.path.join(tmp_path, 'code.py') with open(code_path, write_mode) as f: f.write("@dxpy.entry_point('main')\n") f.write("def main(**job_inputs):\n") f.write("\toutput = {}\n") f.write("\toutput['plant'] = job_inputs['plant']\n") f.write("\treturn output\n") f.write("\n") f.write("dxpy.run()\n") with open(code_path, 'r') as f: code = f.read() # write arguments table. These ara arrays with a single element. arg_table = os.path.join(tmp_path, 'table.csv') with open(arg_table, write_mode) as csvfile: writer = csv.writer(csvfile, delimiter=delimiter) header = ["batch ID", "plant", "plant ID"] writer.writerow(header) writer.writerow(["SRR_1", "[bubbles]", "[" + dxfile.get_id() + "]" ]) applet = dxpy.api.applet_new({ "name": "ident_file_array", "project": self.project, "dxapi": "1.0.0", "inputSpec": [ { "name": "plant", "class": "array:file" } ], "outputSpec": [ { "name": "plant", "class": "array:file" } ], "runSpec": { "interpreter": "python2.7", "code": code, "distribution": "Ubuntu", "release": "14.04" } }) job_id = run("dx run {} --batch-tsv={} --yes --brief" .format(applet["id"], arg_table)).strip() job_desc = dxpy.api.job_describe(job_id) self.assertEqual(job_desc["executableName"], 'ident_file_array') self.assertEqual(job_desc["input"], { "plant": [{ "$dnanexus_link": dxfile.get_id() }] })
def test_dx_docker_add_to_applet(self): os.makedirs('tmpapp') run("docker pull busybox") with self.assertSubprocessFailure(exit_code=1, stderr_regexp='does not appear to have a dxapp.json that parses'): run("dx-docker add-to-applet busybox tmpapp") with open('tmpapp/dxapp.json', 'w') as dxapp: dxapp.write("[]") run("dx-docker add-to-applet busybox tmpapp") self.assertTrue(os.path.isfile(os.path.join('tmpapp', 'resources/tmp/dx-docker-cache/busybox.aci'))) shutil.rmtree('tmpapp')
def test_symlinks(self): dxfile1 = self.download_url_create_symlink("https://s3.amazonaws.com/1000genomes/CHANGELOG", "sym1") dxfile2 = self.download_url_create_symlink("https://documentation.dnanexus.com/", "sym2") # download to PWD run("dx download {}:/{} -o {}".format(self.proj_id, "sym1", "localfile")) # absolute path run("dx download {}:/{} -f -o {}".format(self.proj_id, "sym1", "/tmp/localfile")) # relative path run("dx download {}:/{} -f -o {}".format(self.proj_id, "sym2", "../localfile"))
def test_symlinks(self): dxfile1 = self.download_url_create_symlink( "https://s3.amazonaws.com/1000genomes/CHANGELOG", "sym1") dxfile2 = self.download_url_create_symlink( "https://en.wikipedia.org/wiki/Poitevin_horse", "sym2") # download to PWD run("dx download {}:/{} -o {}".format(self.proj_id, "sym1", "localfile")) # absolute path run("dx download {}:/{} -f -o {}".format(self.proj_id, "sym1", "/tmp/localfile")) # relative path run("dx download {}:/{} -f -o {}".format(self.proj_id, "sym2", "../localfile"))
def test_dx_docker_volume(self): os.makedirs('dxdtestdata') run("dx-docker run -v dxdtestdata:/data-host ubuntu:14.04 touch /data-host/newfile.txt") self.assertTrue(os.path.isfile(os.path.join('dxdtestdata', 'newfile.txt'))) shutil.rmtree('dxdtestdata')
def test_dx_docker_pull_failure(self): with self.assertSubprocessFailure( exit_code=1, stderr_regexp='Failed to obtain image'): run("dx-docker pull busyboxasdf")
def test_dx_docker_additional_container(self): run("dx-docker run busybox ls")
def test_complex_quote(self): run('dx-docker run python:2-slim /bin/sh -c "echo \'{"foo": {"location": "file:///"}}\' > /dev/stdout"' )
def test_dx_docker_entrypoint_cmd(self): docker_out = run("docker run dnanexus/testdocker /bin") dx_docker_out = run("dx-docker run -q dnanexus/testdocker /bin") self.assertEqual(docker_out, dx_docker_out)
def test_dx_docker_run_rm(self): run("dx-docker run --rm ubuntu ls")
def test_dx_docker_basic_commands(self): run("dx-docker run ubuntu:14.04 ls --color") run("dx-docker run ubuntu:15.04 ls")
def test_dx_docker_run_error_codes(self): with self.assertSubprocessFailure(exit_code=1): run("dx-docker run ubuntu:14.04 false") run("dx-docker run ubuntu:14.04 true")
def setUpClass(cls): run("docker pull ubuntu:14.04") run("docker pull busybox")
def test_dx_docker_pull_failure(self): with self.assertSubprocessFailure(exit_code=1, stderr_regexp='Failed to obtain image'): run("dx-docker pull busyboxasdf")
def test_dx_docker_working_dir_override(self): run("dx-docker run -v $PWD:/tmp -w /tmp quay.io/ucsc_cgl/samtools faidx test.fa")
def test_dx_docker_pull_quay(self): run("dx-docker pull quay.io/ucsc_cgl/samtools") self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'quay.io%2Fucsc_cgl%2Fsamtools.aci')))
def test_dx_docker_pull_silent(self): dx_docker_out = run("dx-docker pull -q busybox").strip() self.assertEqual(dx_docker_out, '')
def test_dx_docker_pull(self): run("dx-docker pull ubuntu:14.04") self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'ubuntu%3A14.04.aci'))) run("dx-docker pull ubuntu:15.04") self.assertTrue(os.path.isfile(os.path.join(CACHE_DIR, 'ubuntu%3A15.04.aci')))
def test_basic(self): # write python code into code.py file tmp_path = tempfile.mkdtemp() code_path = os.path.join(tmp_path, 'code.py') with open(code_path, write_mode) as f: f.write("@dxpy.entry_point('main')\n") f.write("def main(**job_inputs):\n") f.write("\toutput = {}\n") f.write("\toutput['thresholds'] = job_inputs['thresholds']\n") f.write("\toutput['pie'] = job_inputs['pie'] + 1\n") f.write("\toutput['misc'] = {'n': 'non', 'y': 'oui'}\n") f.write("\treturn output\n") f.write("\n") f.write("dxpy.run()\n") with open(code_path, 'r') as f: code = f.read() # write arguments table arg_table = os.path.join(tmp_path, 'table.csv') with open(arg_table, write_mode) as csvfile: writer = csv.writer(csvfile, delimiter=delimiter) header = ["batch ID", "thresholds", "pie", "misc"] writer.writerow(header) writer.writerow(["SRR_1", "[10,81]", "3.12", "{}"]) applet = dxpy.api.applet_new({ "name": "copy_all", "project": self.project, "dxapi": "1.0.0", "inputSpec": [{ "name": "thresholds", "class": "array:int" }, { "name": "pie", "class": "float" }, { "name": "misc", "class": "hash" }], "outputSpec": [{ "name": "thresholds", "class": "array:int" }, { "name": "pie", "class": "float" }, { "name": "misc", "class": "hash" }], "runSpec": { "interpreter": "python2.7", "code": code, "distribution": "Ubuntu", "release": "14.04" } }) # run in batch mode job_id = run("dx run {} --batch-tsv={} --yes --brief".format( applet["id"], arg_table)).strip() job_desc = dxpy.api.job_describe(job_id) self.assertEqual(job_desc["executableName"], 'copy_all') self.assertEqual(job_desc["input"], { "thresholds": [10, 81], "misc": {}, "pie": 3.12 }) # run in batch mode with --batch-folders job_id = run( "dx run {} --batch-tsv={} --batch-folders --yes --brief".format( applet["id"], arg_table)).strip() job_desc = dxpy.api.job_describe(job_id) self.assertEqual(job_desc["folder"], "/SRR_1") # run in batch mode with --batch-folders and --destination job_id = run( "dx run {} --batch-tsv={} --batch-folders --destination={}:/run_01 --yes --brief" .format(applet["id"], arg_table, self.project)).strip() job_desc = dxpy.api.job_describe(job_id) self.assertEqual(job_desc["folder"], "/run_01/SRR_1")
def test_dx_docker_run_canonical(self): run("dx-docker run quay.io/ucsc_cgl/samtools --help")
def test_dx_docker_run_from_hash(self): repo = "dnanexus/testdocker@sha256:4f983c07e762f5afadf9c45ccd6a557e1a414460e769676826b01c99c4ccb1cb" run("dx-docker run {}".format(repo))
def test_dx_docker_home_dir(self): run("dx-docker run julia:0.5.0 julia -E 'println(\"hello world\")'")
def test_dx_docker_set_env(self): dx_docker_out = run( "dx-docker run --env HOME=/somethingelse busybox env") self.assertTrue(dx_docker_out.find("HOME=/somethingelse") != -1)
def test_dx_docker_set_env(self): dx_docker_out = run("dx-docker run --env HOME=/somethingelse busybox env") self.assertTrue(dx_docker_out.find("HOME=/somethingelse") != -1)
def test_dx_docker_working_dir_override(self): run("dx-docker run -v $PWD:/tmp -w /tmp busybox ls")
def test_complex_quote(self): run('dx-docker run python:2-slim /bin/sh -c "echo \'{"foo": {"location": "file:///"}}\' > /dev/stdout"')
def test_basic(self): # write python code into code.py file tmp_path = tempfile.mkdtemp() code_path = os.path.join(tmp_path, 'code.py') with open(code_path, write_mode) as f: f.write("@dxpy.entry_point('main')\n") f.write("def main(**job_inputs):\n") f.write("\toutput = {}\n") f.write("\toutput['thresholds'] = job_inputs['thresholds']\n") f.write("\toutput['pie'] = job_inputs['pie'] + 1\n") f.write("\toutput['misc'] = {'n': 'non', 'y': 'oui'}\n") f.write("\treturn output\n") f.write("\n") f.write("dxpy.run()\n") with open(code_path, 'r') as f: code = f.read() # write arguments table arg_table = os.path.join(tmp_path, 'table.csv') with open(arg_table, write_mode) as csvfile: writer = csv.writer(csvfile, delimiter=delimiter) header = ["batch ID", "thresholds", "pie", "misc"] writer.writerow(header) writer.writerow(["SRR_1", "[10,81]", "3.12", "{}"]) applet = dxpy.api.applet_new({ "name": "copy_all", "project": self.project, "dxapi": "1.0.0", "inputSpec": [ { "name": "thresholds", "class": "array:int"}, { "name": "pie", "class": "float" }, { "name": "misc", "class": "hash" } ], "outputSpec": [ { "name": "thresholds", "class": "array:int" }, { "name": "pie", "class": "float" }, { "name": "misc", "class": "hash" } ], "runSpec": { "interpreter": "python2.7", "code": code, "distribution": "Ubuntu", "release": "14.04" } }) # run in batch mode job_id = run("dx run {} --batch-tsv={} --yes --brief" .format(applet["id"], arg_table)).strip() job_desc = dxpy.api.job_describe(job_id) self.assertEqual(job_desc["executableName"], 'copy_all') self.assertEqual(job_desc["input"], { "thresholds": [10,81], "misc": {}, "pie": 3.12 }) # run in batch mode with --batch-folders job_id = run("dx run {} --batch-tsv={} --batch-folders --yes --brief" .format(applet["id"], arg_table)).strip() job_desc = dxpy.api.job_describe(job_id) self.assertEqual(job_desc["folder"], "/SRR_1") # run in batch mode with --batch-folders and --destination job_id = run("dx run {} --batch-tsv={} --batch-folders --destination={}:/run_01 --yes --brief" .format(applet["id"], arg_table, self.project)).strip() job_desc = dxpy.api.job_describe(job_id) self.assertEqual(job_desc["folder"], "/run_01/SRR_1")