def test_file_optional(self):
''' Tests that optional and non-optional file output arguments are
handled correctly '''
with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(
os.path.join(TEST_APPS, 'file_optional'),
dxproj.get_id())
# Run the applet. This checks a correct scenario where
# the applet generates:
# 1) an empty directory for an optional file output
# 2) a file for a non-optional file output.
applet_args = ["-icreate_seq3=true"]
cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
cmd_args.extend(applet_args)
job_id = run(cmd_args, env=env).strip()
dxpy.DXJob(job_id).wait_on_done()
# Run the applet --- this will not create the seq3 output file.
# This should cause an exception from the job manager.
applet_args = ["-icreate_seq3=false"]
cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
cmd_args.extend(applet_args)
job_id = run(cmd_args, env=env).strip()
job = dxpy.DXJob(job_id)
with self.assertRaises(DXJobFailureError):
job.wait_on_done()
desc = job.describe()
self.assertEqual(desc["failureReason"], "OutputError")
def test_prefix_patterns(self):
""" Tests that the bash prefix variable works correctly, and
respects patterns.
"""
with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
filenames = ["A.bar", "A.json.dot.bar", "A.vcf.pam", "A.foo.bar",
"fooxxx.bam", "A.bar.gz", "x13year23.sam"]
for fname in filenames:
dxpy.upload_string("1234", project=dxproj.get_id(), name=fname)
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'prefix_patterns'), dxproj.get_id())
# Run the applet
applet_args = ['-iseq1=A.bar',
'-iseq2=A.json.dot.bar',
'-igene=A.vcf.pam',
'-imap=A.foo.bar',
'-imap2=fooxxx.bam',
'-imap3=A.bar',
'-imap4=A.bar.gz',
'-imulti=x13year23.sam']
cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def test_vars(self):
''' Quick test for the bash variables '''
with temporary_project(
'TestDXBashHelpers.test_app1 temporary project') as p:
env = update_environ(DX_PROJECT_CONTEXT_ID=p.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=p.get_id(), name="A.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(
os.path.join(TEST_APPS, 'vars'), p.get_id())
# Run the applet
applet_args = [
'-iseq1=A.txt', '-iseq2=A.txt', '-igenes=A.txt',
'-igenes=A.txt', '-ii=5', '-ix=4.2', '-ib=true', '-is=hello',
'-iil=6', '-iil=7', '-iil=8', '-ixl=3.3', '-ixl=4.4',
'-ixl=5.0', '-ibl=true', '-ibl=false', '-ibl=true',
'-isl=hello', '-isl=world', '-isl=next',
'-imisc={"hello": "world", "foo": true}'
]
cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def test_file_optional(self):
''' Tests that optional and non-optional file output arguments are
handled correctly '''
with temporary_project(
'TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(
os.path.join(TEST_APPS, 'file_optional'), dxproj.get_id())
# Run the applet. This checks a correct scenario where
# the applet generates:
# 1) an empty directory for an optional file output
# 2) a file for a non-optional file output.
applet_args = ["-icreate_seq3=true"]
cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
cmd_args.extend(applet_args)
job_id = run(cmd_args, env=env).strip()
dxpy.DXJob(job_id).wait_on_done()
# Run the applet --- this will not create the seq3 output file.
# This should cause an exception from the job manager.
applet_args = ["-icreate_seq3=false"]
cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
cmd_args.extend(applet_args)
job_id = run(cmd_args, env=env).strip()
job = dxpy.DXJob(job_id)
with self.assertRaises(DXJobFailureError):
job.wait_on_done()
desc = job.describe()
self.assertEqual(desc["failureReason"], "OutputError")
def test_build_asset_with_valid_destination(self):
asset_spec = {
"name": "foo",
"title": "A human readable name",
"description": "A detailed description about the asset",
"version": "0.0.1",
"distribution": "Ubuntu",
"release": "14.04",
"execDepends": [{
"name": "python-numpy"
}]
}
asset_dir = self.write_asset_directory("asset_with_valid_destination",
json.dumps(asset_spec))
with testutil.temporary_project() as other_project:
test_dirname = 'asset_dir'
run('dx mkdir -p {project}:{dirname}'.format(
project=other_project.get_id(), dirname=test_dirname))
asset_bundle_id = json.loads(
run('dx build_asset --json --destination ' +
other_project.get_id() + ':/' + test_dirname + '/ ' +
asset_dir))['id']
self.assertIn('record', asset_bundle_id)
asset_desc = dxpy.describe(asset_bundle_id)
self.assertEqual(asset_desc['project'], other_project.get_id())
self.assertEqual(asset_desc['folder'], '/asset_dir')
def test_prefix_patterns(self):
""" Tests that the bash prefix variable works correctly, and
respects patterns.
"""
with temporary_project(
'TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
filenames = [
"A.bar", "A.json.dot.bar", "A.vcf.pam", "A.foo.bar",
"fooxxx.bam", "A.bar.gz", "x13year23.sam"
]
for fname in filenames:
dxpy.upload_string("1234", project=dxproj.get_id(), name=fname)
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(
os.path.join(TEST_APPS, 'prefix_patterns'), dxproj.get_id())
# Run the applet
applet_args = [
'-iseq1=A.bar', '-iseq2=A.json.dot.bar', '-igene=A.vcf.pam',
'-imap=A.foo.bar', '-imap2=fooxxx.bam', '-imap3=A.bar',
'-imap4=A.bar.gz', '-imulti=x13year23.sam'
]
cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def run_applet_with_flags(self, flag_list, num_files, file_size_bytes):
with temporary_project(
'TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Upload file
self.create_file_of_size("A.txt", file_size_bytes)
remote_file = dxpy.upload_local_file(filename="A.txt",
project=dxproj.get_id(),
folder='/')
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(
os.path.join(TEST_APPS, 'benchmark'), dxproj.get_id())
# Add several files to the output
applet_args = []
applet_args.extend(['-iref=A.txt'] * num_files)
cmd_args = [
'dx', 'run', '--yes', '--watch',
'--instance-type=mem1_ssd1_x2', applet_id
]
cmd_args.extend(applet_args)
cmd_args.extend(flag_list)
run(cmd_args, env=env)
def test_dx_docker_create_asset_with_long_imageid(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
long_id = run("docker images --no-trunc -q ubuntu:14.04").strip()
create_folder_in_project(test_projectid, '/testfolder')
run("dx-docker create-asset {long_id} -o testfolder".format(long_id=long_id))
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, long_id)
def test_dx_docker_create_asset_with_image_digest(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
create_folder_in_project(test_projectid, '/testfolder')
image_digest = run("docker inspect ubuntu:14.04 | jq -r '.[] | .RepoDigests[0]'").strip()
run("dx-docker create-asset {image_digest} -o testfolder".format(image_digest=image_digest))
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, image_digest)
def test_dx_docker_create_asset_with_image_digest(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
create_folder_in_project(test_projectid, '/testfolder')
image_digest = run("docker inspect ubuntu:14.04 | jq -r '.[] | .RepoDigests[0]'").strip()
run("dx-docker create-asset {image_digest} -o testfolder".format(image_digest=image_digest))
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, image_digest)
def test_dx_docker_create_asset_with_long_imageid(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
long_id = run("docker images --no-trunc -q ubuntu:14.04").strip()
create_folder_in_project(test_projectid, '/testfolder')
run("dx-docker create-asset {long_id} -o testfolder".format(long_id=long_id))
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, long_id)
def test_deepdirs(self):
""" Tests the use of subdirectories in the output directory """
def check_output_key(job_output, out_param_name, num_files, dxproj):
""" check that an output key appears, and has the correct number of files """
print("checking output for param={}".format(out_param_name))
if out_param_name not in job_output:
raise "Error: key {} does not appear in the job output".format(out_param_name)
dxlink_id_list = job_output[out_param_name]
if not len(dxlink_id_list) == num_files:
raise Exception(
"Error: key {} should have {} files, but has {}".format(
out_param_name, num_files, len(dxlink_id_list)
)
)
def verify_files_in_dir(path, expected_filenames, dxproj):
""" verify that a particular set of files resides in a directory """
dir_listing = dxproj.list_folder(folder=path, only="objects")
for elem in dir_listing["objects"]:
handler = dxpy.get_handler(elem["id"])
if not isinstance(handler, dxpy.DXFile):
continue
if handler.name not in expected_filenames:
raise Exception("Error: file {} should reside in directory {}".format(handler.name, path))
with temporary_project("TestDXBashHelpers.test_app1 temporary project") as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, "deepdirs"), dxproj.get_id())
# Run the applet
cmd_args = ["dx", "run", "--yes", "--brief", applet_id]
job_id = run(cmd_args, env=env).strip()
dxpy.DXJob(job_id).wait_on_done()
print("Test completed successfully, checking outputs\n")
# Assertions about the output. There should be three result keys
job_handler = dxpy.get_handler(job_id)
job_output = job_handler.output
check_output_key(job_output, "genes", 8, dxproj)
check_output_key(job_output, "phenotypes", 7, dxproj)
check_output_key(job_output, "report", 1, dxproj)
check_output_key(job_output, "helix", 1, dxproj)
verify_files_in_dir("/clue", ["X_1.txt", "X_2.txt", "X_3.txt"], dxproj)
verify_files_in_dir("/hint", ["V_1.txt", "V_2.txt", "V_3.txt"], dxproj)
verify_files_in_dir("/clue2", ["Y_1.txt", "Y_2.txt", "Y_3.txt"], dxproj)
verify_files_in_dir("/hint2", ["Z_1.txt", "Z_2.txt", "Z_3.txt"], dxproj)
verify_files_in_dir("/foo/bar", ["luke.txt"], dxproj)
verify_files_in_dir("/", ["A.txt", "B.txt", "C.txt", "num_chrom.txt"], dxproj)
def test_deepdirs(self):
''' Tests the use of subdirectories in the output directory '''
def check_output_key(job_output, out_param_name, num_files, dxproj):
''' check that an output key appears, and has the correct number of files '''
print('checking output for param={}'.format(out_param_name))
if out_param_name not in job_output:
raise "Error: key {} does not appear in the job output".format(out_param_name)
dxlink_id_list = job_output[out_param_name]
if not len(dxlink_id_list) == num_files:
raise Exception("Error: key {} should have {} files, but has {}".
format(out_param_name, num_files, len(dxlink_id_list)))
def verify_files_in_dir(path, expected_filenames, dxproj):
''' verify that a particular set of files resides in a directory '''
dir_listing = dxproj.list_folder(folder=path, only="objects")
for elem in dir_listing["objects"]:
handler = dxpy.get_handler(elem["id"])
if not isinstance(handler, dxpy.DXFile):
continue
if handler.name not in expected_filenames:
raise Exception("Error: file {} should reside in directory {}".
format(handler.name, path))
with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'deepdirs'), dxproj.get_id())
# Run the applet
cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
job_id = run(cmd_args, env=env).strip()
dxpy.DXJob(job_id).wait_on_done()
print("Test completed successfully, checking outputs\n")
# Assertions about the output. There should be three result keys
job_handler = dxpy.get_handler(job_id)
job_output = job_handler.output
check_output_key(job_output, "genes", 8, dxproj)
check_output_key(job_output, "phenotypes", 7, dxproj)
check_output_key(job_output, "report", 1, dxproj)
check_output_key(job_output, "helix", 1, dxproj)
verify_files_in_dir("/clue", ["X_1.txt", "X_2.txt", "X_3.txt"], dxproj)
verify_files_in_dir("/hint", ["V_1.txt", "V_2.txt", "V_3.txt"], dxproj)
verify_files_in_dir("/clue2", ["Y_1.txt", "Y_2.txt", "Y_3.txt"], dxproj)
verify_files_in_dir("/hint2", ["Z_1.txt", "Z_2.txt", "Z_3.txt"], dxproj)
verify_files_in_dir("/foo/bar", ["luke.txt"], dxproj)
verify_files_in_dir("/", ["A.txt", "B.txt", "C.txt", "num_chrom.txt"], dxproj)
def test_dx_docker_create_asset(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
run("dx-docker create-asset ubuntu:14.04")
self.assertEqual(run("dx ls ubuntu\\\\:14.04").strip(), 'ubuntu:14.04')
create_folder_in_project(test_projectid, '/testfolder')
run("dx-docker create-asset busybox -o testfolder")
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, 'busybox')
ls_out = run("dx ls testfolder\\/busybox.tar.gz").strip()
self.assertEqual(ls_out, 'busybox.tar.gz')
def test_dx_docker_create_asset(self):
with temporary_project(select=True) as temp_project:
test_projectid = temp_project.get_id()
run("docker pull ubuntu:14.04")
run("dx-docker create-asset ubuntu:14.04")
self.assertEqual(run("dx ls ubuntu\\\\:14.04").strip(), 'ubuntu:14.04')
create_folder_in_project(test_projectid, '/testfolder')
run("dx-docker create-asset busybox -o testfolder")
ls_out = run("dx ls /testfolder").strip()
self.assertEqual(ls_out, 'busybox')
ls_out = run("dx ls testfolder\\/busybox.tar.gz").strip()
self.assertEqual(ls_out, 'busybox.tar.gz')
def test_basic(self):
with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'basic'), dxproj.get_id())
# Run the applet
applet_args = ['-iseq1=A.txt', '-iseq2=B.txt', '-iref=A.txt', '-iref=B.txt', "-ivalue=5", "-iages=4"]
cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def test_basic(self):
with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'basic'), dxproj.get_id())
# Run the applet
applet_args = ['-iseq1=A.txt', '-iseq2=B.txt', '-iref=A.txt', '-iref=B.txt', "-ivalue=5", "-iages=4"]
cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def test_parseq(self):
""" Tests the parallel/sequential variations """
with temporary_project("TestDXBashHelpers.test_app1 temporary project") as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, "parseq"), dxproj.get_id())
# Run the applet
applet_args = ["-iseq1=A.txt", "-iseq2=B.txt", "-iref=A.txt", "-iref=B.txt"]
cmd_args = ["dx", "run", "--yes", "--watch", applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def run_applet_with_flags(self, flag_list, num_files, file_size_bytes):
with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Upload file
self.create_file_of_size("A.txt", file_size_bytes);
remote_file = dxpy.upload_local_file(filename="A.txt", project=dxproj.get_id(), folder='/')
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'benchmark'), dxproj.get_id())
# Add several files to the output
applet_args = []
applet_args.extend(['-iref=A.txt'] * num_files)
cmd_args = ['dx', 'run', '--yes', '--watch', '--instance-type=mem1_ssd1_x2', applet_id]
cmd_args.extend(applet_args)
cmd_args.extend(flag_list)
run(cmd_args, env=env)
def test_build_asset_with_valid_destination(self):
asset_spec = {
"name": "foo",
"title": "A human readable name",
"description": "A detailed description about the asset",
"version": "0.0.1",
"distribution": "Ubuntu",
"release": "12.04",
"execDepends": [{"name": "python-numpy"}]
}
asset_dir = self.write_asset_directory("asset_with_valid_destination", json.dumps(asset_spec))
with testutil.temporary_project() as other_project:
test_dirname = 'asset_dir'
run('dx mkdir -p {project}:{dirname}'.format(project=other_project.get_id(), dirname=test_dirname))
asset_bundle_id = json.loads(run('dx build_asset --json --destination ' + other_project.get_id() +
':/' + test_dirname + '/ ' + asset_dir))['id']
self.assertIn('record', asset_bundle_id)
asset_desc = dxpy.describe(asset_bundle_id)
self.assertEqual(asset_desc['project'], other_project.get_id())
self.assertEqual(asset_desc['folder'], '/asset_dir')
def test_vars(self):
""" Quick test for the bash variables """
with temporary_project("TestDXBashHelpers.test_app1 temporary project") as p:
env = update_environ(DX_PROJECT_CONTEXT_ID=p.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=p.get_id(), name="A.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, "vars"), p.get_id())
# Run the applet
applet_args = [
"-iseq1=A.txt",
"-iseq2=A.txt",
"-igenes=A.txt",
"-igenes=A.txt",
"-ii=5",
"-ix=4.2",
"-ib=true",
"-is=hello",
"-iil=6",
"-iil=7",
"-iil=8",
"-ixl=3.3",
"-ixl=4.4",
"-ixl=5.0",
"-ibl=true",
"-ibl=false",
"-ibl=true",
"-isl=hello",
"-isl=world",
"-isl=next",
'-imisc={"hello": "world", "foo": true}',
]
cmd_args = ["dx", "run", "--yes", "--watch", applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def test_xattr_parameters(self):
''' Tests dx-upload-all-outputs uploading with filesystem metadata as properties '''
with temporary_project(
'TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(
os.path.join(TEST_APPS, 'xattr_properties'), dxproj.get_id())
# Run the applet
applet_args = [
"-iseq1=A.txt", "-iseq2=B.txt", "-iref=A.txt", "-iref=B.txt"
]
cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def test_vars(self):
''' Quick test for the bash variables '''
with temporary_project('TestDXBashHelpers.test_app1 temporary project') as p:
env = update_environ(DX_PROJECT_CONTEXT_ID=p.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=p.get_id(), name="A.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'vars'), p.get_id())
# Run the applet
applet_args = ['-iseq1=A.txt', '-iseq2=A.txt', '-igenes=A.txt', '-igenes=A.txt',
'-ii=5', '-ix=4.2', '-ib=true', '-is=hello',
'-iil=6', '-iil=7', '-iil=8',
'-ixl=3.3', '-ixl=4.4', '-ixl=5.0',
'-ibl=true', '-ibl=false', '-ibl=true',
'-isl=hello', '-isl=world', '-isl=next',
'-imisc={"hello": "world", "foo": true}']
cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
cmd_args.extend(applet_args)
run(cmd_args, env=env)
def test_sub_jobs(self):
''' Tests a bash script that generates sub-jobs '''
with temporary_project(
'TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(
os.path.join(TEST_APPS, 'with-subjobs'), dxproj.get_id())
# Run the applet.
# Since the job creates two sub-jobs, we need to be a bit more sophisticated
# in order to wait for completion.
applet_args = ["-ifiles=A.txt", "-ifiles=B.txt"]
cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
cmd_args.extend(applet_args)
job_id = run(cmd_args, env=env).strip()
dxpy.DXJob(job_id).wait_on_done()
# Assertions -- making sure the script worked
# Assertions to make about the job's output after it is done running:
# - *first_file* is a file named first_file.txt containing the string:
# "contents of first_file"
# - *final_file* is a file named final_file.txt containing the
# *concatenation of the two input files in *files*
print("Test completed successfully, checking file content\n")
job_handler = dxpy.get_handler(job_id)
job_output = job_handler.output
def strip_white_space(_str):
return ''.join(_str.split())
def silent_file_remove(filename):
try:
os.remove(filename)
except OSError:
pass
# The output should include two files, this section verifies that they have
# the correct data.
def check_file_content(out_param_name, out_filename, tmp_fname,
str_content):
"""
Download a file, read it from local disk, and verify that it has the correct contents
"""
if not out_param_name in job_output:
raise "Error: key {} does not appear in the job output".format(
out_param_name)
dxlink = job_output[out_param_name]
# check that the filename gets preserved
trg_fname = dxpy.get_handler(dxlink).name
self.assertEqual(trg_fname, out_filename)
# download the file and check the contents
silent_file_remove(tmp_fname)
dxpy.download_dxfile(dxlink, tmp_fname)
with open(tmp_fname, "r") as fh:
data = fh.read()
print(data)
if not (strip_white_space(data)
== strip_white_space(str_content)):
raise Exception(
"contents of file {} do not match".format(
out_param_name))
silent_file_remove(tmp_fname)
check_file_content('first_file', 'first_file.txt', "f1.txt",
"contents of first_file")
check_file_content('final_file', 'final_file.txt', "f2.txt",
"1234ABCD")
def setUpClass(cls):
with testutil.temporary_project(name='dx-jobutil-new-job test project',
cleanup=False) as p:
cls.aux_project = p
def test_sub_jobs(self):
''' Tests a bash script that generates sub-jobs '''
with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
# Upload some files for use by the applet
dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")
# Build the applet, patching in the bash helpers from the
# local checkout
applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'with-subjobs'), dxproj.get_id())
# Run the applet.
# Since the job creates two sub-jobs, we need to be a bit more sophisticated
# in order to wait for completion.
applet_args = ["-ifiles=A.txt", "-ifiles=B.txt"]
cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
cmd_args.extend(applet_args)
job_id = run(cmd_args, env=env).strip()
dxpy.DXJob(job_id).wait_on_done()
# Assertions -- making sure the script worked
# Assertions to make about the job's output after it is done running:
# - *first_file* is a file named first_file.txt containing the string:
# "contents of first_file"
# - *final_file* is a file named final_file.txt containing the
# *concatenation of the two input files in *files*
print("Test completed successfully, checking file content\n")
job_handler = dxpy.get_handler(job_id)
job_output = job_handler.output
def strip_white_space(_str):
return ''.join(_str.split())
def silent_file_remove(filename):
try:
os.remove(filename)
except OSError:
pass
# The output should include two files, this section verifies that they have
# the correct data.
def check_file_content(out_param_name, out_filename, tmp_fname, str_content):
"""
Download a file, read it from local disk, and verify that it has the correct contents
"""
if not out_param_name in job_output:
raise "Error: key {} does not appear in the job output".format(out_param_name)
dxlink = job_output[out_param_name]
# check that the filename gets preserved
trg_fname = dxpy.get_handler(dxlink).name
self.assertEqual(trg_fname, out_filename)
# download the file and check the contents
silent_file_remove(tmp_fname)
dxpy.download_dxfile(dxlink, tmp_fname)
with open(tmp_fname, "r") as fh:
data = fh.read()
print(data)
if not (strip_white_space(data) == strip_white_space(str_content)):
raise Exception("contents of file {} do not match".format(out_param_name))
silent_file_remove(tmp_fname)
check_file_content('first_file', 'first_file.txt', "f1.txt", "contents of first_file")
check_file_content('final_file', 'final_file.txt', "f2.txt", "1234ABCD")
def setUpClass(cls):
with testutil.temporary_project(name='dx-jobutil-new-job test project', cleanup=False) as p:
cls.aux_project = p