예제 #1
0
    def test_file_optional(self):
        ''' Tests that optional and non-optional file output arguments are
        handled correctly '''
        with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(
                os.path.join(TEST_APPS, 'file_optional'),
                dxproj.get_id())

            # Run the applet. This checks a correct scenario where
            # the applet generates:
            # 1) an empty directory for an optional file output
            # 2) a file for a non-optional file output.
            applet_args = ["-icreate_seq3=true"]
            cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
            cmd_args.extend(applet_args)
            job_id = run(cmd_args, env=env).strip()
            dxpy.DXJob(job_id).wait_on_done()

            # Run the applet --- this will not create the seq3 output file.
            # This should cause an exception from the job manager.
            applet_args = ["-icreate_seq3=false"]
            cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
            cmd_args.extend(applet_args)
            job_id = run(cmd_args, env=env).strip()
            job = dxpy.DXJob(job_id)
            with self.assertRaises(DXJobFailureError):
                job.wait_on_done()
            desc = job.describe()
            self.assertEqual(desc["failureReason"], "OutputError")
예제 #2
0
    def test_prefix_patterns(self):
        """ Tests that the bash prefix variable works correctly, and
        respects patterns.
        """
        with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
            filenames = ["A.bar", "A.json.dot.bar", "A.vcf.pam", "A.foo.bar",
                         "fooxxx.bam", "A.bar.gz", "x13year23.sam"]
            for fname in filenames:
                dxpy.upload_string("1234", project=dxproj.get_id(), name=fname)

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'prefix_patterns'), dxproj.get_id())

            # Run the applet
            applet_args = ['-iseq1=A.bar',
                           '-iseq2=A.json.dot.bar',
                           '-igene=A.vcf.pam',
                           '-imap=A.foo.bar',
                           '-imap2=fooxxx.bam',
                           '-imap3=A.bar',
                           '-imap4=A.bar.gz',
                           '-imulti=x13year23.sam']
            cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
예제 #3
0
    def test_vars(self):
        '''  Quick test for the bash variables '''
        with temporary_project(
                'TestDXBashHelpers.test_app1 temporary project') as p:
            env = update_environ(DX_PROJECT_CONTEXT_ID=p.get_id())

            # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=p.get_id(), name="A.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(
                os.path.join(TEST_APPS, 'vars'), p.get_id())

            # Run the applet
            applet_args = [
                '-iseq1=A.txt', '-iseq2=A.txt', '-igenes=A.txt',
                '-igenes=A.txt', '-ii=5', '-ix=4.2', '-ib=true', '-is=hello',
                '-iil=6', '-iil=7', '-iil=8', '-ixl=3.3', '-ixl=4.4',
                '-ixl=5.0', '-ibl=true', '-ibl=false', '-ibl=true',
                '-isl=hello', '-isl=world', '-isl=next',
                '-imisc={"hello": "world", "foo": true}'
            ]
            cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
    def test_file_optional(self):
        ''' Tests that optional and non-optional file output arguments are
        handled correctly '''
        with temporary_project(
                'TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(
                os.path.join(TEST_APPS, 'file_optional'), dxproj.get_id())

            # Run the applet. This checks a correct scenario where
            # the applet generates:
            # 1) an empty directory for an optional file output
            # 2) a file for a non-optional file output.
            applet_args = ["-icreate_seq3=true"]
            cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
            cmd_args.extend(applet_args)
            job_id = run(cmd_args, env=env).strip()
            dxpy.DXJob(job_id).wait_on_done()

            # Run the applet --- this will not create the seq3 output file.
            # This should cause an exception from the job manager.
            applet_args = ["-icreate_seq3=false"]
            cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
            cmd_args.extend(applet_args)
            job_id = run(cmd_args, env=env).strip()
            job = dxpy.DXJob(job_id)
            with self.assertRaises(DXJobFailureError):
                job.wait_on_done()
            desc = job.describe()
            self.assertEqual(desc["failureReason"], "OutputError")
예제 #5
0
 def test_build_asset_with_valid_destination(self):
     asset_spec = {
         "name": "foo",
         "title": "A human readable name",
         "description": "A detailed description about the asset",
         "version": "0.0.1",
         "distribution": "Ubuntu",
         "release": "14.04",
         "execDepends": [{
             "name": "python-numpy"
         }]
     }
     asset_dir = self.write_asset_directory("asset_with_valid_destination",
                                            json.dumps(asset_spec))
     with testutil.temporary_project() as other_project:
         test_dirname = 'asset_dir'
         run('dx mkdir -p {project}:{dirname}'.format(
             project=other_project.get_id(), dirname=test_dirname))
         asset_bundle_id = json.loads(
             run('dx build_asset --json --destination ' +
                 other_project.get_id() + ':/' + test_dirname + '/ ' +
                 asset_dir))['id']
         self.assertIn('record', asset_bundle_id)
         asset_desc = dxpy.describe(asset_bundle_id)
         self.assertEqual(asset_desc['project'], other_project.get_id())
         self.assertEqual(asset_desc['folder'], '/asset_dir')
예제 #6
0
    def test_prefix_patterns(self):
        """ Tests that the bash prefix variable works correctly, and
        respects patterns.
        """
        with temporary_project(
                'TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())
            filenames = [
                "A.bar", "A.json.dot.bar", "A.vcf.pam", "A.foo.bar",
                "fooxxx.bam", "A.bar.gz", "x13year23.sam"
            ]
            for fname in filenames:
                dxpy.upload_string("1234", project=dxproj.get_id(), name=fname)

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(
                os.path.join(TEST_APPS, 'prefix_patterns'), dxproj.get_id())

            # Run the applet
            applet_args = [
                '-iseq1=A.bar', '-iseq2=A.json.dot.bar', '-igene=A.vcf.pam',
                '-imap=A.foo.bar', '-imap2=fooxxx.bam', '-imap3=A.bar',
                '-imap4=A.bar.gz', '-imulti=x13year23.sam'
            ]
            cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
예제 #7
0
    def run_applet_with_flags(self, flag_list, num_files, file_size_bytes):
        with temporary_project(
                'TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Upload file
            self.create_file_of_size("A.txt", file_size_bytes)
            remote_file = dxpy.upload_local_file(filename="A.txt",
                                                 project=dxproj.get_id(),
                                                 folder='/')

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(
                os.path.join(TEST_APPS, 'benchmark'), dxproj.get_id())

            # Add several files to the output
            applet_args = []
            applet_args.extend(['-iref=A.txt'] * num_files)
            cmd_args = [
                'dx', 'run', '--yes', '--watch',
                '--instance-type=mem1_ssd1_x2', applet_id
            ]
            cmd_args.extend(applet_args)
            cmd_args.extend(flag_list)
            run(cmd_args, env=env)
예제 #8
0
 def test_dx_docker_create_asset_with_long_imageid(self):
     with temporary_project(select=True) as temp_project:
         test_projectid = temp_project.get_id()
         run("docker pull ubuntu:14.04")
         long_id = run("docker images --no-trunc -q ubuntu:14.04").strip()
         create_folder_in_project(test_projectid, '/testfolder')
         run("dx-docker create-asset {long_id} -o testfolder".format(long_id=long_id))
         ls_out = run("dx ls /testfolder").strip()
         self.assertEqual(ls_out, long_id)
예제 #9
0
 def test_dx_docker_create_asset_with_image_digest(self):
     with temporary_project(select=True) as temp_project:
         test_projectid = temp_project.get_id()
         run("docker pull ubuntu:14.04")
         create_folder_in_project(test_projectid, '/testfolder')
         image_digest = run("docker inspect ubuntu:14.04 | jq -r '.[] | .RepoDigests[0]'").strip()
         run("dx-docker create-asset {image_digest} -o testfolder".format(image_digest=image_digest))
         ls_out = run("dx ls /testfolder").strip()
         self.assertEqual(ls_out, image_digest)
예제 #10
0
 def test_dx_docker_create_asset_with_image_digest(self):
     with temporary_project(select=True) as temp_project:
         test_projectid = temp_project.get_id()
         run("docker pull ubuntu:14.04")
         create_folder_in_project(test_projectid, '/testfolder')
         image_digest = run("docker inspect ubuntu:14.04 | jq -r '.[] | .RepoDigests[0]'").strip()
         run("dx-docker create-asset {image_digest} -o testfolder".format(image_digest=image_digest))
         ls_out = run("dx ls /testfolder").strip()
         self.assertEqual(ls_out, image_digest)
예제 #11
0
 def test_dx_docker_create_asset_with_long_imageid(self):
     with temporary_project(select=True) as temp_project:
         test_projectid = temp_project.get_id()
         run("docker pull ubuntu:14.04")
         long_id = run("docker images --no-trunc -q ubuntu:14.04").strip()
         create_folder_in_project(test_projectid, '/testfolder')
         run("dx-docker create-asset {long_id} -o testfolder".format(long_id=long_id))
         ls_out = run("dx ls /testfolder").strip()
         self.assertEqual(ls_out, long_id)
예제 #12
0
    def test_deepdirs(self):
        """ Tests the use of subdirectories in the output directory """

        def check_output_key(job_output, out_param_name, num_files, dxproj):
            """ check that an output key appears, and has the correct number of files """
            print("checking output for param={}".format(out_param_name))
            if out_param_name not in job_output:
                raise "Error: key {} does not appear in the job output".format(out_param_name)
            dxlink_id_list = job_output[out_param_name]
            if not len(dxlink_id_list) == num_files:
                raise Exception(
                    "Error: key {} should have {} files, but has {}".format(
                        out_param_name, num_files, len(dxlink_id_list)
                    )
                )

        def verify_files_in_dir(path, expected_filenames, dxproj):
            """ verify that a particular set of files resides in a directory """
            dir_listing = dxproj.list_folder(folder=path, only="objects")
            for elem in dir_listing["objects"]:
                handler = dxpy.get_handler(elem["id"])
                if not isinstance(handler, dxpy.DXFile):
                    continue
                if handler.name not in expected_filenames:
                    raise Exception("Error: file {} should reside in directory {}".format(handler.name, path))

        with temporary_project("TestDXBashHelpers.test_app1 temporary project") as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, "deepdirs"), dxproj.get_id())

            # Run the applet
            cmd_args = ["dx", "run", "--yes", "--brief", applet_id]
            job_id = run(cmd_args, env=env).strip()

            dxpy.DXJob(job_id).wait_on_done()

            print("Test completed successfully, checking outputs\n")

            # Assertions about the output. There should be three result keys
            job_handler = dxpy.get_handler(job_id)
            job_output = job_handler.output

            check_output_key(job_output, "genes", 8, dxproj)
            check_output_key(job_output, "phenotypes", 7, dxproj)
            check_output_key(job_output, "report", 1, dxproj)
            check_output_key(job_output, "helix", 1, dxproj)

            verify_files_in_dir("/clue", ["X_1.txt", "X_2.txt", "X_3.txt"], dxproj)
            verify_files_in_dir("/hint", ["V_1.txt", "V_2.txt", "V_3.txt"], dxproj)
            verify_files_in_dir("/clue2", ["Y_1.txt", "Y_2.txt", "Y_3.txt"], dxproj)
            verify_files_in_dir("/hint2", ["Z_1.txt", "Z_2.txt", "Z_3.txt"], dxproj)
            verify_files_in_dir("/foo/bar", ["luke.txt"], dxproj)
            verify_files_in_dir("/", ["A.txt", "B.txt", "C.txt", "num_chrom.txt"], dxproj)
예제 #13
0
    def test_deepdirs(self):
        ''' Tests the use of subdirectories in the output directory '''
        def check_output_key(job_output, out_param_name, num_files, dxproj):
            ''' check that an output key appears, and has the correct number of files '''
            print('checking output for param={}'.format(out_param_name))
            if out_param_name not in job_output:
                raise "Error: key {} does not appear in the job output".format(out_param_name)
            dxlink_id_list = job_output[out_param_name]
            if not len(dxlink_id_list) == num_files:
                raise Exception("Error: key {} should have {} files, but has {}".
                                format(out_param_name, num_files, len(dxlink_id_list)))

        def verify_files_in_dir(path, expected_filenames, dxproj):
            ''' verify that a particular set of files resides in a directory '''
            dir_listing = dxproj.list_folder(folder=path, only="objects")
            for elem in dir_listing["objects"]:
                handler = dxpy.get_handler(elem["id"])
                if not isinstance(handler, dxpy.DXFile):
                    continue
                if handler.name not in expected_filenames:
                    raise Exception("Error: file {} should reside in directory {}".
                                    format(handler.name, path))

        with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'deepdirs'), dxproj.get_id())

            # Run the applet
            cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
            job_id = run(cmd_args, env=env).strip()

            dxpy.DXJob(job_id).wait_on_done()

            print("Test completed successfully, checking outputs\n")

            # Assertions about the output. There should be three result keys
            job_handler = dxpy.get_handler(job_id)
            job_output = job_handler.output

            check_output_key(job_output, "genes", 8, dxproj)
            check_output_key(job_output, "phenotypes", 7, dxproj)
            check_output_key(job_output, "report", 1, dxproj)
            check_output_key(job_output, "helix", 1, dxproj)

            verify_files_in_dir("/clue", ["X_1.txt", "X_2.txt", "X_3.txt"], dxproj)
            verify_files_in_dir("/hint", ["V_1.txt", "V_2.txt", "V_3.txt"], dxproj)
            verify_files_in_dir("/clue2", ["Y_1.txt", "Y_2.txt", "Y_3.txt"], dxproj)
            verify_files_in_dir("/hint2", ["Z_1.txt", "Z_2.txt", "Z_3.txt"], dxproj)
            verify_files_in_dir("/foo/bar", ["luke.txt"], dxproj)
            verify_files_in_dir("/", ["A.txt", "B.txt", "C.txt", "num_chrom.txt"], dxproj)
예제 #14
0
    def test_dx_docker_create_asset(self):
        with temporary_project(select=True) as temp_project:
            test_projectid = temp_project.get_id()
            run("docker pull ubuntu:14.04")
            run("dx-docker create-asset ubuntu:14.04")
            self.assertEqual(run("dx ls ubuntu\\\\:14.04").strip(), 'ubuntu:14.04')

            create_folder_in_project(test_projectid, '/testfolder')
            run("dx-docker create-asset busybox -o testfolder")

            ls_out = run("dx ls /testfolder").strip()
            self.assertEqual(ls_out, 'busybox')

            ls_out = run("dx ls testfolder\\/busybox.tar.gz").strip()
            self.assertEqual(ls_out, 'busybox.tar.gz')
예제 #15
0
    def test_dx_docker_create_asset(self):
        with temporary_project(select=True) as temp_project:
            test_projectid = temp_project.get_id()
            run("docker pull ubuntu:14.04")
            run("dx-docker create-asset ubuntu:14.04")
            self.assertEqual(run("dx ls ubuntu\\\\:14.04").strip(), 'ubuntu:14.04')

            create_folder_in_project(test_projectid, '/testfolder')
            run("dx-docker create-asset busybox -o testfolder")

            ls_out = run("dx ls /testfolder").strip()
            self.assertEqual(ls_out, 'busybox')

            ls_out = run("dx ls testfolder\\/busybox.tar.gz").strip()
            self.assertEqual(ls_out, 'busybox.tar.gz')
예제 #16
0
    def test_basic(self):
        with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
            dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'basic'), dxproj.get_id())

            # Run the applet
            applet_args = ['-iseq1=A.txt', '-iseq2=B.txt', '-iref=A.txt', '-iref=B.txt', "-ivalue=5", "-iages=4"]
            cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
예제 #17
0
    def test_basic(self):
        with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
            dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'basic'), dxproj.get_id())

            # Run the applet
            applet_args = ['-iseq1=A.txt', '-iseq2=B.txt', '-iref=A.txt', '-iref=B.txt', "-ivalue=5", "-iages=4"]
            cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
예제 #18
0
    def test_parseq(self):
        """ Tests the parallel/sequential variations """
        with temporary_project("TestDXBashHelpers.test_app1 temporary project") as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
            dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, "parseq"), dxproj.get_id())

            # Run the applet
            applet_args = ["-iseq1=A.txt", "-iseq2=B.txt", "-iref=A.txt", "-iref=B.txt"]
            cmd_args = ["dx", "run", "--yes", "--watch", applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
예제 #19
0
    def run_applet_with_flags(self, flag_list, num_files, file_size_bytes):
        with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Upload file
            self.create_file_of_size("A.txt", file_size_bytes);
            remote_file = dxpy.upload_local_file(filename="A.txt", project=dxproj.get_id(), folder='/')

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'benchmark'), dxproj.get_id())

            # Add several files to the output
            applet_args = []
            applet_args.extend(['-iref=A.txt'] * num_files)
            cmd_args = ['dx', 'run', '--yes', '--watch', '--instance-type=mem1_ssd1_x2', applet_id]
            cmd_args.extend(applet_args)
            cmd_args.extend(flag_list)
            run(cmd_args, env=env)
예제 #20
0
 def test_build_asset_with_valid_destination(self):
     asset_spec = {
         "name": "foo",
         "title": "A human readable name",
         "description": "A detailed description about the asset",
         "version": "0.0.1",
         "distribution": "Ubuntu",
         "release": "12.04",
         "execDepends": [{"name": "python-numpy"}]
     }
     asset_dir = self.write_asset_directory("asset_with_valid_destination", json.dumps(asset_spec))
     with testutil.temporary_project() as other_project:
         test_dirname = 'asset_dir'
         run('dx mkdir -p {project}:{dirname}'.format(project=other_project.get_id(), dirname=test_dirname))
         asset_bundle_id = json.loads(run('dx build_asset --json --destination ' + other_project.get_id() +
                                          ':/' + test_dirname + '/ ' + asset_dir))['id']
         self.assertIn('record', asset_bundle_id)
         asset_desc = dxpy.describe(asset_bundle_id)
         self.assertEqual(asset_desc['project'], other_project.get_id())
         self.assertEqual(asset_desc['folder'], '/asset_dir')
예제 #21
0
    def test_vars(self):
        """  Quick test for the bash variables """
        with temporary_project("TestDXBashHelpers.test_app1 temporary project") as p:
            env = update_environ(DX_PROJECT_CONTEXT_ID=p.get_id())

            # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=p.get_id(), name="A.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, "vars"), p.get_id())

            # Run the applet
            applet_args = [
                "-iseq1=A.txt",
                "-iseq2=A.txt",
                "-igenes=A.txt",
                "-igenes=A.txt",
                "-ii=5",
                "-ix=4.2",
                "-ib=true",
                "-is=hello",
                "-iil=6",
                "-iil=7",
                "-iil=8",
                "-ixl=3.3",
                "-ixl=4.4",
                "-ixl=5.0",
                "-ibl=true",
                "-ibl=false",
                "-ibl=true",
                "-isl=hello",
                "-isl=world",
                "-isl=next",
                '-imisc={"hello": "world", "foo": true}',
            ]
            cmd_args = ["dx", "run", "--yes", "--watch", applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
예제 #22
0
    def test_xattr_parameters(self):
        ''' Tests dx-upload-all-outputs uploading with filesystem metadata as properties '''
        with temporary_project(
                'TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
            dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(
                os.path.join(TEST_APPS, 'xattr_properties'), dxproj.get_id())

            # Run the applet
            applet_args = [
                "-iseq1=A.txt", "-iseq2=B.txt", "-iref=A.txt", "-iref=B.txt"
            ]
            cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
예제 #23
0
    def test_vars(self):
        '''  Quick test for the bash variables '''
        with temporary_project('TestDXBashHelpers.test_app1 temporary project') as p:
            env = update_environ(DX_PROJECT_CONTEXT_ID=p.get_id())

            # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=p.get_id(), name="A.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'vars'), p.get_id())

            # Run the applet
            applet_args = ['-iseq1=A.txt', '-iseq2=A.txt', '-igenes=A.txt', '-igenes=A.txt',
                           '-ii=5', '-ix=4.2', '-ib=true', '-is=hello',
                           '-iil=6', '-iil=7', '-iil=8',
                           '-ixl=3.3', '-ixl=4.4', '-ixl=5.0',
                           '-ibl=true', '-ibl=false', '-ibl=true',
                           '-isl=hello', '-isl=world', '-isl=next',
                           '-imisc={"hello": "world", "foo": true}']
            cmd_args = ['dx', 'run', '--yes', '--watch', applet_id]
            cmd_args.extend(applet_args)
            run(cmd_args, env=env)
예제 #24
0
    def test_sub_jobs(self):
        '''  Tests a bash script that generates sub-jobs '''
        with temporary_project(
                'TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

            # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
            dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(
                os.path.join(TEST_APPS, 'with-subjobs'), dxproj.get_id())
            # Run the applet.
            # Since the job creates two sub-jobs, we need to be a bit more sophisticated
            # in order to wait for completion.
            applet_args = ["-ifiles=A.txt", "-ifiles=B.txt"]
            cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
            cmd_args.extend(applet_args)
            job_id = run(cmd_args, env=env).strip()

            dxpy.DXJob(job_id).wait_on_done()

            # Assertions -- making sure the script worked
            # Assertions to make about the job's output after it is done running:
            # - *first_file* is a file named first_file.txt containing the string:
            #     "contents of first_file"
            # - *final_file* is a file named final_file.txt containing the
            #   *concatenation of the two input files in *files*
            print("Test completed successfully, checking file content\n")

            job_handler = dxpy.get_handler(job_id)
            job_output = job_handler.output

            def strip_white_space(_str):
                return ''.join(_str.split())

            def silent_file_remove(filename):
                try:
                    os.remove(filename)
                except OSError:
                    pass

            # The output should include two files, this section verifies that they have
            # the correct data.
            def check_file_content(out_param_name, out_filename, tmp_fname,
                                   str_content):
                """
                Download a file, read it from local disk, and verify that it has the correct contents
                """
                if not out_param_name in job_output:
                    raise "Error: key {} does not appear in the job output".format(
                        out_param_name)
                dxlink = job_output[out_param_name]

                # check that the filename gets preserved
                trg_fname = dxpy.get_handler(dxlink).name
                self.assertEqual(trg_fname, out_filename)

                # download the file and check the contents
                silent_file_remove(tmp_fname)
                dxpy.download_dxfile(dxlink, tmp_fname)
                with open(tmp_fname, "r") as fh:
                    data = fh.read()
                    print(data)
                    if not (strip_white_space(data)
                            == strip_white_space(str_content)):
                        raise Exception(
                            "contents of file {} do not match".format(
                                out_param_name))
                silent_file_remove(tmp_fname)

            check_file_content('first_file', 'first_file.txt', "f1.txt",
                               "contents of first_file")
            check_file_content('final_file', 'final_file.txt', "f2.txt",
                               "1234ABCD")
예제 #25
0
 def setUpClass(cls):
     with testutil.temporary_project(name='dx-jobutil-new-job test project',
                                     cleanup=False) as p:
         cls.aux_project = p
예제 #26
0
    def test_sub_jobs(self):
        '''  Tests a bash script that generates sub-jobs '''
        with temporary_project('TestDXBashHelpers.test_app1 temporary project') as dxproj:
            env = update_environ(DX_PROJECT_CONTEXT_ID=dxproj.get_id())

             # Upload some files for use by the applet
            dxpy.upload_string("1234\n", project=dxproj.get_id(), name="A.txt")
            dxpy.upload_string("ABCD\n", project=dxproj.get_id(), name="B.txt")

            # Build the applet, patching in the bash helpers from the
            # local checkout
            applet_id = build_app_with_bash_helpers(os.path.join(TEST_APPS, 'with-subjobs'), dxproj.get_id())
             # Run the applet.
            # Since the job creates two sub-jobs, we need to be a bit more sophisticated
            # in order to wait for completion.
            applet_args = ["-ifiles=A.txt", "-ifiles=B.txt"]
            cmd_args = ['dx', 'run', '--yes', '--brief', applet_id]
            cmd_args.extend(applet_args)
            job_id = run(cmd_args, env=env).strip()

            dxpy.DXJob(job_id).wait_on_done()

            # Assertions -- making sure the script worked
            # Assertions to make about the job's output after it is done running:
            # - *first_file* is a file named first_file.txt containing the string:
            #     "contents of first_file"
            # - *final_file* is a file named final_file.txt containing the
            #   *concatenation of the two input files in *files*
            print("Test completed successfully, checking file content\n")

            job_handler = dxpy.get_handler(job_id)
            job_output = job_handler.output

            def strip_white_space(_str):
                return ''.join(_str.split())

            def silent_file_remove(filename):
                try:
                    os.remove(filename)
                except OSError:
                    pass

            # The output should include two files, this section verifies that they have
            # the correct data.
            def check_file_content(out_param_name, out_filename, tmp_fname, str_content):
                """
                Download a file, read it from local disk, and verify that it has the correct contents
                """
                if not out_param_name in job_output:
                    raise "Error: key {} does not appear in the job output".format(out_param_name)
                dxlink = job_output[out_param_name]

                # check that the filename gets preserved
                trg_fname = dxpy.get_handler(dxlink).name
                self.assertEqual(trg_fname, out_filename)

                # download the file and check the contents
                silent_file_remove(tmp_fname)
                dxpy.download_dxfile(dxlink, tmp_fname)
                with open(tmp_fname, "r") as fh:
                    data = fh.read()
                    print(data)
                    if not (strip_white_space(data) == strip_white_space(str_content)):
                        raise Exception("contents of file {} do not match".format(out_param_name))
                silent_file_remove(tmp_fname)

            check_file_content('first_file', 'first_file.txt', "f1.txt", "contents of first_file")
            check_file_content('final_file', 'final_file.txt', "f2.txt", "1234ABCD")
예제 #27
0
 def setUpClass(cls):
     with testutil.temporary_project(name='dx-jobutil-new-job test project', cleanup=False) as p:
         cls.aux_project = p