class TestContainerSubmitParameterExamples(TestCase2):


    def setup_method(self,method):

        self.remove_files = []

        # get user account info
        self.username,self.userpass = self.testdata.find_account_for('registeredworkspace')
        hubname = self.testdata.find_url_for('https')

        # access a tool session container
        cm = ContainerManager()
        self.ws = cm.access(host=hubname,
                            username=self.username,
                            password=self.userpass)

        # copy the executable to the session directory
        self.sftp = SFTPClient(
            host=hubname, username=self.username, password=self.userpass)

        local_exe_path = os.path.join(hubcheck.conf.settings.data_dir,
                                 'capacitor_voltage','sim1.py')

        self.ws.execute('cd $SESSIONDIR')
        sessiondir,es = self.ws.execute('pwd')

        self.exe_fn = 'sim1.py'
        self.exe_path = os.path.join(sessiondir,self.exe_fn)
        self.remove_files.append(self.exe_path)

        self.sftp.chdir(sessiondir)
        self.sftp.put(local_exe_path,self.exe_fn)
        self.sftp.chmod(self.exe_path,0700)

        # shouldn't take more than 60 seconds
        # to run submit --local commands
        self.ws.timeout = 60


    def teardown_method(self,method):

        # remove the executable and config files
        for fname in self.remove_files:
            self.sftp.remove(fname)
        self.sftp.close()

        # exit the workspace
        self.ws.close()


    def test_submit_single_parameter_substitution(self):
        """
        submit single parameter substitution in input deck

        submit --local -p @@C=10e-12,100e-12,1e-6 \
            sim1.py --inputdeck @:sim1.indeck.template
        """

        self.indeckfn = 'sim1.indeck.template'
        num_sweep_params = 3
        command = 'submit --local -p @@C=10e-12,100e-12,1e-6 %s --inputdeck @:%s' \
            % (self.exe_path,self.indeckfn)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input deck to disk in the container.
        indeck_template = "[inputs]\nC = @@C\n"
        self.ws.write_file(self.indeckfn,indeck_template)
        self.remove_files.append(self.indeckfn)

        # run the command
        output,es = self.ws.execute(command)

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    def test_submit_multiple_parameter_substitution(self):
        """
        submit multiple parameter substitution in input deck

        submit --local -p @@Vin=0:0.2:5 -p @@C=10e-12,100e-12,1e-6 \
            sim1.py --inputdeck @:sim1.indeck.template
        """

        self.indeckfn = 'sim1.indeck.template'
        num_sweep_params = 26*3
        command  = 'submit --local'
        command += ' -p @@Vin=0:0.2:5'
        command += ' -p @@C=10e-12,100e-12,1e-6'
        command += ' %s --inputdeck @:%s'
        command = command % (self.exe_path,self.indeckfn)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input deck to disk in the container.
        indeck_template = "[inputs]\nC = @@C\nVin = @@Vin\n"
        self.ws.write_file(self.indeckfn,indeck_template)
        self.remove_files.append(self.indeckfn)

        # adjust the timeout to allow for 5 minutes to run the tests
        old_timeout = self.ws.timeout
        self.ws.timeout = 300

        # run the command
        output,es = self.ws.execute(command)

        # adjust the timeout back to the default
        self.ws.timeout = old_timeout

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    def test_submit_read_parameters_from_file(self):
        """
        submit multiple parameter substitutions, reading parameters
        from a file named 'params'. perform substitutions in an input deck

        submit --local -p params sim1.py --inputdeck @:sim1.indeck.template
        """

        self.indeckfn = 'sim1.indeck.template'
        self.paramsfn = 'params'
        num_sweep_params = 26*3
        command  = 'submit --local -p %s %s --inputdeck @:%s'
        command = command % (self.paramsfn,self.exe_path,self.indeckfn)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input deck to disk in the container.
        indeck_template = "[inputs]\nC = @@C\nVin = @@Vin\n"
        self.ws.write_file(self.indeckfn,indeck_template)
        self.remove_files.append(self.indeckfn)

        # write the parameters file to disk in the container.
        params_data  = "parameter @@Vin=0:0.2:5\n" \
                       + "parameter @@C = 10e-12,100e-12,1e-6\n"
        self.ws.write_file(self.paramsfn,params_data)
        self.remove_files.append(self.paramsfn)

        # adjust the timeout to allow for 5 minutes to run the tests
        old_timeout = self.ws.timeout
        self.ws.timeout = 300

        # run the command
        output,es = self.ws.execute(command)

        # adjust the timeout back to the default
        self.ws.timeout = old_timeout

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    def test_submit_read_params_file_load_extra_params(self):
        """
        read submit parameters from a file, load additional parameters
        from the command line.

        submit --local -p "params;@@Vin=5-7;@@R=100e3" \
            sim1.py --inputdeck @:sim1.indeck.template
        """

        self.indeckfn = 'sim1.indeck.template'
        self.paramsfn = 'params'
        # 3 C values * 3 Vin values * 1 R value
        num_sweep_params = 3*3*1
        command  = 'submit --local' \
                   ' -p "%s;@@Vin=5-7;@@R=100e3"' \
                   ' %s --inputdeck @:%s' \
                   % (self.paramsfn,self.exe_path,self.indeckfn)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input deck to disk in the container.
        indeck_template = "[inputs]\nC = @@C\nVin = @@Vin\nR = @@R\n"
        self.ws.write_file(self.indeckfn,indeck_template)
        self.remove_files.append(self.indeckfn)

        # write the parameters file to disk in the container.
        params_data = "parameter @@C = 10e-12,100e-12,1e-6\n"
        self.ws.write_file(self.paramsfn,params_data)
        self.remove_files.append(self.paramsfn)

        # run the command
        output,es = self.ws.execute(command)

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    def test_submit_read_params_from_csv_file(self):
        """
        read submit parameters from a csv file

        submit --local -d input.csv \
            sim1.py --inputdeck @:sim1.indeck.template
        """

        self.indeckfn = 'sim1.indeck.template'
        self.paramsfn = 'input.csv'
        # 4 Vin & C combinations
        num_sweep_params = 4
        command  = 'submit --local -d %s'
        command += ' %s --inputdeck @:%s'
        command = command % (self.paramsfn,self.exe_path,self.indeckfn)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input deck to disk in the container.
        indeck_template = "[inputs]\nC = @@C\nVin = @@Vin\n"
        self.ws.write_file(self.indeckfn,indeck_template)
        self.remove_files.append(self.indeckfn)

        # write the parameters file to disk in the container.
        params_data = "@@Vin, @@C\n1.1, 1e-12\n2.2, 1e-12\n1.1, 10e-12\n2.2, 10e-12"
        self.ws.write_file(self.paramsfn,params_data)
        self.remove_files.append(self.paramsfn)

        # run the command
        output,es = self.ws.execute(command)

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    @pytest.mark.stalenfs
    def test_submit_read_csv_params_load_extras_from_args(self):
        """
        read submit parameters from a csv file, load extra params

        submit --local -d input.csv -p "@@R=1e3-1e5 in 3 log"\
            sim1.py --inputdeck @:sim1.indeck.template
        """

        self.indeckfn = 'sim1.indeck.template'
        self.paramsfn = 'input.csv'
        # 4 Vin & C combinations * 3 R values
        num_sweep_params = 4*3
        command  = 'submit --local -d %s'
        command += ' -p "@@R=1e3-1e5 in 3 log"'
        command += ' %s --inputdeck @:%s'
        command = command % (self.paramsfn,self.exe_path,self.indeckfn)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input deck to disk in the container.
        indeck_template = "[inputs]\nC = @@C\nVin = @@Vin\nR = @@R\n"
        self.ws.write_file(self.indeckfn,indeck_template)
        self.remove_files.append(self.indeckfn)

        # write the parameters file to disk in the container.
        params_data = "@@Vin, @@C\n1.1, 1e-12\n2.2, 1e-12\n1.1, 10e-12\n2.2, 10e-12"
        self.ws.write_file(self.paramsfn,params_data)
        self.remove_files.append(self.paramsfn)

        # run the command
        output,es = self.ws.execute(command)

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    @pytest.mark.stalenfs
    @pytest.mark.clparse
    def test_submit_read_csv_params_load_extras_from_file(self):
        """
        read submit parameters from a csv file, load extra params

        submit --local -d input.csv -i @:data.txt \
            sim1.py --inputdeck @:sim1.indeck.template
        """

        self.indeckfn = 'sim1.indeck.template'
        self.paramsfn = 'input.csv'
        self.extrafn = 'data.txt'
        # 4 Vin & C combinations
        num_sweep_params = 4
        command  = 'submit --local -d %s -i @:%s %s --inputdeck @:%s' \
                    % (self.paramsfn,self.extrafn,self.exe_path,self.indeckfn)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input deck to disk in the container.
        indeck_template = "[inputs]\nC = @@C\nVin = @@Vin\n"
        self.ws.write_file(self.indeckfn,indeck_template)
        self.remove_files.append(self.indeckfn)
        self.ws.execute('ls {0}'.format(self.indeckfn))

        # write the input deck to disk in the container.
        extra_template = "# extra templated data file\nVin = @@Vin\nC = @@C\n"
        self.ws.write_file(self.extrafn,extra_template)
        self.remove_files.append(self.extrafn)
        self.ws.execute('ls {0}'.format(self.extrafn))

        # write the parameters file to disk in the container.
        params_data = "@@Vin, @@C\n1.1, 1e-12\n2.2, 1e-12\n1.1, 10e-12\n2.2, 10e-12"
        self.ws.write_file(self.paramsfn,params_data)
        self.remove_files.append(self.paramsfn)
        self.ws.execute('ls {0}'.format(self.paramsfn))

        # run the command
        output,es = self.ws.execute(command)

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    def test_submit_change_separator(self):
        """
        Change the separator to slash

        submit --local -s / -p @@Vin=5/6/7 -s , -p @@C=1e-12,10e-12 \
            sim1.py --inputdeck @:sim1.indeck.template
        """

        self.indeckfn = 'sim1.indeck.template'
        # 3 Vin values * 2 C values
        num_sweep_params = 3*2
        command  = 'submit --local' \
                   + ' -s / -p @@Vin=5/6/7' \
                   + ' -s , -p @@C=1e-12,10e-12' \
                   + ' %s --inputdeck @:%s' \
                   % (self.exe_path,self.indeckfn)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input deck to disk in the container.
        indeck_template = "[inputs]\nC = @@C\nVin = @@Vin\n"
        self.ws.write_file(self.indeckfn,indeck_template)
        self.remove_files.append(self.indeckfn)

        # run the command
        output,es = self.ws.execute(command)

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    def test_submit_parameter_substitute_in_command_arguments(self):
        """
        Substitute parameters into command line arguments

        submit --local -p @@Vin=1-5 sim1.py --Vin @@Vin
        """

        # 5 Vin values
        num_sweep_params = 5
        command  = 'submit --local -p @@Vin=1-5 %s --Vin @@Vin'
        command = command % (self.exe_path)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # run the command
        output,es = self.ws.execute(command)

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    def test_submit_parameter_glob_file_search(self):
        """
        Perform glob style file pattern matching to generate parameters
        substitutions.

        submit --local -p @@file=glob:sim1.indeck* sim1.py --inputdeck @@file
        """

        self.fbase = 'sim1.indeck'
        # 5 Vin values
        num_sweep_params = 4
        command  = 'submit --local -p @@file=glob:%s* %s --inputdeck @@file'
        command = command % (self.fbase,self.exe_path)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # write the input decks to disk in the container.
        counter = 0
        for (Vin,C) in [(1.1,1e-12),(2.2,1e-12),(1.1,10e-12),(2.2,10e-12)]:
            counter += 1
            indeck_template = "[inputs]\nC = %s\nVin = %s\n" % (C,Vin)
            fname = self.fbase + '.%s' % (counter)
            self.ws.write_file(fname,indeck_template)
            self.remove_files.append(fname)


        # run the command
        output,es = self.ws.execute(command)

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # get the directory names of sweep results
        output,es = self.ws.execute('ls -d %s/*/' % resultsdir)

        # check if the number of directories matches the number of
        # parameters we used in the sweep
        results_dirs = output.split()
        num_results_dirs = len(results_dirs)
        assert num_results_dirs == num_sweep_params, \
            "num_results_dirs = %s, num_sweep_params = %s" \
            % (num_results_dirs,num_sweep_params)

        # check that the output file was copied back to
        # the results directory
        for r in results_dirs:
            command = '[[ -r %s/out.log ]] && echo 1 || echo 0' % r
            exists,es = self.ws.execute(command)
            assert exists == '1', "missing %s/out.log" % r


    @pytest.mark.submit_parameter_error
    def test_submit_parameter_error_code(self):
        """
        Check for submit errors after substituting parameters
        into command line arguments

        submit --local -p @@Vin=1-3 sim1.py --Vin @@Vin
        """

        # 3 Vin values
        num_sweep_params = 3
        command  = 'submit --local -p @@Vin=1-3 %s --Vin @@Vin'
        command = command % (self.exe_path)

        # we redirect stdin to a file for submit commands with
        # parameters so the ncurses window doesn't pop up and
        # interfere with our expect like terminal parsing.
        command += ' 0</dev/null'

        # run the command
        output,es = self.ws.execute(command,fail_on_exit_code=False)

        if es == 0:
            return

        # search for results
        results_re = re.compile('Results are stored in directory ([^\s]+)')
        match = results_re.search(output)
        assert match is not None, \
            "could not find results directory in output: %s" % output
        resultsdir = match.group(1)

        # look for a stderr file
        stderr_file = os.path.basename(resultsdir)
        stderr_file_path = os.path.join(resultsdir,stderr_file) + ".stderr"

        stderr_file_exists = self.ws.bash_test('-e %s' % (stderr_file_path))
        assert stderr_file_exists, \
            'The command "%s" produced the output "%s"' % (command,output) \
            + ' and exited with exit code %s, but' % (es) \
            + ' the stderr file "%s" does not exist' % (stderr_file_path)

        read_stderr_cmd = 'cat %s' % (stderr_file_path)
        stderr_out,stderr_es = self.ws.execute(read_stderr_cmd)

        # we don't really want the file to be empty, but we want to
        # capture whatever is in the file.
        assert stderr_out == '', \
            'The command "%s", produced the output "%s",' % (command,output) \
            + ' exited with exit code %s,' % (es) \
            + ' and wrote the following to stderr file' \
            + ' "%s": %s' % (stderr_file_path, stderr_out)
class container_invokeapp(TestCase):


    def setUp(self):

        self.remove_files = []

        # get user account info
        self.username,self.userpass = self.testdata.find_account_for('registeredworkspace')
        hubname = self.testdata.find_url_for('https')

        cm = ContainerManager()

        self.ws = cm.access(host=hubname,
                            username=self.username,
                            password=self.userpass)

        # cd into the session directory
        self.sftp = SFTPClient(
            host=hubname, username=self.username, password=self.userpass)

        self.ws.execute('cd $SESSIONDIR')
        self.sessiondir,es = self.ws.execute('pwd')
        self.sftp.chdir(self.sessiondir)


        # write data and script files to disk in the container.
        for fname,fprop in FILES.items():
            with self.sftp.open(fname,mode='w') as f:
                f.write(fprop['contents'])
            self.remove_files.append(os.path.join(self.sessiondir,fname))
            self.sftp.chmod(fname,fprop['mode'])


    def tearDown(self):

        # remove the executable and config files
        for fname in self.remove_files:
            self.sftp.remove(fname)
        self.sftp.close()

        # exit the workspace
        # shut down the ssh connection
        self.ws.close()


    def _run_invoke_app(self,command,parameters_text=None):

        if parameters_text is not None:
            with self.sftp.open(PARAMETERS_PATH,'w') as f:
                f.write(parameters_text)
            self.remove_files.append(os.path.join(self.sessiondir,PARAMETERS_PATH))
            cmd = 'export TOOL_PARAMETERS=%s' % (PARAMETERS_PATH)
            self.ws.execute(cmd)

        # allow up to 30 seconds for the command to run
        oldtimeout = self.ws.timeout
        self.ws.timeout = 30

        # run the command in xvfb to handle toolparams popup windows
        command = 'xvfb-run -s "-screen 0 800x600x24" ' + command

        # run the invoke_app command
        result,err = self.ws.execute(command,fail_on_exit_code=False)

        # reset the command timeout
        self.ws.timeout = oldtimeout

        # check for error while running invoke_app command
        toolout = None
        matches = re.search("\nexec'ing[^\n]+\n(.*)",result,re.DOTALL)
        if matches:
            # grab the output
            # strip off submit metrics from toolout
            pattern = "=SUBMIT-METRICS=>.*$"
            toolout = re.sub(pattern,'',matches.group(1),flags=re.MULTILINE)
            toolout = toolout.strip()

        return (result,err,toolout)


    def _find_bg_command(self,command):

        grep_cmd = 'ps aux | grep "' + command + '" | grep -v -e "bash\|grep"'
        result,err = self.ws.execute(grep_cmd)

        import re
        rexp = '(?P<user>[^\s]+)\s+' + \
               '(?P<pid>[^\s]+)\s+'  + \
               '(?P<cpu>[^\s]+)\s+'  + \
               '(?P<mem>[^\s]+)\s+'  + \
               '(?P<vsz>[^\s]+)\s+'  + \
               '(?P<rss>[^\s]+)\s+'  + \
               '(?P<tty>[^\s]+)\s+'  + \
               '(?P<stat>[^\s]+)\s+' + \
               '(?P<start>[^\s]+)\s+' + \
               '(?P<time>[^\s]+)\s+' + \
               '(?P<command>[^\n]+)'

        m = re.search(rexp,result)
        if m is None:
            return None

        # make sure we return the correct command
        if m.group('command') != command:
            # didn't find the correct command
            return None

        return m.groupdict()


    def test_1_command_no_templates(self):
        """launching invoke_app with one -C command (not template) should run the command
           ex: invoke_app -C "sh ./slow_echo hi"
           should produce: hi
        """

        # build our invoke_app command
        command = INVOKE_APP_PATH + ' -C "sh ./slow_echo hi"'

        expected_out = "hi"

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_3_commands_no_templates(self):
        """launching invoke_app with multiple non-templated -C commands
           should run the last command.
           ex: invoke_app -C "sh ./slow_echo hi" -C "sh ./slow_echo bye" -C "sh ./slow_echo yeah"
           should produce: yeah
        """

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "sh ./slow_echo bye"' \
                  + ' -C "sh ./slow_echo yeah"'

        expected_out = "yeah"

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_template_0_default_run_template_1(self):
        """launching invoke_app with one -C template command should
           launch toolparams to run the command
           ex: invoke_app -C "cat @@file(datafile1)"
           should launch: toolparams 'cat @@file(datafile1)'
           and produce: this is datafile1
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"'

        expected_out = FILES['datafile1']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_template_1_default_run_template_1(self):
        """launching invoke_app with one -C template command
           and one -C non-template command should launch
           toolparams to run the command. when the parameters file
           has a valid reference to file(datafile1), the templated
           command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "sh ./slow_echo hi"
           should launch: toolparams 'cat @@file(datafile1)' -default 'sh ./slow_echo hi'
           and produce: this is datafile1
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "sh ./slow_echo hi"'

        expected_out = FILES['datafile1']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_template_1_default_run_default(self):
        """launching invoke_app with one -C template command
           and one -C non-template command should launch
           toolparams to run the command. when the parameters file
           does not have a valid reference to file(datafile1), the
           non-templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "sh ./slow_echo hi"
           should launch: toolparams 'cat @@file(datafile1)' -default 'sh ./slow_echo hi'
           and produce: hi
        """

        # create our parameters file
        parameters_text = '\n'.join([])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "sh ./slow_echo hi"'

        expected_out = "hi"

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_default_1_template_run_template_1(self):
        """launching invoke_app with one -C non-template command
           and one -C template command should launch
           toolparams to run the command. when the parameters file
           has a valid reference to file(datafile1), the templated
           command should be run.
           ex: invoke_app -C "sh ./slow_echo hi" -C "cat @@file(datafile1)"
           should launch: toolparams 'cat @@file(datafile1)' -default 'sh ./slow_echo hi'
           and produce: this is datafile1
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "cat @@file(datafile1)"'

        expected_out = FILES['datafile1']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_default_1_template_run_default(self):
        """launching invoke_app with one -C non-template command
           and one -C template command should launch
           toolparams to run the command. when the parameters file
           does not have a valid reference to file(datafile1), the
           non-templated command should be run.
           ex: invoke_app -C "sh ./slow_echo hi" -C "cat @@file(datafile1)"
           should launch: toolparams 'cat @@file(datafile1)' -default 'sh ./slow_echo hi'
           and produce: hi
        """

        # create our parameters file
        parameters_text = '\n'.join([])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "cat @@file(datafile1)"'

        expected_out = "hi"

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_2_templates_0_default_run_template_1(self):
        """launching invoke_app with two -C template commands
           and zero -C non-template command should launch
           toolparams to run the command. when the parameters file
           has a valid reference to file(datafile1), the
           appropriate templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "cat @@file(datafile2)"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)'
           and produce: this is datafile1
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "cat @@file(datafile2)"'

        expected_out = FILES['datafile1']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_2_templates_0_default_run_template_2(self):
        """launching invoke_app with two -C template commands
           and zero -C non-template command should launch
           toolparams to run the command. when the parameters file
           has a valid reference to file(datafile2), the
           appropriate templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "cat @@file(datafile2)"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)'
           and produce: this is datafile2
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile2):%s" % os.path.join(self.sessiondir,'datafile2'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "cat @@file(datafile2)"'

        expected_out = FILES['datafile2']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_2_templates_1_default_run_template_1(self):
        """launching invoke_app with two -C template commands
           and one -C non-template command should launch
           toolparams to run the command. when the parameters file
           has a valid reference to file(datafile1), the
           appropriate templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "cat @@file(datafile2)" -C "sh ./slow_echo hi"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: this is datafile1
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "cat @@file(datafile2)"' \
                  + ' -C "sh ./slow_echo hi"'

        expected_out = FILES['datafile1']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_2_templates_1_default_run_template_2(self):
        """launching invoke_app with two -C template commands
           and one -C non-template command should launch
           toolparams to run the command. when the parameters file
           has a valid reference to file(datafile2), the
           appropriate templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "cat @@file(datafile2)" -C "sh ./slow_echo hi"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: this is datafile2
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile2):%s" % os.path.join(self.sessiondir,'datafile2'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "cat @@file(datafile2)"' \
                  + ' -C "sh ./slow_echo hi"'

        expected_out = FILES['datafile2']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_2_templates_1_default_run_default(self):
        """launching invoke_app with two -C template commands
           and one -C non-template command should launch
           toolparams to run the command. when the parameters file
           does not have a valid reference, the
           appropriate non-templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "cat @@file(datafile2)" -C "sh ./slow_echo hi"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: hi
        """

        # create our parameters file
        parameters_text = '\n'.join([])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "cat @@file(datafile2)"' \
                  + ' -C "sh ./slow_echo hi"'

        expected_out = 'hi'

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_template_1_default_1_template_run_template_1(self):
        """launching invoke_app with one -C template command
           and one -C non-template command and a second
           -C template comamnd should launch toolparams
           to run the command. when the parameters file
           has a valid reference to file(datafile1), the
           appropriate templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "sh ./slow_echo hi" -C "cat @@file(datafile2)"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: this is datafile1
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "cat @@file(datafile2)"'

        expected_out = FILES['datafile1']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_template_1_default_1_template_run_template_2(self):
        """launching invoke_app with one -C template commands
           and one -C non-template command and a second
           -C template command should launch toolparams
           to run the command. when the parameters file
           has a valid reference to file(datafile2), the
           appropriate templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "sh ./slow_echo hi" -C "cat @@file(datafile2)"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: this is datafile2
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile2):%s" % os.path.join(self.sessiondir,'datafile2'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "cat @@file(datafile2)"'

        expected_out = FILES['datafile2']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_template_1_default_1_template_run_default(self):
        """launching invoke_app with one -C template command
           and one -C non-template command and a second
           -C template command should launch toolparams
           to run the command. when the parameters file
           does not have a valid reference, the
           appropriate non-templated command should be run.
           ex: invoke_app -C "cat @@file(datafile1)" -C "sh ./slow_echo hi" -C "cat @@file(datafile2)"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: hi
        """

        # create our parameters file
        parameters_text = '\n'.join([])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "cat @@file(datafile2)"'

        expected_out = 'hi'

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_default_2_templates_run_template_1(self):
        """launching invoke_app with one -C non-template command
           and two -C template commands should launch toolparams
           to run the command. when the parameters file
           has a valid reference to file(datafile1), the
           appropriate templated command should be run.
           ex: invoke_app -C "sh ./slow_echo hi" -C "cat @@file(datafile1)" -C "cat @@file(datafile2)"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: this is datafile1
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "cat @@file(datafile2)"'

        expected_out = FILES['datafile1']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_default_2_templates_run_template_2(self):
        """launching invoke_app with one -C non-template command
           and two -C template commands should launch toolparams
           to run the command. when the parameters file
           has a valid reference to file(datafile2), the
           appropriate templated command should be run.
           ex: invoke_app -C "sh ./slow_echo hi" -C "cat @@file(datafile1)" -C "cat @@file(datafile2)"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: this is datafile2
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile2):%s" % os.path.join(self.sessiondir,'datafile2'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "cat @@file(datafile2)"'

        expected_out = FILES['datafile2']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_1_default_2_template_run_default(self):
        """launching invoke_app with one -C non-template command
           and two -C template command should launch toolparams
           to run the command. when the parameters file
           does not have a valid reference, the
           appropriate non-templated command should be run.
           ex: invoke_app -C "sh ./slow_echo hi" -C "cat @@file(datafile1)" -C "cat @@file(datafile2)"
           should launch: toolparams 'cat @@file(datafile1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: hi
        """

        # create our parameters file
        parameters_text = '\n'.join([])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "sh ./slow_echo hi"' \
                  + ' -C "cat @@file(datafile1)"' \
                  + ' -C "cat @@file(datafile2)"'

        expected_out = 'hi'

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_2_templates_1_default_run_index_1(self):
        """launching invoke_app with two -C template commands
           and one -C non-template command should launch
           toolparams to run the command. when the parameters file
           has a valid positional reference, the
           appropriate templated command should be run.
           ex: invoke_app -C "cat @@file(#1)" -C "cat @@file(datafile2)" -C "sh ./slow_echo hi"
           should launch: toolparams 'cat @@file(#1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
           and produce: this is datafile1
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(#1)"' \
                  + ' -C "cat @@file(datafile2)"' \
                  + ' -C "sh ./slow_echo hi"'

        expected_out = FILES['datafile1']['contents']

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


#    def test_positional_2_templates_1_default_run_index_1_1(self):
#        """launching invoke_app with two -C template commands
#           and one -C non-template command should launch
#           toolparams to run the command. when the parameters file
#           has two valid positional references, the first matching
#           templated command should be run.
#           ex: invoke_app -C "cat @@file(#1)" -C "cat @@file(#1) @@file(#2)" -C "sh ./slow_echo hi"
#           should launch: toolparams 'cat @@file(#1)' -or 'cat @@file(datafile2)' -default 'sh ./slow_echo hi'
#           and produce: this is datafile1
#        """
#
#        # create our parameters file
#        parameters_text = '\n'.join([
#            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
#            "file(datafile2):%s" % os.path.join(self.sessiondir,'datafile2'),
#        ])
#
#        # build our invoke_app command
#        command = INVOKE_APP_PATH \
#                  + ' -C "cat @@file(#1)"' \
#                  + ' -C "cat @@file(#1) @@file(#2)"' \
#                  + ' -C "sh ./slow_echo hi"'
#
#        expected_out = FILES['datafile1']['contents']
#
#        # run invoke_app
#        result,err,toolout = self._run_invoke_app(command,parameters_text)
#
#        # check result
#        self.assertTrue(err == 0,
#            "Error while executing '%s': %s" % (command,toolout))
#
#        # parse the output
#        self.assertTrue(toolout == expected_out,
#            'Error while executing "%s": ' % (command)
#            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_positional_2_templates_1_default_run_index_1_2(self):
        """launching invoke_app with two -C template commands
           and one -C non-template command should launch
           toolparams to run the command. when the parameters file
           has two valid positional references, the first matching
           templated command should be run.
           ex: invoke_app -C "cat @@file(#1) @@file(#2)" -C "cat @@file(#1)" -C "sh ./slow_echo hi"
           should launch: toolparams 'cat @@file(#1) @@file(#2)' -or 'cat @@file(#1)' -default 'sh ./slow_echo hi'
           and produce:
           this is datafile1
           this is datafile2
        """

        # create our parameters file
        parameters_text = '\n'.join([
            "file(datafile1):%s" % os.path.join(self.sessiondir,'datafile1'),
            "file(datafile2):%s" % os.path.join(self.sessiondir,'datafile2'),
        ])

        # build our invoke_app command
        command = INVOKE_APP_PATH \
                  + ' -C "cat @@file(#1) @@file(#2)"' \
                  + ' -C "cat @@file(#1)"' \
                  + ' -C "sh ./slow_echo hi"'

        expected_out = "%s%s" % (FILES['datafile1']['contents'],
                                 FILES['datafile2']['contents'])

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command,parameters_text)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_command_arguments_1(self):
        """launching invoke_app with the -A flag, sending additional arguments
           ex: invoke_app -C "sh ./slow_echo" -A "hi pete"
           should produce: hi pete
        """

        # build our invoke_app command
        command = INVOKE_APP_PATH + ' -C "sh ./slow_echo" -A "hi pete"'

        expected_out = "hi pete"

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_command_arguments_2(self):
        """launching invoke_app with a blank -A flag,
           sending an empty string as additional arguments
           ex: invoke_app -C "sh ./slow_echo hi" -A ""
           should produce: hi
        """

        # build our invoke_app command
        command = INVOKE_APP_PATH + ' -C "sh ./slow_echo hi" -A ""'

        expected_out = "hi"

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_background_command_1(self):
        """launching invoke_app with a single -c flag,
           launching a background command
           ex: invoke_app -C "sh ./slow_echo hi" -c "sleep 23995946712"
           should produce: hi
        """

        # build our invoke_app command
        bg_cmd = "sleep 23995946712"
        command = INVOKE_APP_PATH + ' -C "sh ./slow_echo hi" -c "' + bg_cmd + '"'

        expected_out = "hi"

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


        # check that the background job was started
        result = self._find_bg_command(bg_cmd)
        self.assertTrue(result is not None,)


    def test_working_directory_1(self):
        """launching invoke_app with a -d flag to change the working directory,
           ex: invoke_app -C "sh \${SESSIONDIR}/slow_echo \${PWD}" -d ${HOME}
           should produce: ${HOME}
        """

        # build our invoke_app command
        homedir,err = self.ws.execute('sh ./slow_echo ${HOME}')
        command = INVOKE_APP_PATH \
            + ' -C "sh \${SESSIONDIR}/slow_echo \${PWD}" -d ${HOME}'

        expected_out = homedir

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_environment_variable_1(self):
        """launching invoke_app with a -e flag to set an environment variable,
           ex: invoke_app -C "sh ./slow_echo \${FOO}" -e FOO=blahh
           should produce: blahh
        """

        # build our invoke_app command
        expected_out = 'blahh'
        command = INVOKE_APP_PATH \
            + ' -C "sh ./slow_echo \${FOO}"' \
            + ' -e FOO={0}'.format(expected_out)


        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_fullscreen_1(self):
        """launching invoke_app with a -f flag to
           not set the FULLSCREEN environment variable,
           ex: invoke_app -C "sh ./slow_echo \${FULLSCREEN}" -f
           should produce: ""
        """

        # build our invoke_app command
        command = INVOKE_APP_PATH + ' -C "sh ./slow_echo \${FULLSCREEN}" -f'

        # because invoke_app checks if the command is associated with a tty
        # before starting a window manager or setting the FULLSCREEN
        # environment variable, we have to nohup the command and capture
        # the output.
        command = 'nohup {0} > nohup.out && cat nohup.out'.format(command)

        expected_out = ''

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # cleanup output, removing "XIO:  fatal IO error 11"
        # messages that arise because we close down the xvfb
        # in an unorthodox way (i think)
        toolout = re.sub(r'XIO.+remaining\.','',toolout,flags=re.DOTALL)
        toolout = toolout.strip()

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


    def test_fullscreen_2(self):
        """launching invoke_app without a -f flag to
           set the FULLSCREEN environment variable to "yes",
           ex: invoke_app -C "sh ./slow_echo \${FULLSCREEN}"
           should produce: yes
        """

        # build our invoke_app command
        command = INVOKE_APP_PATH + ' -C "sh ./slow_echo \${FULLSCREEN}"'

        # because invoke_app checks if the command is associated with a tty
        # before starting a window manager or setting the FULLSCREEN
        # environment variable, we have to nohup the command and capture
        # the output.
        command = 'nohup {0} > nohup.out && cat nohup.out'.format(command)

        expected_out = 'yes'

        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # cleanup output, removing "XIO:  fatal IO error 11"
        # messages that arise because we close down the xvfb
        # in an unorthodox way (i think)
        toolout = re.sub(r'XIO.+remaining\.','',toolout,flags=re.DOTALL)
        toolout = toolout.strip()

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))


#    def test_nanowhim_1(self):
#        """launching invoke_app with a -n flag to
#           setup the nanowhim version,
#           ex: invoke_app -C "sh ./slow_echo " -n dev
#           should produce:
#        """
#
#        # build our invoke_app command
#        command = INVOKE_APP_PATH + ' -C "sh ./slow_echo ${FULLSCREEN}"'
#
#        expected_out = 'yes'
#
#        # run invoke_app
#        result,err,toolout = self._run_invoke_app(command)
#
#        # check result
#        self.assertTrue(err == 0,
#            "Error while executing '%s': %s" % (command,toolout))
#
#        # parse the output
#        self.assertTrue(toolout == expected_out,
#            'Error while executing "%s": ' % (command)
#            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))
#


    def test_path_environment_variable_1(self):
        """launching invoke_app with a -p flag to set
           the PATH environment variable,
           ex: invoke_app -C "sh ./slow_echo \${PATH} | cut -d\":\" -f 1" -p /blahh
           should produce: /blahh
        """

        # build our invoke_app command
        expected_out = '/blahh'
        command = INVOKE_APP_PATH \
            + ' -C "sh ./slow_echo \${PATH} | cut -d\":\" -f 1"'\
            + ' -p {0}'.format(expected_out)


        # run invoke_app
        result,err,toolout = self._run_invoke_app(command)

        # check result
        self.assertTrue(err == 0,
            "Error while executing '%s': %s" % (command,toolout))

        # parse the output
        self.assertTrue(toolout == expected_out,
            'Error while executing "%s": ' % (command)
            + 'expected "%s"\nreceived "%s"' % (expected_out,toolout))