def test_mode_full_fresh_no_existing_repo(self): self.setup_step( mercurial.Mercurial(repourl='http://hg.mozilla.org', mode='full', method='fresh', branchType='inrepo')) self.expect_commands( ExpectShell(workdir='wkdir', command=['hg', '--verbose', '--version']).exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True).exit(1), ExpectStat(file='wkdir/.hg', log_environ=True).exit(1), ExpectShell(workdir='wkdir', command=[ 'hg', '--verbose', 'clone', '--noupdate', 'http://hg.mozilla.org', '.' ]).exit(0), ExpectShell(workdir='wkdir', command=[ 'hg', '--verbose', 'update', '--clean', '--rev', 'default' ], log_environ=True).exit(0), ExpectShell( workdir='wkdir', command=[ 'hg', '--verbose', 'parents', '--template', '{node}\\n' ]).stdout('\n').stdout( 'f6ad368298bd941e934a41f3babc827b2aa95a1d').exit(0)) self.expect_outcome(result=SUCCESS) return self.run_step()
def test_mode_incremental_no_existing_repo_dirname(self): self.setup_step( mercurial.Mercurial(repourl='http://hg.mozilla.org', mode='incremental', branchType='dirname'), ) self.expect_commands( ExpectShell(workdir='wkdir', command=['hg', '--verbose', '--version']).exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True).exit(1), ExpectStat(file='wkdir/.hg', log_environ=True).exit(1), # does not exist ExpectShell(workdir='wkdir', command=[ 'hg', '--verbose', 'clone', '--noupdate', 'http://hg.mozilla.org', '.' ]).exit(0), ExpectShell(workdir='wkdir', command=['hg', '--verbose', 'locate', 'set:added()']).exit(1), ExpectShell(workdir='wkdir', command=['hg', '--verbose', 'update', '--clean']).exit(0), ExpectShell( workdir='wkdir', command=[ 'hg', '--verbose', 'parents', '--template', '{node}\\n' ]).stdout('\n').stdout( 'f6ad368298bd941e934a41f3babc827b2aa95a1d').exit(0)) self.expect_outcome(result=SUCCESS) return self.run_step()
def test_mode_full_clean_different_project(self): self.setup_step( gerrit.Gerrit(repourl='http://github.com/buildbot/buildbot.git', mode='full', method='clean', codebase='buildbot')) self.build.setProperty("event.change.project", "buildbot") self.sourcestamp.project = 'not_buildbot' self.build.setProperty("gerrit_change", "1234/567") self.expect_commands( ExpectShell(workdir='wkdir', command=['git', '--version' ]).stdout('git version 1.7.5').exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True).exit(1), ExpectListdir(dir='wkdir').files(['.git']).exit(0), ExpectShell(workdir='wkdir', command=['git', 'clean', '-f', '-f', '-d']).exit(0), ExpectShell(workdir='wkdir', command=[ 'git', 'fetch', '-f', '-t', 'http://github.com/buildbot/buildbot.git', 'HEAD', '--progress' ]).exit(0), ExpectShell(workdir='wkdir', command=['git', 'checkout', '-f', 'FETCH_HEAD']).exit(0), ExpectShell(workdir='wkdir', command=[ 'git', 'rev-parse', 'HEAD' ]).stdout('f6ad368298bd941e934a41f3babc827b2aa95a1d').exit(0)) self.expect_outcome(result=SUCCESS) return self.run_step()
def test_mode_full_clobber_revision_worker_2_16(self): self.setup_step(darcs.Darcs(repourl='http://localhost/darcs', mode='full', method='clobber'), dict(revision='abcdef01'), worker_version={'*': '2.16'}) self.expect_commands( ExpectShell(workdir='wkdir', command=['darcs', '--version']).exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True).exit(1), ExpectRmdir(dir='wkdir', log_environ=True).exit(0), ExpectDownloadFile(blocksize=32768, maxsize=None, reader=ExpectRemoteRef( remotetransfer.StringFileReader), slavedest='.darcs-context', workdir='wkdir', mode=None).exit(0), ExpectShell(workdir='.', command=[ 'darcs', 'get', '--verbose', '--lazy', '--repo-name', 'wkdir', '--context', '.darcs-context', 'http://localhost/darcs' ]).exit(0), ExpectShell( workdir='wkdir', command=['darcs', 'changes', '--max-count=1']).stdout( 'Tue Aug 20 09:18:41 IST 2013 [email protected]').exit(0)) self.expect_outcome(result=SUCCESS) self.expect_property('got_revision', 'Tue Aug 20 09:18:41 IST 2013 [email protected]', 'Darcs') return self.run_step()
def test_mode_full_copy(self): self.setup_step( darcs.Darcs(repourl='http://localhost/darcs', mode='full', method='copy')) self.expect_commands( ExpectShell(workdir='wkdir', command=['darcs', '--version']).exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True).exit(1), ExpectRmdir(dir='wkdir', log_environ=True, timeout=1200).exit(0), ExpectStat(file='source/_darcs', log_environ=True).exit(0), ExpectShell(workdir='source', command=['darcs', 'pull', '--all', '--verbose']).exit(0), ExpectCpdir(fromdir='source', todir='build', log_environ=True, timeout=1200).exit(0), ExpectShell( workdir='build', command=['darcs', 'changes', '--max-count=1']).stdout( 'Tue Aug 20 09:18:41 IST 2013 [email protected]').exit(0)) self.expect_outcome(result=SUCCESS) self.expect_property('got_revision', 'Tue Aug 20 09:18:41 IST 2013 [email protected]', 'Darcs') return self.run_step()
def test_mode_full_clobber_no_existing_db(self): self.setup_step( mtn.Monotone(repourl='mtn://localhost/monotone', mode='full', method='clobber', branch='master')) self.expect_commands( ExpectShell(workdir='wkdir', command=['mtn', '--version']) .stdout(self.MTN_VER) .exit(0), ExpectStat(file='db.mtn', log_environ=True) .exit(1), ExpectShell(workdir='.', command=['mtn', 'db', 'init', '--db', 'db.mtn']) .exit(0), ExpectShell(workdir='.', command=['mtn', 'pull', 'mtn://localhost/monotone?master', '--db', 'db.mtn', '--ticker=dot']) .exit(0), ExpectRmdir(dir='wkdir', log_environ=True) .exit(0), ExpectShell(workdir='.', command=['mtn', 'checkout', 'wkdir', '--db', 'db.mtn', '--branch', 'master']) .exit(0), ExpectShell(workdir='wkdir', command=['mtn', 'automate', 'select', 'w:']) .stdout(self.REVID) .exit(0) ) self.expect_outcome(result=SUCCESS) self.expect_property('got_revision', self.REVID, 'Monotone') return self.run_step()
def testShellArgsAreRenderedAnewAtEachBuild(self): """Unit test to ensure that ShellArg instances are properly re-rendered. This unit test makes sure that ShellArg instances are rendered anew at each new build. """ arg = shellsequence.ShellArg( command=WithProperties('make %s', 'project')) step = shellsequence.ShellSequence(commands=[arg], workdir='build') # First "build" self.setup_step(step) self.properties.setProperty("project", "BUILDBOT-TEST-1", "TEST") self.expect_commands( ExpectShell(workdir='build', command='make BUILDBOT-TEST-1').exit(0)) self.expect_outcome(result=SUCCESS, state_string="'make BUILDBOT-TEST-1'") yield self.run_step() # Second "build" self.setup_step(step) self.properties.setProperty("project", "BUILDBOT-TEST-2", "TEST") self.expect_commands( ExpectShell(workdir='build', command='make BUILDBOT-TEST-2').exit(0)) self.expect_outcome(result=SUCCESS, state_string="'make BUILDBOT-TEST-2'") yield self.run_step()
def test_mode_full_win32path(self): self.setup_step( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', mode='full', method='fresh')) self.build.path_module = namedModule('ntpath') self.expect_commands( ExpectShell(workdir='wkdir', command=['bzr', '--version']) .exit(0), ExpectStat(file=r'wkdir\.buildbot-patched', log_environ=True) .exit(1), ExpectStat(file=r'wkdir\.bzr', log_environ=True) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'clean-tree', '--force']) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'update']) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'version-info', '--custom', "--template='{revno}"]) .stdout('100') .exit(0) ) self.expect_outcome(result=SUCCESS) return self.run_step()
def test_mode_incremental_branch_change_dirname(self): self.setup_step( mercurial.Mercurial(repourl='http://hg.mozilla.org/', mode='incremental', branchType='dirname', defaultBranch='devel'), dict(branch='stable')) self.expect_commands( ExpectShell(workdir='wkdir', command=['hg', '--verbose', '--version']).exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True).exit(1), ExpectStat(file='wkdir/.hg', log_environ=True).exit(0), ExpectShell(workdir='wkdir', command=[ 'hg', '--verbose', 'pull', 'http://hg.mozilla.org/stable' ]).exit(0), ExpectRmdir(dir='wkdir', log_environ=True).exit(0), ExpectShell(workdir='wkdir', command=[ 'hg', '--verbose', 'clone', '--noupdate', 'http://hg.mozilla.org/stable', '.' ]).exit(0), ExpectShell(workdir='wkdir', command=['hg', '--verbose', 'update', '--clean']).exit(0), ExpectShell( workdir='wkdir', command=[ 'hg', '--verbose', 'parents', '--template', '{node}\\n' ]).stdout('\n').stdout( 'f6ad368298bd941e934a41f3babc827b2aa95a1d').exit(0)) self.expect_outcome(result=SUCCESS) return self.run_step()
def test_mode_full_copy(self): self.setup_step( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', mode='full', method='copy')) self.expect_commands( ExpectShell(workdir='wkdir', command=['bzr', '--version']) .exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True) .exit(1), ExpectRmdir(dir='build', log_environ=True) .exit(0), ExpectStat(file='source/.bzr', log_environ=True) .exit(0), ExpectShell(workdir='source', command=['bzr', 'update']) .exit(0), ExpectCpdir(fromdir='source', log_environ=True, todir='build') .exit(0), ExpectShell(workdir='source', command=['bzr', 'version-info', '--custom', "--template='{revno}"]) .stdout('100') .exit(0) ) self.expect_outcome(result=SUCCESS) self.expect_property('got_revision', '100', 'Bzr') return self.run_step()
def test_mode_incremental_retry(self): self.setup_step( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', mode='incremental', retry=(0, 1))) self.expect_commands( ExpectShell(workdir='wkdir', command=['bzr', '--version']) .exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True) .exit(1), ExpectStat(file='wkdir/.bzr', log_environ=True) .exit(1), ExpectShell(workdir='wkdir', command=['bzr', 'checkout', 'http://bzr.squid-cache.org/bzr/squid3/trunk', '.']) .exit(1), ExpectRmdir(dir='wkdir', log_environ=True) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'checkout', 'http://bzr.squid-cache.org/bzr/squid3/trunk', '.']) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'version-info', '--custom', "--template='{revno}"]) .stdout('100\n') .exit(0) ) self.expect_outcome(result=SUCCESS) self.expect_property('got_revision', '100\n', 'Bzr') return self.run_step()
def test_mode_full_clobber_baseurl_nodefault(self): self.setup_step( bzr.Bzr(baseURL='http://bzr.squid-cache.org/bzr/squid3', defaultBranch='trunk', mode='full', method='clobber'), args=dict(branch='branches/SQUID_3_0')) self.expect_commands( ExpectShell(workdir='wkdir', command=['bzr', '--version']) .exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True) .exit(1), ExpectRmdir(dir='wkdir', log_environ=True) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'checkout', os.path.join('http://bzr.squid-cache.org/bzr/squid3', 'branches/SQUID_3_0'), '.']) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'version-info', '--custom', "--template='{revno}"]) .stdout('100') .exit(0) ) self.expect_outcome(result=SUCCESS) self.expect_property('got_revision', '100', 'Bzr') return self.run_step()
def test_mode_full_clobber_revision(self): self.setup_step( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', mode='full', method='clobber'), args=dict(revision='3730')) self.expect_commands( ExpectShell(workdir='wkdir', command=['bzr', '--version']) .exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True) .exit(1), ExpectRmdir(dir='wkdir', log_environ=True) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'checkout', 'http://bzr.squid-cache.org/bzr/squid3/trunk', '.', '-r', '3730']) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'version-info', '--custom', "--template='{revno}"]) .stdout('100') .exit(0) ) self.expect_outcome(result=SUCCESS) self.expect_property('got_revision', '100', 'Bzr') return self.run_step()
def test_mode_full_clean_patched(self): self.setup_step( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', mode='full', method='clean')) self.expect_commands( ExpectShell(workdir='wkdir', command=['bzr', '--version']) .exit(0), ExpectStat(file='wkdir/.buildbot-patched', log_environ=True) .exit(0), # clean up the applied patch ExpectShell(workdir='wkdir', command=['bzr', 'clean-tree', '--ignored', '--force']) .exit(0), ExpectStat(file='wkdir/.bzr', log_environ=True) .exit(0), # this clean is from 'mode=clean' ExpectShell(workdir='wkdir', command=['bzr', 'clean-tree', '--ignored', '--force']) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'update']) .exit(0), ExpectShell(workdir='wkdir', command=['bzr', 'version-info', '--custom', "--template='{revno}"]) .stdout('100') .exit(0) ) self.expect_outcome(result=SUCCESS) self.expect_property('got_revision', '100', 'Bzr') return self.run_step()
def test_complex_diff(self): self.setup_step(gitdiffinfo.GitDiffInfo()) self.expect_commands( ExpectShell(workdir='wkdir', command=['git', 'merge-base', 'HEAD', 'master' ]).log('stdio-merge-base', stdout='1234123412341234').exit(0), ExpectShell(workdir='wkdir', command=[ 'git', 'diff', '--no-prefix', '-U0', '1234123412341234', 'HEAD' ]).log('stdio-diff', stdout='''\ diff --git file1 file1 deleted file mode 100644 index 42f90fd..0000000 --- file1 +++ /dev/null @@ -1,3 +0,0 @@ -line11 -line12 -line13 diff --git file2 file2 index c337bf1..1cb02b9 100644 --- file2 +++ file2 @@ -4,0 +5,3 @@ line24 +line24n +line24n2 +line24n3 @@ -15,0 +19,3 @@ line215 +line215n +line215n2 +line215n3 diff --git file3 file3 new file mode 100644 index 0000000..632e269 --- /dev/null +++ file3 @@ -0,0 +1,3 @@ +line31 +line32 +line33 ''').exit(0)) self.expect_log_file('stdio-merge-base', '1234123412341234') self.expect_outcome(result=results.SUCCESS, state_string="GitDiffInfo") diff_info = (b'[{"source_file": "file1", "target_file": "/dev/null", ' + b'"is_binary": false, "is_rename": false, ' + b'"hunks": [{"ss": 1, "sl": 3, "ts": 0, "tl": 0}]}, ' + b'{"source_file": "file2", "target_file": "file2", ' + b'"is_binary": false, "is_rename": false, ' + b'"hunks": [{"ss": 4, "sl": 0, "ts": 5, "tl": 3}, ' + b'{"ss": 15, "sl": 0, "ts": 19, "tl": 3}]}, ' + b'{"source_file": "/dev/null", "target_file": "file3", ' + b'"is_binary": false, "is_rename": false, ' + b'"hunks": [{"ss": 0, "sl": 0, "ts": 1, "tl": 3}]}]') self.expect_build_data('diffinfo-master', diff_info, 'GitDiffInfo') return self.run_step()
def testMultipleCommandsAreRun(self): arg1 = shellsequence.ShellArg(command='make p1') arg2 = shellsequence.ShellArg(command='deploy p1') self.setup_step( shellsequence.ShellSequence(commands=[arg1, arg2], workdir='build')) self.expect_commands( ExpectShell(workdir='build', command='make p1').exit(0), ExpectShell(workdir='build', command='deploy p1').exit(0)) self.expect_outcome(result=SUCCESS, state_string="'deploy p1'") return self.run_step()
def testWarningWins(self): arg1 = shellsequence.ShellArg(command='make p1', warnOnFailure=True, flunkOnFailure=False) arg2 = shellsequence.ShellArg(command='deploy p1') self.setup_step( shellsequence.ShellSequence(commands=[arg1, arg2], workdir='build')) self.expect_commands( ExpectShell(workdir='build', command='make p1').exit(1), ExpectShell(workdir='build', command='deploy p1').exit(0)) self.expect_outcome(result=WARNINGS, state_string="'deploy p1' (warnings)") return self.run_step()
def test_plain(self): self.setup_step(CMake()) self.expect_commands( ExpectShell(command=[CMake.DEFAULT_CMAKE], workdir='wkdir').exit(0)) self.expect_outcome(result=SUCCESS) return self.run_step()
def expect_and_run_command(self, *params): command = [CMake.DEFAULT_CMAKE] + list(params) self.expect_commands( ExpectShell(command=command, workdir='wkdir').exit(0)) self.expect_outcome(result=SUCCESS) return self.run_step()
def test_simple(self): self.setup_step(VCx()) self.expect_commands( ExpectShell(workdir='wkdir', command=['command', 'here']).exit(0)) self.expect_outcome(result=SUCCESS, state_string="compile 0 projects 0 files") return self.run_step()
def test_multiple_errors(self): stream = io.BytesIO() client = TestProtocolClient(stream) test1 = FakeTest(id='test1') test2 = FakeTest(id='test2') client.startTest(test1) client.addError(test1, create_error('error1')) client.stopTest(test1) client.startTest(test2) client.addError(test2, create_error('error2')) client.stopTest(test2) self.setup_step(subunit.SubunitShellCommand(command='test')) self.expect_commands( ExpectShell(workdir='wkdir', command="test").stdout(stream.getvalue()).exit(0)) self.expect_outcome( result=FAILURE, state_string="shell Total 2 test(s) 2 errors (failure)") self.expect_log_file('problems', re.compile(r'''test1 testtools.testresult.real._StringException:.*ValueError: invalid literal for int\(\) with base 10: '_error1' test2 testtools.testresult.real._StringException:.*ValueError: invalid literal for int\(\) with base 10: '_error2' .*''', re.MULTILINE | re.DOTALL)) # noqa pylint: disable=line-too-long return self.run_step()
def test_regex_text_2_0_0(self, name, store_results): # pylint 2.0.0 changed default format to include file path self.setup_step(python.PyLint(command=['pylint'], store_results=store_results)) stdout = ( 'test.py:9:4: W0311: Bad indentation. Found 6 spaces, expected 4 (bad-indentation)\n' + 'test.py:1:0: C0114: Missing module docstring (missing-module-docstring)\n' ) self.expect_commands( ExpectShell(workdir='wkdir', command=['pylint']) .stdout(stdout) .exit((python.PyLint.RC_WARNING | python.PyLint.RC_CONVENTION))) self.expect_outcome(result=WARNINGS, state_string='pylint convention=1 warning=1 (warnings)') self.expect_property('pylint-warning', 1) self.expect_property('pylint-convention', 1) self.expect_property('pylint-total', 2) if store_results: self.expect_test_result_sets([('Pylint warnings', 'code_issue', 'message')]) self.expect_test_results([ (1000, 'test.py:9:4: W0311: Bad indentation. Found 6 spaces, expected 4 ' + '(bad-indentation)', None, 'test.py', 9, None), (1000, 'test.py:1:0: C0114: Missing module docstring (missing-module-docstring)', None, 'test.py', 1, None), ]) return self.run_step()
def test_run_jobs(self): """ The C{jobs} kwarg should correspond to trial's -j option ( included since Twisted 12.3.0), and make corresponding changes to logfiles. """ self.setup_step(python_twisted.Trial(workdir='build', tests='testname', testpath=None, jobs=2)) self.expect_commands( ExpectShell(workdir='build', command=['trial', '--reporter=bwverbose', '--jobs=2', 'testname'], logfiles={ 'test.0.log': '_trial_temp/0/test.log', 'err.0.log': '_trial_temp/0/err.log', 'out.0.log': '_trial_temp/0/out.log', 'test.1.log': '_trial_temp/1/test.log', 'err.1.log': '_trial_temp/1/err.log', 'out.1.log': '_trial_temp/1/out.log', }) .stdout("Ran 1 tests\n") .exit(0) ) self.expect_outcome(result=SUCCESS, state_string='1 test passed') return self.run_step()
def test_success(self): self.setup_step(python.PyFlakes()) self.expect_commands( ExpectShell(workdir='wkdir', command=['make', 'pyflakes']) .exit(0)) self.expect_outcome(result=SUCCESS, state_string='pyflakes') return self.run_step()
def test_run(self): self.setup_step(shell.Configure()) self.expect_commands( ExpectShell(workdir='wkdir', command=["./configure"]).exit(0)) self.expect_outcome(result=SUCCESS) return self.run_step()
def test_run_jobsProperties(self): """ C{jobs} should accept Properties """ self.setup_step(python_twisted.Trial(workdir='build', tests='testname', jobs=Property('jobs_count'), testpath=None)) self.properties.setProperty('jobs_count', '2', 'Test') self.expect_commands( ExpectShell(workdir='build', command=['trial', '--reporter=bwverbose', '--jobs=2', 'testname'], logfiles={ 'test.0.log': '_trial_temp/0/test.log', 'err.0.log': '_trial_temp/0/err.log', 'out.0.log': '_trial_temp/0/out.log', 'test.1.log': '_trial_temp/1/test.log', 'err.1.log': '_trial_temp/1/err.log', 'out.1.log': '_trial_temp/1/out.log', }) .stdout("Ran 1 tests\n") .exit(0) ) self.expect_outcome(result=SUCCESS, state_string='1 test passed') return self.run_step()
def test_failures(self): self.setup_step(maxq.MaxQ(testdir='x')) self.expect_commands( ExpectShell(workdir='wkdir', command=["run_maxq.py", "x"]).stdout( '\nTEST FAILURE: foo\n' * 10).exit(2)) self.expect_outcome(result=FAILURE, state_string='10 maxq failures') return self.run_step()
def test_run_simple(self): self.setup_step( shell.ShellCommand(workdir='build', command="echo hello")) self.expect_commands( ExpectShell(workdir='build', command='echo hello').exit(0)) self.expect_outcome(result=SUCCESS, state_string="'echo hello'") return self.run_step()
def _run_simple_test(self, source, destination, expected_args=None, expected_code=0, expected_res=SUCCESS, **kwargs): s = mswin.Robocopy(source, destination, **kwargs) self.setup_step(s) s.rendered = True command = ['robocopy', source, destination] if expected_args: command += expected_args command += ['/TEE', '/NP'] self.expect_commands( ExpectShell( workdir='wkdir', command=command, ).exit(expected_code)) state_string = f"'robocopy {source} ...'" if expected_res != SUCCESS: state_string += f' ({Results[expected_res]})' self.expect_outcome(result=expected_res, state_string=state_string) return self.run_step()
def test_new_version_failed(self): self.setup_step(shell.PerlModuleTest(command="cmd")) self.expect_commands( ExpectShell(workdir='wkdir', command="cmd").stdout( textwrap.dedent("""\ foo.pl .. 1/4""")).stderr( textwrap.dedent("""\ # Failed test 2 in foo.pl at line 6 # foo.pl line 6 is: ok(0);""")).stdout( textwrap.dedent("""\ foo.pl .. Failed 1/4 subtests Test Summary Report ------------------- foo.pl (Wstat: 0 Tests: 4 Failed: 1) Failed test: 0 Files=1, Tests=4, 0 wallclock secs ( 0.06 usr 0.01 sys + 0.03 cusr 0.01 csys = 0.11 CPU) Result: FAIL""")).stderr( textwrap.dedent("""\ Failed 1/1 test programs. 1/4 subtests failed.""")).exit( 1)) self.expect_outcome(result=FAILURE, state_string='4 tests 3 passed 1 failed (failure)') return self.run_step()