def test_build_failure(self): """Verify stdio report works with a build failure.""" responses.add(responses.GET, "http://patchwork.example.com/patch/1/mbox", body="Subject: Patch #1") responses.add(responses.GET, "http://patchwork.example.com/patch/2/mbox", body="Subject: Patch #2") self.basecfg['buildlog'] = self.make_file('build.log', 'build failed') testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'Subject: FAIL: Build failed', 'Overall result: FAILED', 'Merge: OK', 'Compile: FAILED', 'Repo: git://git.example.com/kernel.git', 'http://patchwork.example.com/patch/1', 'http://patchwork.example.com/patch/2', 'We compiled the kernel for 1 architecture:', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)
def test_baseline_success(self, mock_grt): """Verify stdio report works with a successful baseline run.""" responses.add(responses.GET, 'http://example.com', body="Linux version 3.10.0") responses.add(responses.GET, "http://example.com/machinedesc.log", body="Machine information from beaker goes here") mock_grt.return_value = self.beaker_pass_results self.basecfg['retcode'] = '0' self.basecfg['localpatches'] = [] self.basecfg['patchworks'] = [] testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'Subject: PASS: Test report for kernel 3.10.0 (kernel)', 'Overall result: PASS', 'Merge: OK', 'Compile: OK', 'Tests: OK', 'Repo: git://git.example.com/kernel.git', 'We compiled the kernel for 1 architecture:', '- URL: https://github.com/CKI-project/tests-beaker/', 'distribution/kpkginstall', '/test/we/ran', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)
def test_merge_failure_empty_log(self): """Verify stdio report works with a merge failure w/empty log.""" responses.add(responses.GET, "http://patchwork.example.com/patch/1/mbox", body="Subject: Patch #1") responses.add(responses.GET, "http://patchwork.example.com/patch/2/mbox", body="Subject: Patch #2") self.basecfg['mergelog'] = self.make_file('merge.log', '') # The kernel config should not be present after a merge failure. if os.path.isfile(self.tempconfig): os.unlink(self.tempconfig) testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'Subject: FAIL: Patch application failed', 'Overall result: FAILED', 'Merge: FAILED', 'Repo: git://git.example.com/kernel.git', 'http://patchwork.example.com/patch/1', 'http://patchwork.example.com/patch/2', 'When we attempted to merge the patches, we received an error:', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)
def test_multireport_partial_success(self, mock_grt, mock_load): """Verify multireport partial success output.""" # pylint: disable=invalid-name responses.add(responses.GET, "http://patchwork.example.com/patch/1/mbox", body="Subject: Patch #1") responses.add(responses.GET, "http://patchwork.example.com/patch/2/mbox", body="Subject: Patch #2") responses.add(responses.GET, 'http://example.com', body="Linux version 3.10.0") responses.add(responses.GET, "http://example.com/machinedesc.log", body="Machine information from beaker goes here") mock_grt.side_effect = [ self.beaker_fail_results, self.beaker_pass_results, ] self.basecfg['retcode'] = '1' self.basecfg['result'] = ['state1', 'state2'] # Create our two mocked state files for two different arches state1 = self.basecfg.copy() state1['kernel_arch'] = 's390x' state2 = self.basecfg.copy() state2['kernel_arch'] = 'x86_64' # Mock the loading of these state files mock_load.side_effect = [state1, state2] testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'FAIL: Test report for kernel 3.10.0 (kernel)', 'Overall result: FAILED', 'Tests: FAILED', 's390x: FAILED', 'x86_64: PASSED', 'http://patchwork.example.com/patch/1', 'http://patchwork.example.com/patch/2', '- URL: https://github.com/CKI-project/tests-beaker/', 'distribution/kpkginstall', '/test/we/ran', 'We compiled the kernel for 2 architectures:', 'Beaker results:', 'https://beaker.engineering.redhat.com/jobs/2547021', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)
def test_run_waived(self, mock_grt): """ Verify stdio report works and that waived tests are marked clearly in results and that failure on waived test doesn't mean a failed run.""" responses.add(responses.GET, "http://patchwork.example.com/patch/1/mbox", body="Subject: Patch #1") responses.add(responses.GET, "http://patchwork.example.com/patch/2/mbox", body="Subject: Patch #2") responses.add(responses.GET, 'http://example.com', body="Linux version 3.10.0") responses.add(responses.GET, "http://example.com/machinedesc.log", body="Machine information from beaker goes here") # Expected values: # the return xml has fails, but retcode is 0, because waived tests # are hidden mock_grt.return_value = self.beaker_fail_results self.basecfg['retcode'] = '0' testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'Subject: PASS: Test report for kernel 3.10.0 (kernel)', 'Overall result: PASSED', 'Merge: OK', 'Compile: OK', 'Tests: OK', 'Repo: git://git.example.com/kernel.git', 'http://patchwork.example.com/patch/1', 'http://patchwork.example.com/patch/2', '- URL: https://github.com/CKI-project/tests-beaker/', 'distribution/kpkginstall', 'WAIVED: FAILED: /a/failed/waived/test', 'WAIVED: PASSED: /a/passed/waived/test', 'We compiled the kernel for 1 architecture:', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)
def test_run_failure(self, mock_grt): """Verify stdio report works with a failed run.""" responses.add(responses.GET, "http://patchwork.example.com/patch/1/mbox", body="Subject: Patch #1") responses.add(responses.GET, "http://patchwork.example.com/patch/2/mbox", body="Subject: Patch #2") responses.add(responses.GET, 'http://example.com', body="Linux version 3.10.0") responses.add(responses.GET, "http://example.com/machinedesc.log", body="Machine information from beaker goes here") mock_grt.return_value = self.beaker_fail_results self.basecfg['retcode'] = '1' testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) # disable waiving for this test! rptclass.waiving = False rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'Subject: FAIL: Test report for kernel 3.10.0 (kernel)', 'Overall result: FAILED', 'Merge: OK', 'Compile: OK', 'Tests: FAILED', 'Beaker results:', 'https://beaker.engineering.redhat.com/jobs/2547021', 'Repo: git://git.example.com/kernel.git', 'http://patchwork.example.com/patch/1', 'http://patchwork.example.com/patch/2', '- URL: https://github.com/CKI-project/tests-beaker/', 'distribution/kpkginstall', '/a/failed/waived/test', '/a/passed/waived/test', 'We compiled the kernel for 1 architecture:', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)
def test_multireport_panic(self, mock_grt, mock_load): """Verify multireport works with a kernel panic result.""" responses.add(responses.GET, "http://patchwork.example.com/patch/1/mbox", body="Subject: Patch #1") responses.add(responses.GET, "http://patchwork.example.com/patch/2/mbox", body="Subject: Patch #2") responses.add(responses.GET, 'http://example.com', body="Linux version 3.10.0") responses.add(responses.GET, "http://example.com/machinedesc.log", body="Machine information from beaker goes here") mock_grt.return_value = self.beaker_panic_results self.basecfg['retcode'] = '1' self.basecfg['result'] = ['state'] state = self.basecfg.copy() state['kernel_arch'] = 'x86_64' mock_load.side_effect = [state] testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'FAIL: Test report for kernel 3.10.0 (kernel)', 'Overall result: FAILED', 'Tests: FAILED', 'x86_64: FAILED', 'http://patchwork.example.com/patch/1', 'http://patchwork.example.com/patch/2', '- URL: https://github.com/CKI-project/tests-beaker/', 'distribution/kpkginstall', 'http://example.com/console', 'We compiled the kernel for 1 architecture:', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)
def test_run_success_multiple_patches(self, mock_grt): """Verify stdio report works with success + multiple patches.""" # pylint: disable=invalid-name responses.add(responses.GET, "http://patchwork.example.com/patch/1/mbox", body="Subject: Patch #1") responses.add(responses.GET, "http://patchwork.example.com/patch/2/mbox", body="Subject: Patch #2") responses.add(responses.GET, 'http://example.com/', body="Linux version 3.10.0") responses.add(responses.GET, "http://example.com/machinedesc.log", body="Machine information from beaker goes here") mock_grt.return_value = self.beaker_pass_results self.basecfg['retcode'] = '0' testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'Subject: PASS: Test report for kernel 3.10.0 (kernel)', 'Overall result: PASSED', 'Patch merge: OK', 'Compile: OK', 'Kernel tests: OK', 'Repo: git://git.example.com/kernel.git', 'http://patchwork.example.com/patch/1', 'http://patchwork.example.com/patch/2', '- URL: https://github.com/CKI-project/tests-beaker/', 'distribution/kpkginstall', '/test/we/ran', 'We compiled the kernel for 1 architecture:', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)
def test_run_success_no_runner(self): """Verify stdio report works without a runner. Test the case of no 'mergerepos' set and with 'cfgurl' for better coverage. """ responses.add(responses.GET, "http://patchwork.example.com/patch/1/mbox", body="Subject: Patch #1") responses.add(responses.GET, "http://patchwork.example.com/patch/2/mbox", body="Subject: Patch #2") responses.add(responses.GET, "http://example.com/config", body="Config from configurl") self.basecfg['cfgurl'] = "http://example.com/config" del self.basecfg['runner'] testprint = StringIO.StringIO() rptclass = reporter.StdioReporter(self.basecfg) rptclass.report(printer=testprint) report = testprint.getvalue().strip() required_strings = [ 'Subject: PASS: Test report', 'Commit: 1234abcdef None', 'Repo: git://git.example.com/kernel.git', 'Overall result: PASS', 'Merge: OK', 'Compile: OK', 'http://patchwork.example.com/patch/1', 'http://patchwork.example.com/patch/2', 'We then merged the following patches with `git am`:', self.basecfg['basehead'], self.basecfg['baserepo'], ] for required_string in required_strings: self.assertIn(required_string, report)