def test_build_failure(self):
"""Verify stdio report works with a build failure."""
responses.add(responses.GET,
"http://patchwork.example.com/patch/1/mbox",
body="Subject: Patch #1")
responses.add(responses.GET,
"http://patchwork.example.com/patch/2/mbox",
body="Subject: Patch #2")
self.basecfg['buildlog'] = self.make_file('build.log', 'build failed')
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'Subject: FAIL: Build failed',
'Overall result: FAILED',
'Merge: OK',
'Compile: FAILED',
'Repo: git://git.example.com/kernel.git',
'http://patchwork.example.com/patch/1',
'http://patchwork.example.com/patch/2',
'We compiled the kernel for 1 architecture:',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
def test_baseline_success(self, mock_grt):
"""Verify stdio report works with a successful baseline run."""
responses.add(responses.GET,
'http://example.com',
body="Linux version 3.10.0")
responses.add(responses.GET,
"http://example.com/machinedesc.log",
body="Machine information from beaker goes here")
mock_grt.return_value = self.beaker_pass_results
self.basecfg['retcode'] = '0'
self.basecfg['localpatches'] = []
self.basecfg['patchworks'] = []
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'Subject: PASS: Test report for kernel 3.10.0 (kernel)',
'Overall result: PASS',
'Merge: OK',
'Compile: OK',
'Tests: OK',
'Repo: git://git.example.com/kernel.git',
'We compiled the kernel for 1 architecture:',
'- URL: https://github.com/CKI-project/tests-beaker/',
'distribution/kpkginstall',
'/test/we/ran',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
def test_merge_failure_empty_log(self):
"""Verify stdio report works with a merge failure w/empty log."""
responses.add(responses.GET,
"http://patchwork.example.com/patch/1/mbox",
body="Subject: Patch #1")
responses.add(responses.GET,
"http://patchwork.example.com/patch/2/mbox",
body="Subject: Patch #2")
self.basecfg['mergelog'] = self.make_file('merge.log', '')
# The kernel config should not be present after a merge failure.
if os.path.isfile(self.tempconfig):
os.unlink(self.tempconfig)
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'Subject: FAIL: Patch application failed',
'Overall result: FAILED',
'Merge: FAILED',
'Repo: git://git.example.com/kernel.git',
'http://patchwork.example.com/patch/1',
'http://patchwork.example.com/patch/2',
'When we attempted to merge the patches, we received an error:',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
def test_multireport_partial_success(self, mock_grt, mock_load):
"""Verify multireport partial success output."""
# pylint: disable=invalid-name
responses.add(responses.GET,
"http://patchwork.example.com/patch/1/mbox",
body="Subject: Patch #1")
responses.add(responses.GET,
"http://patchwork.example.com/patch/2/mbox",
body="Subject: Patch #2")
responses.add(responses.GET,
'http://example.com',
body="Linux version 3.10.0")
responses.add(responses.GET,
"http://example.com/machinedesc.log",
body="Machine information from beaker goes here")
mock_grt.side_effect = [
self.beaker_fail_results,
self.beaker_pass_results,
]
self.basecfg['retcode'] = '1'
self.basecfg['result'] = ['state1', 'state2']
# Create our two mocked state files for two different arches
state1 = self.basecfg.copy()
state1['kernel_arch'] = 's390x'
state2 = self.basecfg.copy()
state2['kernel_arch'] = 'x86_64'
# Mock the loading of these state files
mock_load.side_effect = [state1, state2]
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'FAIL: Test report for kernel 3.10.0 (kernel)',
'Overall result: FAILED',
'Tests: FAILED',
's390x: FAILED',
'x86_64: PASSED',
'http://patchwork.example.com/patch/1',
'http://patchwork.example.com/patch/2',
'- URL: https://github.com/CKI-project/tests-beaker/',
'distribution/kpkginstall',
'/test/we/ran',
'We compiled the kernel for 2 architectures:',
'Beaker results:',
'https://beaker.engineering.redhat.com/jobs/2547021',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
def test_run_waived(self, mock_grt):
""" Verify stdio report works and that waived tests are marked
clearly in results and that failure on waived test doesn't
mean a failed run."""
responses.add(responses.GET,
"http://patchwork.example.com/patch/1/mbox",
body="Subject: Patch #1")
responses.add(responses.GET,
"http://patchwork.example.com/patch/2/mbox",
body="Subject: Patch #2")
responses.add(responses.GET,
'http://example.com',
body="Linux version 3.10.0")
responses.add(responses.GET,
"http://example.com/machinedesc.log",
body="Machine information from beaker goes here")
# Expected values:
# the return xml has fails, but retcode is 0, because waived tests
# are hidden
mock_grt.return_value = self.beaker_fail_results
self.basecfg['retcode'] = '0'
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'Subject: PASS: Test report for kernel 3.10.0 (kernel)',
'Overall result: PASSED',
'Merge: OK',
'Compile: OK',
'Tests: OK',
'Repo: git://git.example.com/kernel.git',
'http://patchwork.example.com/patch/1',
'http://patchwork.example.com/patch/2',
'- URL: https://github.com/CKI-project/tests-beaker/',
'distribution/kpkginstall',
'WAIVED: FAILED: /a/failed/waived/test',
'WAIVED: PASSED: /a/passed/waived/test',
'We compiled the kernel for 1 architecture:',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
def test_run_failure(self, mock_grt):
"""Verify stdio report works with a failed run."""
responses.add(responses.GET,
"http://patchwork.example.com/patch/1/mbox",
body="Subject: Patch #1")
responses.add(responses.GET,
"http://patchwork.example.com/patch/2/mbox",
body="Subject: Patch #2")
responses.add(responses.GET,
'http://example.com',
body="Linux version 3.10.0")
responses.add(responses.GET,
"http://example.com/machinedesc.log",
body="Machine information from beaker goes here")
mock_grt.return_value = self.beaker_fail_results
self.basecfg['retcode'] = '1'
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
# disable waiving for this test!
rptclass.waiving = False
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'Subject: FAIL: Test report for kernel 3.10.0 (kernel)',
'Overall result: FAILED',
'Merge: OK',
'Compile: OK',
'Tests: FAILED',
'Beaker results:',
'https://beaker.engineering.redhat.com/jobs/2547021',
'Repo: git://git.example.com/kernel.git',
'http://patchwork.example.com/patch/1',
'http://patchwork.example.com/patch/2',
'- URL: https://github.com/CKI-project/tests-beaker/',
'distribution/kpkginstall',
'/a/failed/waived/test',
'/a/passed/waived/test',
'We compiled the kernel for 1 architecture:',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
def test_multireport_panic(self, mock_grt, mock_load):
"""Verify multireport works with a kernel panic result."""
responses.add(responses.GET,
"http://patchwork.example.com/patch/1/mbox",
body="Subject: Patch #1")
responses.add(responses.GET,
"http://patchwork.example.com/patch/2/mbox",
body="Subject: Patch #2")
responses.add(responses.GET,
'http://example.com',
body="Linux version 3.10.0")
responses.add(responses.GET,
"http://example.com/machinedesc.log",
body="Machine information from beaker goes here")
mock_grt.return_value = self.beaker_panic_results
self.basecfg['retcode'] = '1'
self.basecfg['result'] = ['state']
state = self.basecfg.copy()
state['kernel_arch'] = 'x86_64'
mock_load.side_effect = [state]
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'FAIL: Test report for kernel 3.10.0 (kernel)',
'Overall result: FAILED',
'Tests: FAILED',
'x86_64: FAILED',
'http://patchwork.example.com/patch/1',
'http://patchwork.example.com/patch/2',
'- URL: https://github.com/CKI-project/tests-beaker/',
'distribution/kpkginstall',
'http://example.com/console',
'We compiled the kernel for 1 architecture:',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
def test_run_success_multiple_patches(self, mock_grt):
"""Verify stdio report works with success + multiple patches."""
# pylint: disable=invalid-name
responses.add(responses.GET,
"http://patchwork.example.com/patch/1/mbox",
body="Subject: Patch #1")
responses.add(responses.GET,
"http://patchwork.example.com/patch/2/mbox",
body="Subject: Patch #2")
responses.add(responses.GET,
'http://example.com/',
body="Linux version 3.10.0")
responses.add(responses.GET,
"http://example.com/machinedesc.log",
body="Machine information from beaker goes here")
mock_grt.return_value = self.beaker_pass_results
self.basecfg['retcode'] = '0'
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'Subject: PASS: Test report for kernel 3.10.0 (kernel)',
'Overall result: PASSED',
'Patch merge: OK',
'Compile: OK',
'Kernel tests: OK',
'Repo: git://git.example.com/kernel.git',
'http://patchwork.example.com/patch/1',
'http://patchwork.example.com/patch/2',
'- URL: https://github.com/CKI-project/tests-beaker/',
'distribution/kpkginstall',
'/test/we/ran',
'We compiled the kernel for 1 architecture:',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
def test_run_success_no_runner(self):
"""Verify stdio report works without a runner.
Test the case of no 'mergerepos' set and with 'cfgurl' for better
coverage.
"""
responses.add(responses.GET,
"http://patchwork.example.com/patch/1/mbox",
body="Subject: Patch #1")
responses.add(responses.GET,
"http://patchwork.example.com/patch/2/mbox",
body="Subject: Patch #2")
responses.add(responses.GET,
"http://example.com/config",
body="Config from configurl")
self.basecfg['cfgurl'] = "http://example.com/config"
del self.basecfg['runner']
testprint = StringIO.StringIO()
rptclass = reporter.StdioReporter(self.basecfg)
rptclass.report(printer=testprint)
report = testprint.getvalue().strip()
required_strings = [
'Subject: PASS: Test report',
'Commit: 1234abcdef None',
'Repo: git://git.example.com/kernel.git',
'Overall result: PASS',
'Merge: OK',
'Compile: OK',
'http://patchwork.example.com/patch/1',
'http://patchwork.example.com/patch/2',
'We then merged the following patches with `git am`:',
self.basecfg['basehead'],
self.basecfg['baserepo'],
]
for required_string in required_strings:
self.assertIn(required_string, report)
