def test_no_policies_enforced(self): """ Assert correct behavior when policies are not enforced. When test gating is disabled, each Update's test_gating_status will be None. """ update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing # Clear pending messages self.db.info['messages'] = [] update.test_gating_status = None self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post') as mock_greenwave: mock_greenwave.return_value = RuntimeError('The error was blablabla') check_policies_main() update = self.db.query(models.Update).filter(models.Update.id == update.id).one() # The test_gating_status should still be None. assert update.test_gating_status is None expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable', 'subject': [ {'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build'}, {'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update'}], 'verbose': False } mock_greenwave.assert_called_once_with(config['greenwave_api_url'] + '/decision', expected_query)
def test_policies_satisfied(self): """Assert correct behavior when the policies enforced by Greenwave are satisfied""" update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing update.critpath = True # Clear pending messages self.db.info['messages'] = [] self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post') as mock_greenwave: greenwave_responses = [ { 'policies_satisfied': True, 'summary': 'All required tests passed', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements', 'bodhiupdate_bodhipush_openqa' ], 'satisfied_requirements': [ { 'result_id': 39603316, 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-passed' }, ], 'unsatisfied_requirements': [] }, { 'policies_satisfied': True, 'summary': 'no tests are required', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [], } ] mock_greenwave.side_effect = greenwave_responses check_policies_main() update = self.db.query(models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.passed expected_queries = [ { 'product_version': 'fedora-17', 'decision_context': context, 'subject': [ {'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build'}, {'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update'}], 'verbose': False } for context in ('bodhi_update_push_stable_critpath', 'bodhi_update_push_stable') ] expected_calls = [ call(config['greenwave_api_url'] + '/decision', query) for query in expected_queries ] assert mock_greenwave.call_args_list == expected_calls
def test_policies_pending_satisfied(self): """Assert that Updates whose status is pending are checked against greenwave with the ``bodhi_update_push_testing`` decision context. """ update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.pending update.critpath = True self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post' ) as mock_greenwave: greenwave_response = { 'policies_satisfied': True, 'summary': 'All required tests passed', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements', 'bodhiupdate_bodhipush_openqa' ], 'satisfied_requirements': [ { 'result_id': 39603316, 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-passed' }, ], 'unsatisfied_requirements': [] } mock_greenwave.return_value = greenwave_response check_policies_main() update = self.db.query( models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.passed expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_testing_critpath', 'subject': [{ 'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build' }, { 'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update' }], 'verbose': False, } mock_greenwave.assert_called_once_with( config['greenwave_api_url'] + '/decision', expected_query)
def test_unrestricted_policy(self): """Assert correct behavior when an unrestricted policy is applied""" update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing # Clear pending messages self.db.info['messages'] = [] self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post' ) as mock_greenwave: greenwave_response = { 'policies_satisfied': True, 'summary': 'no tests are required', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [], } mock_greenwave.return_value = greenwave_response check_policies_main() update = self.db.query( models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.ignored # Check for the comment expected_comment = "This update's test gating status has been changed to 'ignored'." assert update.comments[-1].text == expected_comment expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable', 'subject': [{ 'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build' }, { 'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update' }], 'verbose': False } mock_greenwave.assert_called_once_with( config['greenwave_api_url'] + '/decision', expected_query)
def test_pushed_update(self): """Assert that check() operates on pushed updates.""" update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing # Clear pending messages self.db.info['messages'] = [] update.pushed = True self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post' ) as mock_greenwave: greenwave_response = { 'policies_satisfied': False, 'summary': 'it broke', 'applicable_policies': ['bodhi-unrestricted'], } mock_greenwave.return_value = greenwave_response check_policies_main() update = self.db.query( models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.failed expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable', 'subject': [{ 'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build' }, { 'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update' }], 'verbose': False } mock_greenwave.assert_called_once_with( config['greenwave_api_url'] + '/decision', expected_query) # Check for the comment expected_comment = "This update's test gating status has been changed to 'failed'." assert update.comments[-1].text == expected_comment
def test_archived_release_updates(self): """Assert that updates for archived releases isn't being considered by the script. """ # Archive the F17 release rel = self.db.query(models.Release).filter_by(name='F17').one() rel.state = models.ReleaseState.archived self.db.commit() update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing # Clear pending messages self.db.info['messages'] = [] self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post') as mock_greenwave: mock_greenwave.side_effect = Exception( 'Greenwave should not be accessed for archived releases.') check_policies_main() assert mock_greenwave.call_count == 0
def test_policies_satisfied(self): """Assert correct behavior when the policies enforced by Greenwave are satisfied""" update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing # Clear pending messages self.db.info['messages'] = [] self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post' ) as mock_greenwave: greenwave_response = { 'policies_satisfied': True, 'summary': 'All tests passed', 'applicable_policies': ['taskotron_release_critical_tasks'], 'unsatisfied_requirements': [] } mock_greenwave.return_value = greenwave_response check_policies_main() update = self.db.query( models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.passed expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable', 'subject': [{ 'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build' }, { 'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update' }], 'verbose': False } mock_greenwave.assert_called_once_with( config['greenwave_api_url'] + '/decision', expected_query)
def test_policies_unsatisfied(self): """Assert correct behavior when the policies enforced by Greenwave are unsatisfied""" update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing # Clear pending messages self.db.info['messages'] = [] self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post' ) as mock_greenwave: greenwave_response = { 'policies_satisfied': False, 'summary': '1 of 2 tests are failed', 'applicable_policies': ['taskotron_release_critical_tasks'], 'unsatisfied_requirements': [{ 'testcase': 'dist.rpmdeplint', 'item': { 'item': 'glibc-1.0-1.f26', 'type': 'koji_build' }, 'type': 'test-result-missing', 'scenario': None }, { 'testcase': 'dist.rpmdeplint', 'item': { 'item': update.alias, 'type': 'bodhi_update' }, 'type': 'test-result-missing', 'scenario': None }] } mock_greenwave.return_value = greenwave_response check_policies_main() update = self.db.query( models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.failed # Check for the comment expected_comment = "This update's test gating status has been changed to 'failed'." assert update.comments[-1].text == expected_comment expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable', 'subject': [{ 'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build' }, { 'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update' }], 'verbose': False } mock_greenwave.assert_called_once_with( config['greenwave_api_url'] + '/decision', expected_query)
def test_pushed_update(self): """Assert that check() operates on pushed updates.""" update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing # Clear pending messages self.db.info['messages'] = [] update.critpath = True update.pushed = True self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post') as mock_greenwave: item = 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year) greenwave_responses = [ { 'policies_satisfied': False, 'summary': '1 of 2 required tests failed, 1 result missing', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements', 'bodhiupdate_bodhipush_openqa' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [ { 'item': { 'item': item, 'type': 'bodhi_update' }, 'scenario': 'fedora.updates-everything-boot-iso.x86_64.64bit', 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-failed' }, { 'item': { 'item': item, 'type': 'bodhi_update' }, 'scenario': 'fedora.updates-everything-boot-iso.x86_64.uefi', 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-missing' }, ] }, { 'policies_satisfied': True, 'summary': 'no tests are required', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [], } ] mock_greenwave.side_effect = greenwave_responses check_policies_main() update = self.db.query(models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.failed expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable_critpath', 'subject': [{'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build'}, {'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update'}], 'verbose': False } # we only expect *one* call here, as with the earlier # 'failed' test mock_greenwave.assert_called_once_with(config['greenwave_api_url'] + '/decision', expected_query) # Check for the comment expected_comment = "This update's test gating status has been changed to 'failed'." assert update.comments[-1].text == expected_comment
def test_policies_unsatisfied_failed(self): """Assert correct behavior when the policies enforced by Greenwave are unsatisfied: failed tests always means failed status. This also tests that we behave correctly even if the *first* query shows requirements satisfied, but the *second* query has failed required tests. """ update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing update.critpath = True update.date_submitted = datetime.datetime.utcnow() # Clear pending messages self.db.info['messages'] = [] self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post') as mock_greenwave: # here, we're mocking the scenario from # https://pagure.io/fedora-ci/general/issue/263 , where # openQA tests passed, but a package in the update had a # local gating config that only specified the context # bodhi_update_push_stable (not _push_stable_critpath), # and a test specified in that local policy failed greenwave_responses = [ { 'policies_satisfied': True, 'summary': 'All required tests passed', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements', 'bodhiupdate_bodhipush_openqa' ], 'satisfied_requirements': [ { 'result_id': 39603316, 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-passed' }, ], 'unsatisfied_requirements': [] }, { 'policies_satisfied': False, 'summary': '1 of 1 required tests failed', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [ { 'item': { 'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build' }, 'scenario': None, 'subject_type': 'koji_build', 'testcase': 'fedora-ci.koji-build.tier0.functional', 'type': 'test-result-failed' }, ], } ] mock_greenwave.side_effect = greenwave_responses check_policies_main() update = self.db.query(models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.failed # Check for the comment expected_comment = "This update's test gating status has been changed to 'failed'." assert update.comments[-1].text == expected_comment expected_queries = [ { 'product_version': 'fedora-17', 'decision_context': context, 'subject': [ {'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build'}, {'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update'}], 'verbose': False } for context in ('bodhi_update_push_stable_critpath', 'bodhi_update_push_stable') ] expected_calls = [ call(config['greenwave_api_url'] + '/decision', query) for query in expected_queries ] assert mock_greenwave.call_args_list == expected_calls
def test_policies_unsatisfied_waiting_too_long(self): """Assert correct behavior when the policies enforced by Greenwave are unsatisfied: results missing, no failures, more than two hours since update modification results in 'failed' status. """ update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing update.critpath = True # Clear pending messages self.db.info['messages'] = [] update.date_submitted = datetime.datetime.utcnow() - datetime.timedelta(days=1) self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post') as mock_greenwave: item = 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year) greenwave_responses = [ { 'policies_satisfied': False, 'summary': '2 of 2 required test results missing', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements', 'bodhiupdate_bodhipush_openqa' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [ { 'item': { 'item': item, 'type': 'bodhi_update' }, 'scenario': 'fedora.updates-everything-boot-iso.x86_64.64bit', 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-missing' }, { 'item': { 'item': item, 'type': 'bodhi_update' }, 'scenario': 'fedora.updates-everything-boot-iso.x86_64.uefi', 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-missing' }, ] }, { 'policies_satisfied': True, 'summary': 'no tests are required', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [], } ] mock_greenwave.side_effect = greenwave_responses check_policies_main() update = self.db.query(models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.failed # Check for the comment expected_comment = "This update's test gating status has been changed to 'failed'." assert update.comments[-1].text == expected_comment expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable_critpath', 'subject': [ {'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build'}, {'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update'}], 'verbose': False } # we only expect *one* call here because the *first* query # (on the _critpath context) should be enough to conclude the # status is failed: it would be wrong to needlessly run the # second query. note the mock responses are in the order we # expect the queries to be run, critpath first mock_greenwave.assert_called_once_with(config['greenwave_api_url'] + '/decision', expected_query)
def test_policies_unsatisfied_waiting(self): """Assert correct behavior when the policies enforced by Greenwave are unsatisfied: results missing, no failures, less than two hours since update creation results in 'waiting' status. """ update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing update.critpath = True # Clear pending messages self.db.info['messages'] = [] update.date_submitted = datetime.datetime.utcnow() self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post') as mock_greenwave: item = 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year) greenwave_responses = [ { 'policies_satisfied': False, 'summary': '2 of 2 required test results missing', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements', 'bodhiupdate_bodhipush_openqa' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [ { 'item': { 'item': item, 'type': 'bodhi_update' }, 'scenario': 'fedora.updates-everything-boot-iso.x86_64.64bit', 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-missing' }, { 'item': { 'item': item, 'type': 'bodhi_update' }, 'scenario': 'fedora.updates-everything-boot-iso.x86_64.uefi', 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-missing' }, ] }, { 'policies_satisfied': True, 'summary': 'no tests are required', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [], } ] mock_greenwave.side_effect = greenwave_responses check_policies_main() update = self.db.query(models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.waiting # Check for the comment expected_comment = "This update's test gating status has been changed to 'waiting'." assert update.comments[-1].text == expected_comment expected_queries = [ { 'product_version': 'fedora-17', 'decision_context': context, 'subject': [ {'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build'}, {'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update'}], 'verbose': False } for context in ('bodhi_update_push_stable_critpath', 'bodhi_update_push_stable') ] expected_calls = [ call(config['greenwave_api_url'] + '/decision', query) for query in expected_queries ] assert mock_greenwave.call_args_list == expected_calls
def test_policies_unsatisfied_failed(self): """Assert correct behavior when the policies enforced by Greenwave are unsatisfied: failed tests always means failed status. """ update = self.db.query(models.Update).all()[0] update.status = models.UpdateStatus.testing update.critpath = True update.date_submitted = datetime.datetime.utcnow() # Clear pending messages self.db.info['messages'] = [] self.db.commit() with patch('bodhi.server.models.util.greenwave_api_post' ) as mock_greenwave: greenwave_response = { 'policies_satisfied': False, 'summary': '1 of 2 required tests failed, 1 result missing', 'applicable_policies': [ 'kojibuild_bodhipush_no_requirements', 'kojibuild_bodhipush_remoterule', 'bodhiupdate_bodhipush_no_requirements', 'bodhiupdate_bodhipush_openqa' ], 'satisfied_requirements': [], 'unsatisfied_requirements': [ { 'item': { 'item': 'FEDORA-{}-a3bbe1a8f2'.format( datetime.datetime.utcnow().year), 'type': 'bodhi_update' }, 'scenario': 'fedora.updates-everything-boot-iso.x86_64.64bit', 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-failed' }, { 'item': { 'item': 'FEDORA-{}-a3bbe1a8f2'.format( datetime.datetime.utcnow().year), 'type': 'bodhi_update' }, 'scenario': 'fedora.updates-everything-boot-iso.x86_64.uefi', 'subject_type': 'bodhi_update', 'testcase': 'update.install_default_update_netinst', 'type': 'test-result-missing' }, ] } mock_greenwave.return_value = greenwave_response check_policies_main() update = self.db.query( models.Update).filter(models.Update.id == update.id).one() assert update.test_gating_status == models.TestGatingStatus.failed # Check for the comment expected_comment = "This update's test gating status has been changed to 'failed'." assert update.comments[-1].text == expected_comment expected_query = { 'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable_critpath', 'subject': [{ 'item': 'bodhi-2.0-1.fc17', 'type': 'koji_build' }, { 'item': 'FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year), 'type': 'bodhi_update' }], 'verbose': False } mock_greenwave.assert_called_once_with( config['greenwave_api_url'] + '/decision', expected_query)