def test_group_exceed_max_testcases(self): """Test that group auto-shrinks when it exceeds maximum number of testcases.""" for i in range(1, 31): testcase = test_utils.create_generic_testcase() testcase.crash_type = 'Heap-buffer-overflow' testcase.crash_state = 'abcdefgh' + str(i) testcase.project_name = 'project' testcase.one_time_crasher_flag = False # Attach actual issues to some testcases. if i in [3, 4, 5]: testcase.bug_information = '123' # Make some testcases unreproducible. if i in [1, 2, 3]: testcase.one_time_crasher_flag = True testcase.put() unrelated_testcase = test_utils.create_generic_testcase() grouper.group_testcases() testcase_ids = list(data_handler.get_open_testcase_id_iterator()) # [1, 2] get removed since they are unreproducible testcases. # [3] is not removed since it has bug attached (even though unreproducible). # [6, 7, 8] are removed to account for max group size. Even though they # are reproducible, they are the ones with least weight. expected_testcase_ids = [3, 4, 5] + list(range( 9, 31)) + [unrelated_testcase.key.id()] self.assertEqual(expected_testcase_ids, testcase_ids)
def test_similar_testcase_reproducible_and_closed_but_issue_open_2(self): """Tests result is true when there is a similar testcase which is reproducible and fixed due to flakiness but issue is kept open. Don't update testcase bug mapping if another reproducible testcase is open and attached to this bug.""" self.issue.save() similar_testcase_1 = test_utils.create_generic_testcase() similar_testcase_1.one_time_crasher_flag = False similar_testcase_1.open = False similar_testcase_1.bug_information = str(self.issue.id) similar_testcase_1.put() similar_testcase_2 = test_utils.create_generic_testcase() similar_testcase_2.one_time_crasher_flag = False similar_testcase_2.open = True similar_testcase_2.bug_information = str(self.issue.id) similar_testcase_2.put() self.assertEqual( True, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker)) testcase = data_handler.get_testcase_by_id(self.testcase.key.id()) self.assertEqual(None, testcase.bug_information) self.assertEqual('', self.issue._monorail_issue.comment)
def test_is_crash_important_4(self): """If the unreproducible testcase has another reproducible testcase in group, then crash is not important.""" testcase_1 = test_utils.create_generic_testcase() testcase_1.one_time_crasher_flag = True testcase_1.group_id = 1 testcase_1.put() testcase_2 = test_utils.create_generic_testcase() testcase_2.one_time_crasher_flag = False testcase_2.group_id = 1 testcase_2.put() self.assertFalse(triage._is_crash_important(testcase_1))
def test_similar_testcase_reproducible_and_closed_but_issue_open_1(self): """Tests result is true when there is a similar testcase which is reproducible and fixed due to flakiness but issue is kept open. Only update testcase bug mapping if similar testcase is fixed longer than the grace period.""" self.issue.save() similar_testcase = test_utils.create_generic_testcase() similar_testcase.one_time_crasher_flag = False similar_testcase.open = False similar_testcase.bug_information = str(self.issue.id) similar_testcase.put() self.assertEqual( True, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker)) testcase = data_handler.get_testcase_by_id(self.testcase.key.id()) self.assertEqual(None, testcase.bug_information) self.assertEqual('', self.issue._monorail_issue.comment) similar_testcase.set_metadata( 'closed_time', test_utils.CURRENT_TIME - datetime.timedelta(hours=data_types.MIN_ELAPSED_TIME_SINCE_FIXED + 1)) self.assertEqual( True, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def test_similar_testcase_with_issue_recently_closed(self): """Tests result is true when there is a similar testcase with issue closed recently.""" self.issue.status = 'Fixed' self.issue._monorail_issue.open = False self.issue._monorail_issue.closed = ( test_utils.CURRENT_TIME - datetime.timedelta( hours=data_types.MIN_ELAPSED_TIME_SINCE_FIXED - 1)) self.issue.save() similar_testcase = test_utils.create_generic_testcase() similar_testcase.one_time_crasher_flag = False similar_testcase.open = False similar_testcase.bug_information = str(self.issue.id) similar_testcase.put() self.assertEqual( True, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker)) testcase = data_handler.get_testcase_by_id(self.testcase.key.id()) self.assertEqual( 'Delaying filing a bug since similar testcase (2) in issue (1) ' 'was just fixed.', testcase.get_metadata(triage.TRIAGE_MESSAGE_KEY))
def test_is_crash_important_9(self): """If this unreproducible testcase is crashing frequently, but its crash type is one of crash type ignores, then it is not an important crash.""" self.mock.get_last_successful_hour.return_value = 417325 indices = [{ 'count': 10, 'hour': day_index } for day_index in range(417325, 416989, -24)] self.mock.get.return_value = (1, [{ 'totalCount': 140, 'groups': [ { 'indices': indices, 'name': 'false', }, ] }]) testcase = test_utils.create_generic_testcase() testcase.one_time_crasher_flag = True testcase.put() for crash_type in ['Out-of-memory', 'Stack-overflow', 'Timeout']: testcase.crash_type = crash_type self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_1(self): """Ensure that a reproducible testcase is important.""" testcase = test_utils.create_generic_testcase() testcase.one_time_crasher_flag = False testcase.put() self.assertTrue(triage._is_crash_important(testcase))
def setUp(self): helpers.patch(self, [ 'libs.issue_management.issue_filer.file_issue', ]) self.testcase = test_utils.create_generic_testcase() self.issue = appengine_test_utils.create_generic_issue() self.issue_tracker = self.issue.issue_tracker
def test_similar_testcase_without_bug_information(self): """Tests result is false when there is a similar testcase but without an associated bug.""" similar_testcase = test_utils.create_generic_testcase() # pylint: disable=unused-variable self.assertEqual( False, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def setUp(self): helpers.patch(self, [ 'clusterfuzz._internal.base.utils.utcnow', ]) self.mock.utcnow.return_value = test_utils.CURRENT_TIME self.testcase = test_utils.create_generic_testcase() self.issue = appengine_test_utils.create_generic_issue() self.issue_tracker = self.issue.issue_tracker
def test_blackbox_fuzzer_job(self): """Test variant task with blackbox fuzzer job.""" testcase = test_utils.create_generic_testcase() testcase.job_type = 'linux_asan_project' testcase.put() variant_testcase = variant_task._get_variant_testcase_for_job( # pylint: disable=protected-access testcase, 'linux_msan_project') self.assertEqual(testcase, variant_testcase)
def test_is_crash_important_3(self): """Ensure that an unreproducible testcase with status Duplicate is not important.""" testcase = test_utils.create_generic_testcase() testcase.one_time_crasher_flag = True testcase.status = 'Duplicate' testcase.put() self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_5(self): """If we don't have any crash stats data for this unreproducible testcase, then we can't make judgement on crash importance, so we return result as False.""" self.mock.get_last_successful_hour.return_value = None testcase = test_utils.create_generic_testcase() testcase.one_time_crasher_flag = True testcase.put() self.assertFalse(triage._is_crash_important(testcase))
def test_similar_testcase_get_issue_failed(self): """Tests result is false when there is a similar testcase with an associated bug but we are unable to fetch it via get_issue.""" similar_testcase = test_utils.create_generic_testcase() similar_testcase.bug_information = '2' # Non-existent. similar_testcase.put() self.assertEqual( False, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def test_similar_testcase_is_reproducible_and_open(self): """Tests result is true when there is a similar testcase which is reproducible, open and has an accessible associated bug.""" self.issue.save() similar_testcase = test_utils.create_generic_testcase() similar_testcase.one_time_crasher_flag = False similar_testcase.open = True similar_testcase.bug_information = str(self.issue.id) similar_testcase.put() self.assertEqual( True, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def test_similar_testcase_unreproducible_but_issue_open(self): """Tests result is true when there is a similar testcase which is unreproducible but issue is kept open. Update testcase bug mapping always since this testcase is reproducible.""" self.issue.save() similar_testcase = test_utils.create_generic_testcase() similar_testcase.one_time_crasher_flag = True similar_testcase.open = False similar_testcase.bug_information = str(self.issue.id) similar_testcase.put() self.assertEqual( True, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def test_no_default_issue_metadata(self): """Test when we only have testcase metadata and no default issue metadata.""" self.mock.get_all_issue_metadata_for_testcase.return_value = { 'issue_owners': '[email protected],[email protected]', 'issue_components': 'component1', 'issue_labels': 'label1,label2,label3' } testcase = test_utils.create_generic_testcase() analyze_task._add_default_issue_metadata(testcase) # pylint: disable=protected-access self.assertEqual('[email protected],[email protected]', testcase.get_metadata('issue_owners')) self.assertEqual('component1', testcase.get_metadata('issue_components')) self.assertEqual('label1,label2,label3', testcase.get_metadata('issue_labels')) self.assertEqual(3, self.mock.log.call_count)
def setUp(self): helpers.patch_environ(self) data_types.Job( name='linux_asan_chrome', environment_string=('APP_ARGS = --orig-arg1 --orig-arg2')).put() data_types.Job( name='linux_msan_chrome_variant', environment_string=( 'APP_ARGS = --arg1 --arg2 --arg3="--flag1 --flag2"')).put() data_types.Job(name='libfuzzer_asan_chrome', environment_string=('')).put() data_types.Job(name='libfuzzer_msan_chrome_variant', environment_string=('')).put() data_types.Job(name='afl_asan_chrome_variant', environment_string=('')).put() self.testcase = test_utils.create_generic_testcase()
def test_is_crash_important_7(self): """If this unreproducible testcase spiked only for a certain interval, then it is not important.""" self.mock.get_last_successful_hour.return_value = 417325 self.mock.get.return_value = (1, [{ 'totalCount': 125, 'groups': [{ 'indices': [{ 'count': 125, 'hour': 417301, }], 'name': 'false', },] }]) testcase = test_utils.create_generic_testcase() testcase.one_time_crasher_flag = True testcase.put() self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_6(self): """If this unreproducible testcase is less than the total crash threshold, then it is not important.""" self.mock.get_last_successful_hour.return_value = 417325 indices = [{ 'count': 1, 'hour': day_index } for day_index in range(417325, 416989, -24)] self.mock.get.return_value = (1, [{ 'totalCount': 14, 'groups': [{ 'indices': indices, 'name': 'false', },] }]) testcase = test_utils.create_generic_testcase() testcase.one_time_crasher_flag = True testcase.put() self.assertFalse(triage._is_crash_important(testcase))
def test_custom_binary(self): """Test custom binary.""" environment.set_value('CUSTOM_BINARY', True) testcase = test_utils.create_generic_testcase() expected_result = { 'result': { 'error_message': 'Not applicable to custom binaries.', 'feedback_url': '', 'found': False, 'project': '', 'suspected_components': '', 'suspected_cls': '', } } self.assertIsNone(blame_task._prepare_predator_message(testcase)) testcase = data_handler.get_testcase_by_id(testcase.key.id()) actual_result = testcase.get_metadata('predator_result') self.assertEqual(actual_result, expected_result) environment.remove_key('CUSTOM_BINARY')
def test_similar_testcase_with_issue_closed_with_ignore_label(self): """Tests result is true when there is a similar testcase with closed issue blacklisted with ignore label.""" self.issue.status = 'WontFix' self.issue._monorail_issue.open = False self.issue.labels.add('ClusterFuzz-Ignore') self.issue.save() similar_testcase = test_utils.create_generic_testcase() similar_testcase.one_time_crasher_flag = False similar_testcase.open = False similar_testcase.bug_information = str(self.issue.id) similar_testcase.put() self.assertEqual( True, triage._check_and_update_similar_bug(self.testcase, self.issue_tracker)) testcase = data_handler.get_testcase_by_id(self.testcase.key.id()) self.assertEqual( 'Skipping filing a bug since similar testcase (2) in issue (1) ' 'is blacklisted with ClusterFuzz-Ignore label.', testcase.get_metadata(triage.TRIAGE_MESSAGE_KEY))
def test_union(self): """Test union of current testcase metadata and default issue metadata.""" self.mock.get_all_issue_metadata_for_testcase.return_value = { 'issue_owners': '[email protected], [email protected]', 'issue_components': 'component1', 'issue_labels': 'label1, label2 ,label3' } testcase = test_utils.create_generic_testcase() testcase.set_metadata('issue_owners', '[email protected],[email protected]') testcase.set_metadata('issue_components', 'component2') testcase.set_metadata('issue_labels', 'label4,label5, label2,') analyze_task._add_default_issue_metadata(testcase) # pylint: disable=protected-access self.assertEqual( '[email protected],[email protected],[email protected]', testcase.get_metadata('issue_owners')) self.assertEqual('component1,component2', testcase.get_metadata('issue_components')) self.assertEqual('label1,label2,label3,label4,label5', testcase.get_metadata('issue_labels')) self.assertEqual(3, self.mock.log.call_count)
def test_reproducible_invalid_regression_range(self): """Test reproducible with invalid regression range.""" testcase = test_utils.create_generic_testcase() testcase.one_time_crasher_flag = False testcase.regression = 'BAD' testcase.put() expected_result = { 'result': { 'error_message': 'Invalid regression range BAD.', 'feedback_url': '', 'found': False, 'project': '', 'suspected_components': '', 'suspected_cls': '', } } self.assertIsNone(blame_task._prepare_predator_message(testcase)) testcase = data_handler.get_testcase_by_id(testcase.key.id()) actual_result = testcase.get_metadata('predator_result') self.assertEqual(actual_result, expected_result)
def test_reproducible_regression_range(self): """Test reproducible with regression range.""" testcase = test_utils.create_generic_testcase() testcase.crash_revision = 398287 testcase.regression = '398287:399171' testcase.put() expected_message = { 'stack_trace': 'crashy_function()', 'customized_data': { 'crash_address': '0xdeadbeef', 'job_type': 'test_content_shell_drt', 'dependencies': [{ 'dep_path': 'src/third_party/bidichecker', 'repo_url': 'https://chromium.googlesource.com/' 'external/bidichecker/lib', 'revision': '97f2aa645b74c28c57eca56992235c79850fa9e0' }, { 'dep_path': 'src/third_party/pdfium', 'repo_url': 'https://pdfium.googlesource.com/pdfium', 'revision': 'f7e108b2d0c2f67a143e99693df084bfff7037ec' }, { 'dep_path': 'src/third_party/skia', 'repo_url': 'https://skia.googlesource.com/skia', 'revision': 'ee295645bd91fcbe1714847c5fe5341759037cc5' }, { 'dep_path': 'src/v8', 'repo_url': 'https://chromium.googlesource.com/v8/v8', 'revision': 'cba1fdd4d72e7c5b874f9eeb07901792f26c871a' }, { 'dep_path': 'src', 'repo_url': 'https://chromium.googlesource.com/chromium/src', 'revision': 'e4eb2a57c8667ab31903237e3c316fcaf4afe718' }], 'dependency_rolls': [{ 'dep_path': 'src/third_party/gperf', 'repo_url': 'https://chromium.googlesource.com/chromium/deps/gperf', 'new_revision': '97f2aa645b74c28c57eca56992235c79850fa9e0' }, { 'dep_path': 'src/third_party/pdfium', 'repo_url': 'https://pdfium.googlesource.com/pdfium', 'new_revision': '855665d4889853f8ac71519de8ff004dba8eb056', 'old_revision': 'f7e108b2d0c2f67a143e99693df084bfff7037ec' }, { 'dep_path': 'src/third_party/skia', 'repo_url': 'https://skia.googlesource.com/skia', 'new_revision': '4772bd537d153922cd772020e4ad4820090be51a', 'old_revision': 'ee295645bd91fcbe1714847c5fe5341759037cc5' }, { 'dep_path': 'src/v8', 'repo_url': 'https://chromium.googlesource.com/v8/v8', 'new_revision': '3a590058de9b3640f73741b1e95f815f5c089988', 'old_revision': 'cba1fdd4d72e7c5b874f9eeb07901792f26c871a' }, { 'dep_path': 'src', 'repo_url': 'https://chromium.googlesource.com/chromium/src', 'new_revision': '52523d4e58d99cdb768791bf9ac532c917522460', 'old_revision': 'e4eb2a57c8667ab31903237e3c316fcaf4afe718' }], 'regression_range': { 'old_revision': 'e4eb2a57c8667ab31903237e3c316fcaf4afe718', 'new_revision': '52523d4e58d99cdb768791bf9ac532c917522460', 'dep_path': 'src', 'repo_url': 'https://chromium.googlesource.com/chromium/src' }, 'security_flag': False, 'testcase_id': 1, 'sanitizer': 'ASAN', 'crash_type': 'fake type' }, 'crash_revision': 'e4eb2a57c8667ab31903237e3c316fcaf4afe718', 'platform': 'linux', 'client_id': 'clusterfuzz', 'signature': 'crashy_function()' } actual_message = blame_task._prepare_predator_message(testcase) actual_message = json.loads(actual_message.data) self.assertDictEqual(actual_message, expected_message)
def test_unreproducible(self): """Test unreproducible.""" testcase = test_utils.create_generic_testcase() testcase.one_time_crasher_flag = True testcase.crash_revision = 399171 testcase.regression = 'NA' testcase.put() expected_message = { 'stack_trace': 'crashy_function()', 'customized_data': { 'crash_address': '0xdeadbeef', 'job_type': 'test_content_shell_drt', 'dependencies': [{ 'dep_path': 'src/third_party/gperf', 'repo_url': 'https://chromium.googlesource.com/chromium/deps/gperf', 'revision': '97f2aa645b74c28c57eca56992235c79850fa9e0' }, { 'dep_path': 'src/third_party/pdfium', 'repo_url': 'https://pdfium.googlesource.com/pdfium', 'revision': '855665d4889853f8ac71519de8ff004dba8eb056' }, { 'dep_path': 'src/third_party/skia', 'repo_url': 'https://skia.googlesource.com/skia', 'revision': '4772bd537d153922cd772020e4ad4820090be51a' }, { 'dep_path': 'src/v8', 'repo_url': 'https://chromium.googlesource.com/v8/v8', 'revision': '3a590058de9b3640f73741b1e95f815f5c089988' }, { 'dep_path': 'src', 'repo_url': 'https://chromium.googlesource.com/chromium/src', 'revision': '52523d4e58d99cdb768791bf9ac532c917522460' }], 'dependency_rolls': [], 'regression_range': { 'old_revision': None, 'new_revision': None, 'dep_path': 'src', 'repo_url': 'https://chromium.googlesource.com/chromium/src' }, 'security_flag': False, 'testcase_id': 1, 'sanitizer': 'ASAN', 'crash_type': 'fake type' }, 'crash_revision': '52523d4e58d99cdb768791bf9ac532c917522460', 'platform': 'linux', 'client_id': 'clusterfuzz', 'signature': 'crashy_function()' } actual_message = blame_task._prepare_predator_message(testcase) actual_message = json.loads(actual_message.data) self.assertDictEqual(actual_message, expected_message)
def test_same_job(self): """Test variant task with same job.""" testcase = test_utils.create_generic_testcase() variant_testcase = variant_task._get_variant_testcase_for_job( # pylint: disable=protected-access testcase, testcase.job_type) self.assertEqual(testcase, variant_testcase)
def setUp(self): self.testcases = [ test_utils.create_generic_testcase(), test_utils.create_generic_testcase() ]
def test_find_testcase(self): """Ensure that find_testcase behaves as expected.""" crash_type = 'find_testcase_test_type' crash_state = 'find_testcase_test_state' security_flag = True nonsecurity = test_utils.create_generic_testcase() reproducible_with_bug = test_utils.create_generic_testcase() reproducible_with_bug.bug_information = '123456' reproducible_with_bug.one_time_crasher_flag = False reproducible_no_bug = test_utils.create_generic_testcase() reproducible_no_bug.bug_information = '' reproducible_no_bug.one_time_crasher_flag = False unreproducible_with_bug = test_utils.create_generic_testcase() unreproducible_with_bug.bug_information = '123456' unreproducible_with_bug.one_time_crasher_flag = True unreproducible_no_bug = test_utils.create_generic_testcase() unreproducible_no_bug.bug_information = '' unreproducible_no_bug.one_time_crasher_flag = True testcases = [ nonsecurity, reproducible_with_bug, reproducible_no_bug, unreproducible_with_bug, unreproducible_no_bug, ] # Apply generic information to each test case, and put. for testcase in testcases: testcase.crash_type = crash_type testcase.crash_state = crash_state testcase.security_flag = security_flag testcase.put() # We also want to test one non-security issue, so update it. nonsecurity.security_flag = False nonsecurity.put() # Ensure that we don't return anything if we search for an unused state or # type. self.assertIsNone( data_handler.find_testcase('project', crash_type, 'missing_state', security_flag)) self.assertIsNone( data_handler.find_testcase('project', 'missing type', crash_state, security_flag)) # Ensure that we respect security flag when searching. result = data_handler.find_testcase('project', crash_type, crash_state, False) self.assertTrue(test_utils.entities_equal(result, nonsecurity)) # Ensure that we properly prioritize issues. no_constraint_result = data_handler.find_testcase( 'project', crash_type, crash_state, security_flag) self.assertTrue( test_utils.entities_equal(no_constraint_result, reproducible_with_bug)) # Favor reproducibility over having a bug filed, and ensure that test cases # can be excluded from results. exclude_result = data_handler.find_testcase( 'project', crash_type, crash_state, security_flag, testcase_to_exclude=reproducible_with_bug) self.assertTrue( test_utils.entities_equal(exclude_result, reproducible_no_bug))