def test_group_exceed_max_testcases(self):
"""Test that group auto-shrinks when it exceeds maximum number of
testcases."""
for i in range(1, 31):
testcase = test_utils.create_generic_testcase()
testcase.crash_type = 'Heap-buffer-overflow'
testcase.crash_state = 'abcdefgh' + str(i)
testcase.project_name = 'project'
testcase.one_time_crasher_flag = False
# Attach actual issues to some testcases.
if i in [3, 4, 5]:
testcase.bug_information = '123'
# Make some testcases unreproducible.
if i in [1, 2, 3]:
testcase.one_time_crasher_flag = True
testcase.put()
unrelated_testcase = test_utils.create_generic_testcase()
grouper.group_testcases()
testcase_ids = list(data_handler.get_open_testcase_id_iterator())
# [1, 2] get removed since they are unreproducible testcases.
# [3] is not removed since it has bug attached (even though unreproducible).
# [6, 7, 8] are removed to account for max group size. Even though they
# are reproducible, they are the ones with least weight.
expected_testcase_ids = [3, 4, 5] + list(range(
9, 31)) + [unrelated_testcase.key.id()]
self.assertEqual(expected_testcase_ids, testcase_ids)
def test_similar_testcase_reproducible_and_closed_but_issue_open_2(self):
"""Tests result is true when there is a similar testcase which is
reproducible and fixed due to flakiness but issue is kept open. Don't update
testcase bug mapping if another reproducible testcase is open and attached
to this bug."""
self.issue.save()
similar_testcase_1 = test_utils.create_generic_testcase()
similar_testcase_1.one_time_crasher_flag = False
similar_testcase_1.open = False
similar_testcase_1.bug_information = str(self.issue.id)
similar_testcase_1.put()
similar_testcase_2 = test_utils.create_generic_testcase()
similar_testcase_2.one_time_crasher_flag = False
similar_testcase_2.open = True
similar_testcase_2.bug_information = str(self.issue.id)
similar_testcase_2.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(None, testcase.bug_information)
self.assertEqual('', self.issue._monorail_issue.comment)
def test_is_crash_important_4(self):
"""If the unreproducible testcase has another reproducible testcase in
group, then crash is not important."""
testcase_1 = test_utils.create_generic_testcase()
testcase_1.one_time_crasher_flag = True
testcase_1.group_id = 1
testcase_1.put()
testcase_2 = test_utils.create_generic_testcase()
testcase_2.one_time_crasher_flag = False
testcase_2.group_id = 1
testcase_2.put()
self.assertFalse(triage._is_crash_important(testcase_1))
def test_similar_testcase_reproducible_and_closed_but_issue_open_1(self):
"""Tests result is true when there is a similar testcase which is
reproducible and fixed due to flakiness but issue is kept open. Only update
testcase bug mapping if similar testcase is fixed longer than the grace
period."""
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(None, testcase.bug_information)
self.assertEqual('', self.issue._monorail_issue.comment)
similar_testcase.set_metadata(
'closed_time',
test_utils.CURRENT_TIME -
datetime.timedelta(hours=data_types.MIN_ELAPSED_TIME_SINCE_FIXED + 1))
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def test_similar_testcase_with_issue_recently_closed(self):
"""Tests result is true when there is a similar testcase with issue closed
recently."""
self.issue.status = 'Fixed'
self.issue._monorail_issue.open = False
self.issue._monorail_issue.closed = (
test_utils.CURRENT_TIME - datetime.timedelta(
hours=data_types.MIN_ELAPSED_TIME_SINCE_FIXED - 1))
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker))
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(
'Delaying filing a bug since similar testcase (2) in issue (1) '
'was just fixed.',
testcase.get_metadata(triage.TRIAGE_MESSAGE_KEY))
def test_is_crash_important_9(self):
"""If this unreproducible testcase is crashing frequently, but its crash
type is one of crash type ignores, then it is not an important crash."""
self.mock.get_last_successful_hour.return_value = 417325
indices = [{
'count': 10,
'hour': day_index
} for day_index in range(417325, 416989, -24)]
self.mock.get.return_value = (1, [{
'totalCount':
140,
'groups': [
{
'indices': indices,
'name': 'false',
},
]
}])
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
for crash_type in ['Out-of-memory', 'Stack-overflow', 'Timeout']:
testcase.crash_type = crash_type
self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_1(self):
"""Ensure that a reproducible testcase is important."""
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = False
testcase.put()
self.assertTrue(triage._is_crash_important(testcase))
def setUp(self):
helpers.patch(self, [
'libs.issue_management.issue_filer.file_issue',
])
self.testcase = test_utils.create_generic_testcase()
self.issue = appengine_test_utils.create_generic_issue()
self.issue_tracker = self.issue.issue_tracker
def test_similar_testcase_without_bug_information(self):
"""Tests result is false when there is a similar testcase but without an
associated bug."""
similar_testcase = test_utils.create_generic_testcase() # pylint: disable=unused-variable
self.assertEqual(
False,
triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def setUp(self):
helpers.patch(self, [
'clusterfuzz._internal.base.utils.utcnow',
])
self.mock.utcnow.return_value = test_utils.CURRENT_TIME
self.testcase = test_utils.create_generic_testcase()
self.issue = appengine_test_utils.create_generic_issue()
self.issue_tracker = self.issue.issue_tracker
def test_blackbox_fuzzer_job(self):
"""Test variant task with blackbox fuzzer job."""
testcase = test_utils.create_generic_testcase()
testcase.job_type = 'linux_asan_project'
testcase.put()
variant_testcase = variant_task._get_variant_testcase_for_job( # pylint: disable=protected-access
testcase, 'linux_msan_project')
self.assertEqual(testcase, variant_testcase)
def test_is_crash_important_3(self):
"""Ensure that an unreproducible testcase with status Duplicate is
not important."""
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.status = 'Duplicate'
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_5(self):
"""If we don't have any crash stats data for this unreproducible testcase,
then we can't make judgement on crash importance, so we return result as
False."""
self.mock.get_last_successful_hour.return_value = None
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_similar_testcase_get_issue_failed(self):
"""Tests result is false when there is a similar testcase with an associated
bug but we are unable to fetch it via get_issue."""
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.bug_information = '2' # Non-existent.
similar_testcase.put()
self.assertEqual(
False,
triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def test_similar_testcase_is_reproducible_and_open(self):
"""Tests result is true when there is a similar testcase which is
reproducible, open and has an accessible associated bug."""
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = True
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def test_similar_testcase_unreproducible_but_issue_open(self):
"""Tests result is true when there is a similar testcase which is
unreproducible but issue is kept open. Update testcase bug mapping always
since this testcase is reproducible."""
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = True
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
def test_no_default_issue_metadata(self):
"""Test when we only have testcase metadata and no default issue
metadata."""
self.mock.get_all_issue_metadata_for_testcase.return_value = {
'issue_owners': '[email protected],[email protected]',
'issue_components': 'component1',
'issue_labels': 'label1,label2,label3'
}
testcase = test_utils.create_generic_testcase()
analyze_task._add_default_issue_metadata(testcase) # pylint: disable=protected-access
self.assertEqual('[email protected],[email protected]',
testcase.get_metadata('issue_owners'))
self.assertEqual('component1',
testcase.get_metadata('issue_components'))
self.assertEqual('label1,label2,label3',
testcase.get_metadata('issue_labels'))
self.assertEqual(3, self.mock.log.call_count)
def setUp(self):
helpers.patch_environ(self)
data_types.Job(
name='linux_asan_chrome',
environment_string=('APP_ARGS = --orig-arg1 --orig-arg2')).put()
data_types.Job(
name='linux_msan_chrome_variant',
environment_string=(
'APP_ARGS = --arg1 --arg2 --arg3="--flag1 --flag2"')).put()
data_types.Job(name='libfuzzer_asan_chrome',
environment_string=('')).put()
data_types.Job(name='libfuzzer_msan_chrome_variant',
environment_string=('')).put()
data_types.Job(name='afl_asan_chrome_variant',
environment_string=('')).put()
self.testcase = test_utils.create_generic_testcase()
def test_is_crash_important_7(self):
"""If this unreproducible testcase spiked only for a certain interval, then
it is not important."""
self.mock.get_last_successful_hour.return_value = 417325
self.mock.get.return_value = (1, [{
'totalCount':
125,
'groups': [{
'indices': [{
'count': 125,
'hour': 417301,
}],
'name': 'false',
},]
}])
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_6(self):
"""If this unreproducible testcase is less than the total crash threshold,
then it is not important."""
self.mock.get_last_successful_hour.return_value = 417325
indices = [{
'count': 1,
'hour': day_index
} for day_index in range(417325, 416989, -24)]
self.mock.get.return_value = (1, [{
'totalCount': 14,
'groups': [{
'indices': indices,
'name': 'false',
},]
}])
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_custom_binary(self):
"""Test custom binary."""
environment.set_value('CUSTOM_BINARY', True)
testcase = test_utils.create_generic_testcase()
expected_result = {
'result': {
'error_message': 'Not applicable to custom binaries.',
'feedback_url': '',
'found': False,
'project': '',
'suspected_components': '',
'suspected_cls': '',
}
}
self.assertIsNone(blame_task._prepare_predator_message(testcase))
testcase = data_handler.get_testcase_by_id(testcase.key.id())
actual_result = testcase.get_metadata('predator_result')
self.assertEqual(actual_result, expected_result)
environment.remove_key('CUSTOM_BINARY')
def test_similar_testcase_with_issue_closed_with_ignore_label(self):
"""Tests result is true when there is a similar testcase with closed issue
blacklisted with ignore label."""
self.issue.status = 'WontFix'
self.issue._monorail_issue.open = False
self.issue.labels.add('ClusterFuzz-Ignore')
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase, self.issue_tracker))
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(
'Skipping filing a bug since similar testcase (2) in issue (1) '
'is blacklisted with ClusterFuzz-Ignore label.',
testcase.get_metadata(triage.TRIAGE_MESSAGE_KEY))
def test_union(self):
"""Test union of current testcase metadata and default issue metadata."""
self.mock.get_all_issue_metadata_for_testcase.return_value = {
'issue_owners': '[email protected], [email protected]',
'issue_components': 'component1',
'issue_labels': 'label1, label2 ,label3'
}
testcase = test_utils.create_generic_testcase()
testcase.set_metadata('issue_owners',
'[email protected],[email protected]')
testcase.set_metadata('issue_components', 'component2')
testcase.set_metadata('issue_labels', 'label4,label5, label2,')
analyze_task._add_default_issue_metadata(testcase) # pylint: disable=protected-access
self.assertEqual(
'[email protected],[email protected],[email protected]',
testcase.get_metadata('issue_owners'))
self.assertEqual('component1,component2',
testcase.get_metadata('issue_components'))
self.assertEqual('label1,label2,label3,label4,label5',
testcase.get_metadata('issue_labels'))
self.assertEqual(3, self.mock.log.call_count)
def test_reproducible_invalid_regression_range(self):
"""Test reproducible with invalid regression range."""
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = False
testcase.regression = 'BAD'
testcase.put()
expected_result = {
'result': {
'error_message': 'Invalid regression range BAD.',
'feedback_url': '',
'found': False,
'project': '',
'suspected_components': '',
'suspected_cls': '',
}
}
self.assertIsNone(blame_task._prepare_predator_message(testcase))
testcase = data_handler.get_testcase_by_id(testcase.key.id())
actual_result = testcase.get_metadata('predator_result')
self.assertEqual(actual_result, expected_result)
def test_reproducible_regression_range(self):
"""Test reproducible with regression range."""
testcase = test_utils.create_generic_testcase()
testcase.crash_revision = 398287
testcase.regression = '398287:399171'
testcase.put()
expected_message = {
'stack_trace': 'crashy_function()',
'customized_data': {
'crash_address':
'0xdeadbeef',
'job_type':
'test_content_shell_drt',
'dependencies': [{
'dep_path':
'src/third_party/bidichecker',
'repo_url':
'https://chromium.googlesource.com/'
'external/bidichecker/lib',
'revision':
'97f2aa645b74c28c57eca56992235c79850fa9e0'
}, {
'dep_path':
'src/third_party/pdfium',
'repo_url':
'https://pdfium.googlesource.com/pdfium',
'revision':
'f7e108b2d0c2f67a143e99693df084bfff7037ec'
}, {
'dep_path':
'src/third_party/skia',
'repo_url':
'https://skia.googlesource.com/skia',
'revision':
'ee295645bd91fcbe1714847c5fe5341759037cc5'
}, {
'dep_path':
'src/v8',
'repo_url':
'https://chromium.googlesource.com/v8/v8',
'revision':
'cba1fdd4d72e7c5b874f9eeb07901792f26c871a'
}, {
'dep_path':
'src',
'repo_url':
'https://chromium.googlesource.com/chromium/src',
'revision':
'e4eb2a57c8667ab31903237e3c316fcaf4afe718'
}],
'dependency_rolls': [{
'dep_path':
'src/third_party/gperf',
'repo_url':
'https://chromium.googlesource.com/chromium/deps/gperf',
'new_revision':
'97f2aa645b74c28c57eca56992235c79850fa9e0'
}, {
'dep_path':
'src/third_party/pdfium',
'repo_url':
'https://pdfium.googlesource.com/pdfium',
'new_revision':
'855665d4889853f8ac71519de8ff004dba8eb056',
'old_revision':
'f7e108b2d0c2f67a143e99693df084bfff7037ec'
}, {
'dep_path':
'src/third_party/skia',
'repo_url':
'https://skia.googlesource.com/skia',
'new_revision':
'4772bd537d153922cd772020e4ad4820090be51a',
'old_revision':
'ee295645bd91fcbe1714847c5fe5341759037cc5'
}, {
'dep_path':
'src/v8',
'repo_url':
'https://chromium.googlesource.com/v8/v8',
'new_revision':
'3a590058de9b3640f73741b1e95f815f5c089988',
'old_revision':
'cba1fdd4d72e7c5b874f9eeb07901792f26c871a'
}, {
'dep_path':
'src',
'repo_url':
'https://chromium.googlesource.com/chromium/src',
'new_revision':
'52523d4e58d99cdb768791bf9ac532c917522460',
'old_revision':
'e4eb2a57c8667ab31903237e3c316fcaf4afe718'
}],
'regression_range': {
'old_revision': 'e4eb2a57c8667ab31903237e3c316fcaf4afe718',
'new_revision': '52523d4e58d99cdb768791bf9ac532c917522460',
'dep_path': 'src',
'repo_url':
'https://chromium.googlesource.com/chromium/src'
},
'security_flag':
False,
'testcase_id':
1,
'sanitizer':
'ASAN',
'crash_type':
'fake type'
},
'crash_revision': 'e4eb2a57c8667ab31903237e3c316fcaf4afe718',
'platform': 'linux',
'client_id': 'clusterfuzz',
'signature': 'crashy_function()'
}
actual_message = blame_task._prepare_predator_message(testcase)
actual_message = json.loads(actual_message.data)
self.assertDictEqual(actual_message, expected_message)
def test_unreproducible(self):
"""Test unreproducible."""
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.crash_revision = 399171
testcase.regression = 'NA'
testcase.put()
expected_message = {
'stack_trace': 'crashy_function()',
'customized_data': {
'crash_address':
'0xdeadbeef',
'job_type':
'test_content_shell_drt',
'dependencies': [{
'dep_path':
'src/third_party/gperf',
'repo_url':
'https://chromium.googlesource.com/chromium/deps/gperf',
'revision':
'97f2aa645b74c28c57eca56992235c79850fa9e0'
}, {
'dep_path':
'src/third_party/pdfium',
'repo_url':
'https://pdfium.googlesource.com/pdfium',
'revision':
'855665d4889853f8ac71519de8ff004dba8eb056'
}, {
'dep_path':
'src/third_party/skia',
'repo_url':
'https://skia.googlesource.com/skia',
'revision':
'4772bd537d153922cd772020e4ad4820090be51a'
}, {
'dep_path':
'src/v8',
'repo_url':
'https://chromium.googlesource.com/v8/v8',
'revision':
'3a590058de9b3640f73741b1e95f815f5c089988'
}, {
'dep_path':
'src',
'repo_url':
'https://chromium.googlesource.com/chromium/src',
'revision':
'52523d4e58d99cdb768791bf9ac532c917522460'
}],
'dependency_rolls': [],
'regression_range': {
'old_revision': None,
'new_revision': None,
'dep_path': 'src',
'repo_url':
'https://chromium.googlesource.com/chromium/src'
},
'security_flag':
False,
'testcase_id':
1,
'sanitizer':
'ASAN',
'crash_type':
'fake type'
},
'crash_revision': '52523d4e58d99cdb768791bf9ac532c917522460',
'platform': 'linux',
'client_id': 'clusterfuzz',
'signature': 'crashy_function()'
}
actual_message = blame_task._prepare_predator_message(testcase)
actual_message = json.loads(actual_message.data)
self.assertDictEqual(actual_message, expected_message)
def test_same_job(self):
"""Test variant task with same job."""
testcase = test_utils.create_generic_testcase()
variant_testcase = variant_task._get_variant_testcase_for_job( # pylint: disable=protected-access
testcase, testcase.job_type)
self.assertEqual(testcase, variant_testcase)
def setUp(self):
self.testcases = [
test_utils.create_generic_testcase(),
test_utils.create_generic_testcase()
]
def test_find_testcase(self):
"""Ensure that find_testcase behaves as expected."""
crash_type = 'find_testcase_test_type'
crash_state = 'find_testcase_test_state'
security_flag = True
nonsecurity = test_utils.create_generic_testcase()
reproducible_with_bug = test_utils.create_generic_testcase()
reproducible_with_bug.bug_information = '123456'
reproducible_with_bug.one_time_crasher_flag = False
reproducible_no_bug = test_utils.create_generic_testcase()
reproducible_no_bug.bug_information = ''
reproducible_no_bug.one_time_crasher_flag = False
unreproducible_with_bug = test_utils.create_generic_testcase()
unreproducible_with_bug.bug_information = '123456'
unreproducible_with_bug.one_time_crasher_flag = True
unreproducible_no_bug = test_utils.create_generic_testcase()
unreproducible_no_bug.bug_information = ''
unreproducible_no_bug.one_time_crasher_flag = True
testcases = [
nonsecurity,
reproducible_with_bug,
reproducible_no_bug,
unreproducible_with_bug,
unreproducible_no_bug,
]
# Apply generic information to each test case, and put.
for testcase in testcases:
testcase.crash_type = crash_type
testcase.crash_state = crash_state
testcase.security_flag = security_flag
testcase.put()
# We also want to test one non-security issue, so update it.
nonsecurity.security_flag = False
nonsecurity.put()
# Ensure that we don't return anything if we search for an unused state or
# type.
self.assertIsNone(
data_handler.find_testcase('project', crash_type, 'missing_state',
security_flag))
self.assertIsNone(
data_handler.find_testcase('project', 'missing type', crash_state,
security_flag))
# Ensure that we respect security flag when searching.
result = data_handler.find_testcase('project', crash_type, crash_state,
False)
self.assertTrue(test_utils.entities_equal(result, nonsecurity))
# Ensure that we properly prioritize issues.
no_constraint_result = data_handler.find_testcase(
'project', crash_type, crash_state, security_flag)
self.assertTrue(
test_utils.entities_equal(no_constraint_result, reproducible_with_bug))
# Favor reproducibility over having a bug filed, and ensure that test cases
# can be excluded from results.
exclude_result = data_handler.find_testcase(
'project',
crash_type,
crash_state,
security_flag,
testcase_to_exclude=reproducible_with_bug)
self.assertTrue(
test_utils.entities_equal(exclude_result, reproducible_no_bug))
