def test_similar_testcase_reproducible_and_closed_but_issue_open_2(self):
"""Tests result is true when there is a similar testcase which is
reproducible and fixed due to flakiness but issue is kept open. Don't update
testcase bug mapping if another reproducible testcase is open and attached
to this bug."""
self.issue.save()
similar_testcase_1 = test_utils.create_generic_testcase()
similar_testcase_1.one_time_crasher_flag = False
similar_testcase_1.open = False
similar_testcase_1.bug_information = str(self.issue.id)
similar_testcase_1.put()
similar_testcase_2 = test_utils.create_generic_testcase()
similar_testcase_2.one_time_crasher_flag = False
similar_testcase_2.open = True
similar_testcase_2.bug_information = str(self.issue.id)
similar_testcase_2.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker),
)
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(None, testcase.bug_information)
self.assertEqual("", self.issue._monorail_issue.comment)
def test_group_exceed_max_testcases(self):
"""Test that group auto-shrinks when it exceeds maximum number of
testcases."""
for i in range(1, 31):
testcase = test_utils.create_generic_testcase()
testcase.crash_type = 'Heap-buffer-overflow'
testcase.crash_state = 'abcdefgh' + str(i)
testcase.project_name = 'project'
testcase.one_time_crasher_flag = False
# Attach actual issues to some testcases.
if i in [3, 4, 5]:
testcase.bug_information = '123'
# Make some testcases unreproducible.
if i in [1, 2, 3]:
testcase.one_time_crasher_flag = True
testcase.put()
unrelated_testcase = test_utils.create_generic_testcase()
grouper.group_testcases()
testcase_ids = list(data_handler.get_open_testcase_id_iterator())
# [1, 2] get removed since they are unreproducible testcases.
# [3] is not removed since it has bug attached (even though unreproducible).
# [6, 7, 8] are removed to account for max group size. Even though they
# are reproducible, they are the ones with least weight.
expected_testcase_ids = [3, 4, 5] + list(range(
9, 31)) + [unrelated_testcase.key.id()]
self.assertEqual(expected_testcase_ids, testcase_ids)
def test_is_crash_important_4(self):
"""If the unreproducible testcase has another reproducible testcase in
group, then crash is not important."""
testcase_1 = test_utils.create_generic_testcase()
testcase_1.one_time_crasher_flag = True
testcase_1.group_id = 1
testcase_1.put()
testcase_2 = test_utils.create_generic_testcase()
testcase_2.one_time_crasher_flag = False
testcase_2.group_id = 1
testcase_2.put()
self.assertFalse(triage._is_crash_important(testcase_1))
def setUp(self):
helpers.patch(self, ["base.utils.utcnow"])
self.mock.utcnow.return_value = test_utils.CURRENT_TIME
self.testcase = test_utils.create_generic_testcase()
self.issue = appengine_test_utils.create_generic_issue()
self.issue_tracker = self.issue.issue_tracker
def test_similar_testcase_reproducible_and_closed_but_issue_open_1(self):
"""Tests result is true when there is a similar testcase which is
reproducible and fixed due to flakiness but issue is kept open. Only update
testcase bug mapping if similar testcase is fixed longer than the grace
period."""
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker),
)
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(None, testcase.bug_information)
self.assertEqual("", self.issue._monorail_issue.comment)
similar_testcase.set_metadata(
"closed_time",
test_utils.CURRENT_TIME - datetime.timedelta(
hours=data_types.MIN_ELAPSED_TIME_SINCE_FIXED + 1),
)
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker),
)
def test_similar_testcase_with_issue_recently_closed(self):
"""Tests result is true when there is a similar testcase with issue closed
recently."""
self.issue.status = "Fixed"
self.issue._monorail_issue.open = False
self.issue._monorail_issue.closed = (
test_utils.CURRENT_TIME - datetime.timedelta(
hours=data_types.MIN_ELAPSED_TIME_SINCE_FIXED - 1))
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker),
)
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(
"Delaying filing a bug since similar testcase (2) in issue (1) "
"was just fixed.",
testcase.get_metadata(triage.TRIAGE_MESSAGE_KEY),
)
def test_similar_testcase_with_issue_closed_with_ignore_label(self):
"""Tests result is true when there is a similar testcase with closed issue
blacklisted with ignore label."""
self.issue.status = "WontFix"
self.issue._monorail_issue.open = False
self.issue.labels.add("ClusterFuzz-Ignore")
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker),
)
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(
"Skipping filing a bug since similar testcase (2) in issue (1) "
"is blacklisted with ClusterFuzz-Ignore label.",
testcase.get_metadata(triage.TRIAGE_MESSAGE_KEY),
)
def test_is_crash_important_1(self):
"""Ensure that a reproducible testcase is important."""
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = False
testcase.put()
self.assertTrue(triage._is_crash_important(testcase))
def test_is_crash_important_9(self):
"""If this unreproducible testcase is crashing frequently, but its crash
type is one of crash type ignores, then it is not an important crash."""
self.mock.get_last_successful_hour.return_value = 417325
indices = [{
'count': 10,
'hour': day_index
} for day_index in range(417325, 416989, -24)]
self.mock.get.return_value = (1, [{
'totalCount':
140,
'groups': [
{
'indices': indices,
'name': 'false',
},
]
}])
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
for crash_type in ['Out-of-memory', 'Stack-overflow', 'Timeout']:
testcase.crash_type = crash_type
self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_9(self):
"""If this unreproducible testcase is crashing frequently, but its crash
type is one of crash type ignores, then it is not an important crash."""
self.mock.get_last_successful_hour.return_value = 417325
indices = [{
"count": 10,
"hour": day_index
} for day_index in range(417325, 416989, -24)]
self.mock.get.return_value = (
1,
[{
"totalCount": 140,
"groups": [{
"indices": indices,
"name": "false"
}]
}],
)
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
for crash_type in ["Out-of-memory", "Stack-overflow", "Timeout"]:
testcase.crash_type = crash_type
self.assertFalse(triage._is_crash_important(testcase))
def test_similar_testcase_reproducible_and_closed_but_issue_open_1(self):
"""Tests result is true when there is a similar testcase which is
reproducible and fixed due to flakiness but issue is kept open. Only update
testcase bug mapping if similar testcase is fixed longer than the grace
period."""
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker))
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual(None, testcase.bug_information)
self.assertEqual('', self.issue._monorail_issue.comment)
similar_testcase.set_metadata(
'closed_time', test_utils.CURRENT_TIME - datetime.timedelta(
hours=data_types.MIN_ELAPSED_TIME_SINCE_FIXED + 1))
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker))
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual('1', testcase.bug_information)
self.assertEqual(
'ClusterFuzz found another reproducible variant for this bug on '
'test_content_shell_drt job: '
'https://test-clusterfuzz.appspot.com/testcase?key=1.',
self.issue._monorail_issue.comment)
def test_similar_testcase_without_bug_information(self):
"""Tests result is false when there is a similar testcase but without an
associated bug."""
similar_testcase = test_utils.create_generic_testcase() # pylint: disable=unused-variable
self.assertEqual(
False,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker))
def test_blackbox_fuzzer_job(self):
"""Test variant task with blackbox fuzzer job."""
testcase = test_utils.create_generic_testcase()
testcase.job_type = 'linux_asan_project'
testcase.put()
variant_testcase = variant_task._get_variant_testcase_for_job( # pylint: disable=protected-access
testcase, 'linux_msan_project')
self.assertEqual(testcase, variant_testcase)
def test_is_crash_important_3(self):
"""Ensure that an unreproducible testcase with status Duplicate is
not important."""
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.status = "Duplicate"
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_5(self):
"""If we don't have any crash stats data for this unreproducible testcase,
then we can't make judgement on crash importance, so we return result as
False."""
self.mock.get_last_successful_hour.return_value = None
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def setUp(self):
helpers.patch_environ(self)
data_types.Job(
name='linux_asan_chrome',
environment_string=(
'APP_ARGS = --arg1 --arg2 --arg3="--flag1 --flag2"')).put()
data_types.Job(name='libfuzzer_asan_chrome',
environment_string=('')).put()
self.testcase = test_utils.create_generic_testcase()
def test_similar_testcase_get_issue_failed(self):
"""Tests result is false when there is a similar testcase with an associated
bug but we are unable to fetch it via get_issue."""
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.bug_information = '2' # Non-existent.
similar_testcase.put()
self.assertEqual(
False,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker))
def test_similar_testcase_is_reproducible_and_open(self):
"""Tests result is true when there is a similar testcase which is
reproducible, open and has an accessible associated bug."""
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = False
similar_testcase.open = True
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker))
def test_similar_testcase_unreproducible_but_issue_open(self):
"""Tests result is true when there is a similar testcase which is
unreproducible but issue is kept open. Update testcase bug mapping always
since this testcase is reproducible."""
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = True
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker))
def test_no_testcase_metadata(self):
"""Test when we only have default issue metadata and no testcase
metadata."""
self.mock.get_all_issue_metadata_for_testcase.return_value = None
testcase = test_utils.create_generic_testcase()
testcase.set_metadata('issue_owners', '[email protected],[email protected]')
testcase.set_metadata('issue_components', 'component1')
testcase.set_metadata('issue_labels', 'label1,label2,label3')
analyze_task._add_default_issue_metadata(testcase) # pylint: disable=protected-access
self.assertEqual('[email protected],[email protected]',
testcase.get_metadata('issue_owners'))
self.assertEqual('component1', testcase.get_metadata('issue_components'))
self.assertEqual('label1,label2,label3',
testcase.get_metadata('issue_labels'))
self.assertEqual(0, self.mock.log.call_count)
def test_no_default_issue_metadata(self):
"""Test when we only have testcase metadata and no default issue
metadata."""
self.mock.get_all_issue_metadata_for_testcase.return_value = {
"issue_owners": "[email protected],[email protected]",
"issue_components": "component1",
"issue_labels": "label1,label2,label3",
}
testcase = test_utils.create_generic_testcase()
analyze_task._add_default_issue_metadata(testcase) # pylint: disable=protected-access
self.assertEqual("[email protected],[email protected]",
testcase.get_metadata("issue_owners"))
self.assertEqual("component1",
testcase.get_metadata("issue_components"))
self.assertEqual("label1,label2,label3",
testcase.get_metadata("issue_labels"))
self.assertEqual(3, self.mock.log.call_count)
def test_union(self):
"""Test union of current testcase metadata and default issue metadata."""
self.mock.get_all_issue_metadata_for_testcase.return_value = {
'issue_owners': '[email protected], [email protected]',
'issue_components': 'component1',
'issue_labels': 'label1, label2 ,label3'
}
testcase = test_utils.create_generic_testcase()
testcase.set_metadata('issue_owners', '[email protected],[email protected]')
testcase.set_metadata('issue_components', 'component2')
testcase.set_metadata('issue_labels', 'label4,label5, label2,')
analyze_task._add_default_issue_metadata(testcase) # pylint: disable=protected-access
self.assertEqual('[email protected],[email protected],[email protected]',
testcase.get_metadata('issue_owners'))
self.assertEqual('component1,component2',
testcase.get_metadata('issue_components'))
self.assertEqual('label1,label2,label3,label4,label5',
testcase.get_metadata('issue_labels'))
self.assertEqual(3, self.mock.log.call_count)
def test_is_crash_important_7(self):
"""If this unreproducible testcase spiked only for a certain interval, then
it is not important."""
self.mock.get_last_successful_hour.return_value = 417325
self.mock.get.return_value = (1, [{
'totalCount':
125,
'groups': [
{
'indices': [{
'count': 125,
'hour': 417301,
}],
'name': 'false',
},
]
}])
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_7(self):
"""If this unreproducible testcase spiked only for a certain interval, then
it is not important."""
self.mock.get_last_successful_hour.return_value = 417325
self.mock.get.return_value = (
1,
[{
"totalCount":
125,
"groups": [{
"indices": [{
"count": 125,
"hour": 417301
}],
"name": "false"
}],
}],
)
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_similar_testcase_unreproducible_but_issue_open(self):
"""Tests result is true when there is a similar testcase which is
unreproducible but issue is kept open. Update testcase bug mapping always
since this testcase is reproducible."""
self.issue.save()
similar_testcase = test_utils.create_generic_testcase()
similar_testcase.one_time_crasher_flag = True
similar_testcase.open = False
similar_testcase.bug_information = str(self.issue.id)
similar_testcase.put()
self.assertEqual(
True,
triage._check_and_update_similar_bug(self.testcase,
self.issue_tracker))
testcase = data_handler.get_testcase_by_id(self.testcase.key.id())
self.assertEqual('1', testcase.bug_information)
self.assertEqual(
'ClusterFuzz found another reproducible variant for this bug on '
'test_content_shell_drt job: '
'https://test-clusterfuzz.appspot.com/testcase?key=1.',
self.issue._monorail_issue.comment)
def test_reproducible_invalid_regression_range(self):
"""Test reproducible with invalid regression range."""
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = False
testcase.regression = 'BAD'
testcase.put()
expected_result = {
'result': {
'error_message': 'Invalid regression range BAD.',
'feedback_url': '',
'found': False,
'project': '',
'suspected_components': '',
'suspected_cls': '',
}
}
self.assertIsNone(blame_task._prepare_predator_message(testcase))
testcase = data_handler.get_testcase_by_id(testcase.key.id())
actual_result = testcase.get_metadata('predator_result')
self.assertEqual(actual_result, expected_result)
def test_custom_binary(self):
"""Test custom binary."""
environment.set_value('CUSTOM_BINARY', True)
testcase = test_utils.create_generic_testcase()
expected_result = {
'result': {
'error_message': 'Not applicable to custom binaries.',
'feedback_url': '',
'found': False,
'project': '',
'suspected_components': '',
'suspected_cls': '',
}
}
self.assertIsNone(blame_task._prepare_predator_message(testcase))
testcase = data_handler.get_testcase_by_id(testcase.key.id())
actual_result = testcase.get_metadata('predator_result')
self.assertEqual(actual_result, expected_result)
environment.remove_key('CUSTOM_BINARY')
def test_is_crash_important_6(self):
"""If this unreproducible testcase is less than the total crash threshold,
then it is not important."""
self.mock.get_last_successful_hour.return_value = 417325
indices = [{
"count": 1,
"hour": day_index
} for day_index in range(417325, 416989, -24)]
self.mock.get.return_value = (
1,
[{
"totalCount": 14,
"groups": [{
"indices": indices,
"name": "false"
}]
}],
)
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_6(self):
"""If this unreproducible testcase is less than the total crash threshold,
then it is not important."""
self.mock.get_last_successful_hour.return_value = 417325
indices = [{
'count': 1,
'hour': day_index
} for day_index in range(417325, 416989, -24)]
self.mock.get.return_value = (1, [{
'totalCount':
14,
'groups': [
{
'indices': indices,
'name': 'false',
},
]
}])
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertFalse(triage._is_crash_important(testcase))
def test_is_crash_important_8(self):
"""If this unreproducible testcase is crashing frequently, then it is an
important crash."""
self.mock.get_last_successful_hour.return_value = 417325
indices = [{
'count': 10,
'hour': day_index
} for day_index in range(417325, 416989, -24)]
self.mock.get.return_value = (1, [{
'totalCount':
140,
'groups': [
{
'indices': indices,
'name': 'false',
},
]
}])
testcase = test_utils.create_generic_testcase()
testcase.one_time_crasher_flag = True
testcase.put()
self.assertTrue(triage.is_crash_important(testcase))