def testNextCommitPositionNewlyAddedFlakyTest(self, mocked_fn):
master_name = 'm'
builder_name = 'b'
build_number = 100
step_name = 's'
test_name = 't'
git_hash = 'r100'
try_job_id = '123'
revision = 'r100'
commit_position = 100
url = 'url'
change_log = ChangeLog(None, None, revision, commit_position, None,
None, url, None)
mocked_fn.return_value = change_log
try_job = FlakeTryJob.Create(master_name, builder_name, step_name,
test_name, revision)
try_job.try_job_ids.append(try_job_id)
try_job.put()
try_job_data = FlakeTryJobData.Create(try_job_id)
try_job_data.try_job_key = try_job.key
try_job_data.put()
analysis = MasterFlakeAnalysis.Create(master_name, builder_name,
build_number, step_name,
test_name)
analysis.status = analysis_status.COMPLETED
analysis.try_job_status = analysis_status.RUNNING
analysis.data_points = [
_GenerateDataPoint(pass_rate=0.9,
commit_position=commit_position,
build_number=12345,
previous_build_commit_position=98,
blame_list=['r99', 'r100']),
_GenerateDataPoint(pass_rate=-1,
commit_position=99,
try_job_url='id1')
]
analysis.suspected_flake_build_number = 12345
analysis.algorithm_parameters = DEFAULT_CONFIG_DATA[
'check_flake_settings']
analysis.Save()
self.MockPipeline(
recursive_flake_try_job_pipeline.RecursiveFlakeTryJobPipeline,
'',
expected_args=[])
pipeline = NextCommitPositionPipeline(analysis.key.urlsafe(),
try_job.key.urlsafe(), 98,
_DEFAULT_CACHE_NAME, None)
pipeline.start(queue_name=constants.DEFAULT_QUEUE)
self.execute_queued_tasks()
culprit = analysis.culprit
self.assertEqual(git_hash, culprit.revision)
self.assertEqual(100, culprit.commit_position)
def testCreateCulprit(self, mocked_module):
revision = 'a1b2c3d4'
commit_position = 12345
url = 'url'
repo_name = 'repo_name'
change_log = ChangeLog(None, None, revision, commit_position, None,
None, url, None)
mocked_module.return_value = change_log
culprit = CreateCulprit(revision, commit_position, 0.6, repo_name)
self.assertEqual(commit_position, culprit.commit_position)
self.assertEqual(revision, culprit.revision)
self.assertEqual(url, culprit.url)
self.assertEqual(repo_name, culprit.repo_name)
def testUpdateFlakeAnalysisDataPointsPipeline(self, mocked_change_log):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.Save()
commit_position = 1000
pass_rate = 0.5
revision = 'r1000'
expected_time = datetime(2018, 9, 18, 0, 0, 0)
committer = Contributor(name='name', email='email', time=expected_time)
change_log = ChangeLog(None, committer, revision, None, None, None,
None, None)
mocked_change_log.return_value = change_log
flakiness = Flakiness(build_number=123,
build_url='url',
commit_position=commit_position,
total_test_run_seconds=100,
error=None,
failed_swarming_task_attempts=0,
iterations=50,
pass_rate=pass_rate,
revision=revision,
try_job_url=None,
task_ids=ListOfBasestring.FromSerializable(
['task_id']))
expected_data_point = DataPoint.Create(build_number=123,
build_url='url',
commit_position=commit_position,
elapsed_seconds=100,
error=None,
failed_swarming_task_attempts=0,
iterations=50,
pass_rate=pass_rate,
git_hash=revision,
try_job_url=None,
task_ids=['task_id'],
commit_timestamp=expected_time)
update_data_points_input = UpdateFlakeAnalysisDataPointsInput(
analysis_urlsafe_key=analysis.key.urlsafe(), flakiness=flakiness)
pipeline_job = UpdateFlakeAnalysisDataPointsPipeline(
update_data_points_input)
pipeline_job.start()
self.execute_queued_tasks()
self.assertEqual(1, len(analysis.data_points))
self.assertEqual(expected_data_point, analysis.data_points[0])
def testPullChangelogs(self, mock_fn):
change_log_rev1_dict = {
'author': {
'name':
'*****@*****.**',
'email':
'*****@*****.**',
'time':
datetime.strptime('Wed Jun 11 19:35:32 2014',
'%a %b %d %H:%M:%S %Y'),
},
'committer': {
'name':
'*****@*****.**',
'email':
'*****@*****.**',
'time':
datetime.strptime('Wed Jun 11 19:35:32 2014',
'%a %b %d %H:%M:%S %Y'),
},
'message':
'Cr-Commit-Position: refs/heads/master@{#175976}',
'commit_position':
175976,
'touched_files': [{
'new_path': 'added_file.js',
'change_type': 'add',
'old_path': '/dev/null'
}],
'commit_url':
'https://chromium.googlesource.com/chromium/src.git/+log/rev0',
'code_review_url':
None,
'revision':
'rev1',
'reverted_revision':
None,
'review_server_host':
None,
'review_change_id':
None,
}
change_log_rev1 = ChangeLog.FromDict(change_log_rev1_dict)
mock_fn.return_value = [change_log_rev1]
expected_change_logs = {'rev1': change_log_rev1}
change_logs = git.PullChangeLogs('rev0', 'rev1')
self.assertEqual(expected_change_logs, change_logs)
def testFindCulpritSucceeds(self, mock_find_culprit):
crash_identifiers = self.GetDummyChromeCrashData()['crash_identifiers']
analysis = FracasCrashAnalysis.Create(crash_identifiers)
analysis.status = analysis_status.RUNNING
analysis.put()
dummy_cl = ChangeLog(
Contributor('AUTHOR_NAME', 'AUTHOR_EMAIL', 'AUTHOR_TIME'),
Contributor('COMITTER_NAME', 'COMITTER_EMAIL', 'COMITTER_TIME'),
'REVISION',
'COMMIT_POSITION',
'MESSAGE',
'TOUCHED_FILES',
'COMMIT_URL',
)
dummy_project_path = 'PROJECT_PATH'
dummy_suspect = Suspect(dummy_cl, dummy_project_path)
dummy_culprit = Culprit(
project='PROJECT',
components=['COMPONENT_1', 'CPOMPONENT_2'],
suspected_cls=[dummy_suspect],
# N.B., we must use a list here for the assertion to work
# TODO(wrengr): fix that.
regression_range=['VERSION_0', 'VERSION_1'],
algorithm='ALGORITHM',
success=True,
)
mock_find_culprit.return_value = dummy_culprit
pipeline = crash_pipeline.CrashAnalysisPipeline(
CrashClient.FRACAS, crash_identifiers)
pipeline.start()
self.execute_queued_tasks()
analysis = FracasCrashAnalysis.Get(crash_identifiers)
self.assertEqual(analysis_status.COMPLETED, analysis.status)
self.assertTrue(analysis.result['found'])
self.assertTrue(analysis.found_suspects)
self.assertTrue(analysis.found_project)
self.assertTrue(analysis.found_components)
dummy_suspect, dummy_tags = dummy_culprit.ToDicts()
self.assertDictEqual(analysis.result, dummy_suspect)
def testChangeLog(self):
change_log_dict = {
'author': {
'name': 'a',
'email': '*****@*****.**',
'time': '2014-08-13 00:53:12',
},
'committer': {
'name': 'c',
'email': '*****@*****.**',
'time': '2014-08-14 00:53:12',
},
'revision':
'aaaa',
'commit_position':
1111,
'touched_files': [{
'change_type': 'copy',
'old_path': 'old_file',
'new_path': 'new_file'
}, {
'change_type': 'modify',
'old_path': 'file',
'new_path': 'file'
}],
'message':
'blabla...',
'commit_url':
'https://chromium.googlesource.com/chromium/src/+/git_hash',
'code_review_url':
'https://codereview.chromium.org/2222',
'reverted_revision':
'8d4a4fa6s18raf3re12tg6r',
'review_server_host':
'codereview.chromium.org',
'review_change_id':
'2222',
}
change_log = ChangeLog.FromDict(change_log_dict)
self.assertEqual(change_log_dict, change_log.ToDict())
def _ParseChangeLogFromLogData(self, data):
change_info = commit_util.ExtractChangeInfo(data['message'], self._ref)
touched_files = []
for file_diff in data['tree_diff']:
change_type = file_diff['type'].lower()
if not diff.IsKnownChangeType(change_type):
raise Exception('Unknown change type "%s"' % change_type)
touched_files.append(
FileChangeInfo(change_type, file_diff['old_path'],
file_diff['new_path']))
reverted_revision = commit_util.GetRevertedRevision(data['message'])
url = '%s/+/%s' % (self.repo_url, data['commit'])
return ChangeLog(self._ContributorFromDict(data['author']),
self._ContributorFromDict(data['committer']),
data['commit'], change_info.get('commit_position'),
data['message'], touched_files, url,
change_info.get('code_review_url'), reverted_revision,
change_info.get('host'), change_info.get('change_id'))
def ChangeLogFromDict(data):
touched_files = [
FileChangeInfo.FromDict(touched_file)
for touched_file in data.get('touched_files', [])
]
author = Contributor(
data.get('author', {}).get('name', 'author'),
data.get('author', {}).get('email', 'email'),
data.get('author', {}).get('time', 'time'))
committer = Contributor(
data.get('committer', {}).get('name', 'committer'),
data.get('committer', {}).get('email', 'email'),
data.get('committer', {}).get('time', 'time'))
return ChangeLog(author, committer, data.get('revision'),
data.get('commit_position'), data.get('message'),
touched_files, data.get('commit_url'),
data.get('code_review_url'),
data.get('reverted_revision'),
data.get('review_server_host'),
data.get('review_change_id'))
def testUpdateCulpritNewCulprit(self, mocked_fn):
revision = 'a1b2c3d4'
commit_position = 12345
url = 'url'
repo_name = 'repo_name'
change_log = ChangeLog(None, None, revision, commit_position, None,
None, url, None)
mocked_fn.return_value = change_log
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
culprit = flake_analysis_util.UpdateCulprit(analysis.key.urlsafe(),
revision, commit_position,
repo_name)
self.assertIsNotNone(culprit)
self.assertEqual([analysis.key.urlsafe()],
culprit.flake_analysis_urlsafe_keys)
self.assertEqual(url, culprit.url)
self.assertEqual(repo_name, culprit.repo_name)
self.assertEqual(revision, culprit.revision)
DUMMY_CHANGELOG1 = ChangeLog.FromDict({
'author': {
'name': '*****@*****.**',
'email': '*****@*****.**',
'time': 'Thu Mar 31 21:24:43 2016',
},
'committer': {
'email': '*****@*****.**',
'time': 'Thu Mar 31 21:28:39 2016',
'name': '*****@*****.**',
},
'message':
'dummy',
'commit_position':
175900,
'touched_files': [
{
'change_type': 'add',
'new_path': 'a.cc',
'old_path': None,
},
],
'commit_url':
'https://repo.test/+/1',
'code_review_url':
'https://codereview.chromium.org/3281',
'revision':
'1',
'reverted_revision':
None
})
_CHANGELOG = ChangeLog.FromDict({
'author': {
'name': '*****@*****.**',
'email': '*****@*****.**',
'time': 'Thu Mar 31 21:24:43 2016',
},
'committer': {
'name': '*****@*****.**',
'email': '*****@*****.**',
'time': 'Thu Mar 31 21:28:39 2016',
},
'message':
'dummy',
'commit_position':
175900,
'touched_files': [
{
'change_type': 'modify',
'new_path': 'src/a.cc',
'old_path': 'src/a.cc',
},
],
'commit_url':
'https://repo.test/+/1',
'code_review_url':
'https://codereview.chromium.org/3281',
'revision':
'1',
'reverted_revision':
None
})
def __call__(self, output, repo_url): # pylint:disable=W
"""Parses output of 'git log --pretty=format:<format>.
For example:
Git changelog output is:
commit 21a8979218c096f4a96b07b67c9531f5f09e28a3
tree 7d9a79c9b060c9a030abe20a8429d2b81ca1d4db
parents 9640406d426a2d153b16e1d9ae7f9105268b36c9
author Test
author-email [email protected]
author-time 2016-10-24 22:21:45
committer Test
committer-email [email protected]
committer-time 2016-10-24 22:25:45
--Message start--
Commit messages...
--Message end--
:100644 100644 25f95f c766f1 M src/a/delta/git_parsers.py
Returns:
Parsed ChangeLog object.
"""
if not output:
return None
is_message_line = False
info = {
'author': {},
'committer': {},
'message': '',
'touched_files': []
}
for line in output.splitlines():
if MESSAGE_START_PATTERN.match(line):
is_message_line = True
continue
if MESSAGE_END_PATTERN.match(line):
is_message_line = False
# Remove the added '\n' at the end.
info['message'] = info['message'][:-1]
continue
if is_message_line:
info['message'] += line + '\n'
elif COMMIT_HASH_PATTERN.match(line):
info['revision'] = COMMIT_HASH_PATTERN.match(line).group(1)
elif AUTHOR_NAME_PATTERN.match(line):
info['author']['name'] = AUTHOR_NAME_PATTERN.match(line).group(
1)
elif AUTHOR_MAIL_PATTERN.match(line):
info['author']['email'] = commit_util.NormalizeEmail(
AUTHOR_MAIL_PATTERN.match(line).group(1))
elif AUTHOR_TIME_PATTERN.match(line):
info['author']['time'] = datetime.strptime(
AUTHOR_TIME_PATTERN.match(line).group(1), DATETIME_FORMAT)
elif COMMITTER_NAME_PATTERN.match(line):
info['committer']['name'] = (
COMMITTER_NAME_PATTERN.match(line).group(1))
elif COMMITTER_MAIL_PATTERN.match(line):
info['committer']['email'] = commit_util.NormalizeEmail(
COMMITTER_MAIL_PATTERN.match(line).group(1))
elif COMMITTER_TIME_PATTERN.match(line):
info['committer']['time'] = datetime.strptime(
COMMITTER_TIME_PATTERN.match(line).group(1),
DATETIME_FORMAT)
elif (CHANGED_FILE_PATTERN1.match(line)
or CHANGED_FILE_PATTERN2.match(line)):
match = (CHANGED_FILE_PATTERN1.match(line)
or CHANGED_FILE_PATTERN2.match(line))
# For modify, add, delete, the pattern is like:
# :100644 100644 df565d 6593e M modules/audio_coding/BUILD.gn
# For rename, copy, the pattern is like:
# :100644 100644 3f2e 20a5 R078 path1 path2
info['touched_files'].append(
GetFileChangeInfo(
GetChangeType(match.group(5)), match.group(6),
None if len(match.groups()) < 7 else match.group(7)))
# If commit is not parsed, the changelog will be {'author': {}, 'committer':
# {}, 'message': ''}, return None instead.
if not 'revision' in info:
return None
change_info = commit_util.ExtractChangeInfo(info['message'])
info['commit_position'] = change_info.get('commit_position')
info['code_review_url'] = change_info.get('code_review_url')
info['reverted_revision'] = commit_util.GetRevertedRevision(
info['message'])
info['commit_url'] = '%s/+/%s' % (repo_url, info['revision'])
return ChangeLog.FromDict(info)
'old_path': '/dev/null'
}],
'commit_url':
'https://chromium.googlesource.com/chromium/src.git/+log/rev4',
'code_review_url':
None,
'revision':
'rev4',
'reverted_revision':
None,
'review_server_host':
None,
'review_change_id':
None,
}
mock_change_log.return_value = ChangeLog.FromDict(change_log_dict)
self.assertTrue(git.ChangeCommittedWithinTime('rev4'))
@mock.patch.object(time_util,
'GetUTCNow',
return_value=datetime(2018, 06, 26, 11, 0, 0))
@mock.patch.object(CachedGitilesRepository, 'GetChangeLog')
def testChangeCommittedWithinTimeTooLate(self, mock_change_log, _):
change_log_dict = {
'author': {
'name': '*****@*****.**',
'email': '*****@*****.**',
'time': datetime(2018, 06, 25, 10, 0, 0),
},
'committer': {
'name': '*****@*****.**',