def _UpdateSwarmingTaskEntity(master_name,
builder_name,
build_number,
step_name,
status=None,
task_id=None,
error=None,
classified_test_results=None,
parameters=None,
canonical_step_name=None,
created_ts=None,
started_ts=None,
completed_ts=None):
task = WfSwarmingTask.Get(master_name, builder_name, build_number,
step_name)
assert task
task.status = status or task.status
task.task_id = task_id or task.task_id
task.error = error.ToSerializable() if error else task.error
task.classified_test_results = task.GetClassifiedTestResults(
classified_test_results or {}) or task.classified_test_results
task.parameters = task.parameters or {}
task.parameters.update(parameters or {})
task.canonical_step_name = canonical_step_name or task.canonical_step_name
task.created_time = task.created_time or time_util.DatetimeFromString(
created_ts)
task.started_time = task.started_time or time_util.DatetimeFromString(
started_ts)
task.completed_time = task.completed_time or time_util.DatetimeFromString(
completed_ts)
task.put()
def testDatetimeFromString(self):
self.assertEqual(None, time_util.DatetimeFromString('None'))
self.assertEqual(None, time_util.DatetimeFromString(None))
iso_time_str = '2016-01-02T01:02:03.123456'
iso_time_datetime = time_util.DatetimeFromString(iso_time_str)
# Check that our function reverses datetime.isoformat
self.assertEqual(iso_time_datetime.isoformat(), iso_time_str)
self.assertEqual(iso_time_datetime, time_util.DatetimeFromString(
iso_time_datetime))
with self.assertRaises(ValueError):
time_util.DatetimeFromString('Yesterday, at 5 o\'clock')
def _ParseFlakeSwarmingTaskOutput(task_data, output_json, error, parameters):
"""Returns swarming task results as a FlakeswarmingTaskOutput object.
Assumption: only 1 test was run in the Swarming task.
"""
assert task_data
iterations = parameters.iterations
if output_json:
# Gets the total numbers of runs and number of successful runs from
# test results
tries, successes = flake_test_results.GetCountsFromSwarmingRerun(
output_json)
if tries is None or successes is None:
# Something went wrong preventing even a single test from being processed
# which counts as an error.
error = error or SwarmingTaskError.GenerateError(
code=swarming_task_error.UNKNOWN)
tries = None
successes = None
elif (tries == 1 and task_data['state'] == constants.STATE_COMPLETED
and not task_data.get('failure')
and not task_data.get('infra_failure')):
# webkit_layout_tests special case: test results will be combined into
# one if all results are the same.
# Use iterations instead assuming the test repeated that many times.
# Currently only do this if task completes successfully.
tries = iterations
successes = iterations * successes
return FlakeSwarmingTaskOutput(
completed_time=time_util.DatetimeFromString(
task_data.get('completed_ts')),
error=error,
iterations=tries,
pass_count=successes,
started_time=time_util.DatetimeFromString(
task_data.get('started_ts')),
task_id=task_data['task_id'])
else:
return FlakeSwarmingTaskOutput(
completed_time=time_util.DatetimeFromString(
task_data.get('completed_ts')),
error=error or
SwarmingTaskError.GenerateError(code=swarming_task_error.UNKNOWN),
iterations=None,
pass_count=None,
started_time=time_util.DatetimeFromString(
task_data.get('started_ts')),
task_id=task_data['task_id'])
def HandleGet(self):
start_time = time_util.DatetimeFromString(
self.request.get('start_time', None))
end_time = time_util.DatetimeFromString(
self.request.get('end_time', None))
closure_counts = {}
for tree_name in _MONITORED_TREES:
start_time = start_time or _GetCurrentCheckingPointForTree(
tree_name)
all_statuses = _RetrieveTreeStatus(tree_name,
start_time,
end_time=end_time)
closure_counts[tree_name] = _DetectTreeClosureForTree(
tree_name, all_statuses)
return {'data': closure_counts}
def commit_attempt_func(cl, message):
matches = _COMMIT_ATTEMPT_REGEX.match(message['text'])
if not matches:
return
timestamp = time_util.DatetimeFromString(message['date'])
committer = message['sender']
patchset_id = str(message['patchset'])
cl.AddCqAttempt(patchset_id, committer, timestamp)
def _CreateTreeStatus(tree_status_entry):
"""Creates a TreeStatus model based on the given json data."""
return TreeStatus(
time=time_util.DatetimeFromString(tree_status_entry['date']),
message=tree_status_entry['message'],
state=tree_status_entry['general_state'],
username=tree_status_entry['username'],
)
def patchset_reverted_to_issue_func(cl, message):
matches = _PATCHSET_REVERTED_TO_ISSUE_REGEX.match(message['text'])
if not matches:
return
patchset_id = str(message['patchset'])
issue_url = matches.group('issueurl')
url_parts = issue_url.split('/')
# Support both https://host/1234 and https://host/1234/
change_id = url_parts[-1] or url_parts[-2]
reverter = message['sender']
timestamp = time_util.DatetimeFromString(message['date'])
revert_cl = self.GetClDetails(change_id)
revert = cl_info.Revert(patchset_id, revert_cl, reverter,
timestamp)
cl.reverts.append(revert)
def patchset_to_revision_func(cl, message):
matches_cq = _PATCHSET_TO_REVISION_REGEX.match(message['text'])
matches_manual = _PATCHSET_TO_REVISION_MANUAL_REGEX.match(
message['text'])
matches_any = matches_cq or matches_manual
if not matches_any:
return
patchset_id = str(message.get('patchset'))
revision = matches_any.group('revision')
timestamp = time_util.DatetimeFromString(message['date'])
commit = cl_info.Commit(patchset_id, revision, [], timestamp)
cl.commits.append(commit)
if matches_manual:
committer = message['sender']
# When a patch is manually landed, there is no cq attempt, but since
# we care about when action was taken, we take the timing of the commit
# itself as the commit attempt timestamp.
cl.AddCqAttempt(patchset_id, committer, timestamp)
def _ParseClInfo(self, data, cl):
def patchset_to_revision_func(cl, message):
matches_cq = _PATCHSET_TO_REVISION_REGEX.match(message['text'])
matches_manual = _PATCHSET_TO_REVISION_MANUAL_REGEX.match(
message['text'])
matches_any = matches_cq or matches_manual
if not matches_any:
return
patchset_id = str(message.get('patchset'))
revision = matches_any.group('revision')
timestamp = time_util.DatetimeFromString(message['date'])
commit = cl_info.Commit(patchset_id, revision, [], timestamp)
cl.commits.append(commit)
if matches_manual:
committer = message['sender']
# When a patch is manually landed, there is no cq attempt, but since
# we care about when action was taken, we take the timing of the commit
# itself as the commit attempt timestamp.
cl.AddCqAttempt(patchset_id, committer, timestamp)
def patchset_reverted_to_issue_func(cl, message):
matches = _PATCHSET_REVERTED_TO_ISSUE_REGEX.match(message['text'])
if not matches:
return
patchset_id = str(message['patchset'])
issue_url = matches.group('issueurl')
url_parts = issue_url.split('/')
# Support both https://host/1234 and https://host/1234/
change_id = url_parts[-1] or url_parts[-2]
reverter = message['sender']
timestamp = time_util.DatetimeFromString(message['date'])
revert_cl = self.GetClDetails(change_id)
revert = cl_info.Revert(patchset_id, revert_cl, reverter,
timestamp)
cl.reverts.append(revert)
def commit_attempt_func(cl, message):
matches = _COMMIT_ATTEMPT_REGEX.match(message['text'])
if not matches:
return
timestamp = time_util.DatetimeFromString(message['date'])
committer = message['sender']
patchset_id = str(message['patchset'])
cl.AddCqAttempt(patchset_id, committer, timestamp)
details_funcs = [
patchset_to_revision_func,
patchset_reverted_to_issue_func,
commit_attempt_func,
]
# Sort by timestamp
messages = sorted(
data['messages'],
key=lambda x: time_util.DatetimeFromString(x.get('date')))
for message in messages:
for f in details_funcs:
f(cl, message)
cl.closed = data['closed']
cl.cc = data['cc']
cl.reviewers = data['reviewers']
cl.auto_revert_off = codereview.IsAutoRevertOff(data['description'])
cl.owner_email = data['owner_email']
return cl
def _ParseClInfo(self, change_info, change_id):
if not change_info: # pragma: no cover
return None
result = cl_info.ClInfo(self._server_hostname, change_id)
result.reviewers = [
x['email']
for x in change_info.get('reviewers', {}).get('REVIEWER', [])
]
result.cc = [
x['email'] for x in change_info.get('reviewers', {}).get('CC', [])
]
result.closed = change_info['status'] == 'MERGED'
result.owner_email = change_info['owner'].get('email')
result.subject = change_info['subject']
result.revert_of = change_info.get('revert_of')
# If the status is merged, look at the commit details for the current
# commit.
if result.closed: # pragma: no branch
current_revision = change_info['current_revision']
revision_info = change_info['revisions'][current_revision]
patchset_id = revision_info['_number']
commit_timestamp = time_util.DatetimeFromString(change_info['submitted'])
revision_commit = revision_info['commit']
parent_revisions = [c['commit'] for c in revision_commit['parents']
] if revision_commit else []
result.commits.append(
cl_info.Commit(patchset_id, current_revision, parent_revisions,
commit_timestamp))
# Detect manual commits.
committer = revision_commit['committer']['email']
if committer not in self.commit_bot_emails:
result.AddCqAttempt(patchset_id, committer, commit_timestamp)
result.description = revision_commit['message']
# Checks for if the culprit owner has turned off auto revert.
result.auto_revert_off = codereview.IsAutoRevertOff(result.description)
# Saves information for each patch set.
for revision, revision_info in change_info['revisions'].iteritems():
patchset_id = revision_info['_number']
commit_info = revision_info.get('commit') or {}
parent_revisions = [c['commit'] for c in commit_info['parents']
] if commit_info else []
result.patchsets[revision] = cl_info.PatchSet(patchset_id, revision,
parent_revisions)
# TO FIND COMMIT ATTEMPTS:
# In messages look for "Patch Set 1: Commit-Queue+2"
# or "Patch Set 4: Code-Review+1 Commit-Queue+2".
cq_pattern = re.compile('^Patch Set \d+:( Code-Review..)? Commit-Queue\+2$')
revert_tag = 'autogenerated:gerrit:revert'
revert_pattern = re.compile(
'Created a revert of this change as (?P<change_id>I[a-f\d]{40})')
for message in change_info.get('messages', []):
if cq_pattern.match(message['message'].splitlines()[0]):
patchset_id = message['_revision_number']
author = message['author']['email']
timestamp = time_util.DatetimeFromString(message['date'])
result.AddCqAttempt(patchset_id, author, timestamp)
# TO FIND REVERT(S):
if message.get('tag') == revert_tag:
patchset_id = message['_revision_number']
author = message['author']['email']
timestamp = time_util.DatetimeFromString(message['date'])
reverting_change_id = revert_pattern.match(
message['message']).group('change_id')
reverting_cl = self.GetClDetails(reverting_change_id)
result.reverts.append(
cl_info.Revert(patchset_id, reverting_cl, author, timestamp))
return result