def __init__(self,
step_names,
minimum_count=5,
combine_results=True,
**kwargs):
"""Initializes the PerfCountNotifier on tests starting with test_name.
Args:
step_names: List of perf steps names. This is needed to know perf steps
from other steps especially when the step is successful.
minimum_count: The number of minimum consecutive (REGRESS|IMPROVE) needed
to notify.
combine_results: Combine summary results email for all builders in one.
"""
# Set defaults.
ChromiumNotifier.__init__(self, **kwargs)
self.minimum_count = minimum_count
self.combine_results = combine_results
self.step_names = step_names
self.recent_results = None
self.error_email = False
self.new_email_results = {}
self.recent_results = FailuresHistory(expiration_time=_EXPIRATION_TIME,
size_limit=1000)
def test_trivial(self): # Put some failures and make sure they're there. # No failures are expected to expire or dropped due to size limit. h = FailuresHistory(expiration_time=3600, size_limit=1024) self.assertEqual(h.GetCount(42), 0) h.Put(42) self.assertEqual(h.GetCount(42), 1) h.Put(42) self.assertEqual(h.GetCount(42), 2) h.Put(13) self.assertEqual(h.GetCount(13), 1) self.assertEqual(h.GetCount(42), 2) self.assertEqual(h.GetCount(77), 0)
def __init__(self, step_names, minimum_count=5, **kwargs):
"""Initializes the PerfCountNotifier on tests starting with test_name.
Args:
step_names: List of perf steps names. This is needed to know perf steps
from other steps especially when the step is successful.
minimum_count: The number of minimum consecutive (REGRESS|IMPROVE) needed
to notify.
"""
# Set defaults.
ChromiumNotifier.__init__(self, **kwargs)
self.minimum_count = minimum_count
self.step_names = step_names
self.recent_results = None
self._InitRecentResults()
self.notifications = FailuresHistory(expiration_time=_EXPIRATION_TIME,
size_limit=1000)
def test_stress_maxsize(self):
TEST_SIZE = 137
h = FailuresHistory(expiration_time=3600, size_limit=TEST_SIZE)
# Many different failures
for f in xrange(10000):
h.Put(f)
# Make sure we always know about the newly added failure.
self.assertTrue(h.GetCount(f) == 1)
# Many failures with the same ID -> should drop most of the other failures.
for _ in xrange(TEST_SIZE * 2):
h.Put(42)
self.assertTrue(h.GetCount(42) > 0)
for _ in xrange(10000):
h.Put(42)
# Make sure there's always enough of repeating failures.
self.assertTrue(h.GetCount(42) >= TEST_SIZE/2)
def test_timer(self):
# Test that the expiration_time works as expected.
# The size of the history is large enough to avoid dropping.
current_time = 0
time.time = lambda: current_time
h = FailuresHistory(expiration_time=5, size_limit=1024)
for _ in xrange(100):
h.Put(42)
current_time += 1
self.assertEqual(h.GetCount(42), 4)
for _ in xrange(16):
h.Put(42)
self.assertEqual(h.GetCount(42), 20)
current_time += 10
self.assertEqual(h.GetCount(42), 0)
class PerfCountNotifier(ChromiumNotifier):
"""This is a status notifier that only alerts on consecutive perf changes.
The notifier only notifies when a number of consecutive REGRESS or IMPROVE
perf results are recorded.
See builder.interfaces.IStatusReceiver for more information about
parameters type.
"""
def __init__(self,
step_names,
minimum_count=5,
combine_results=True,
**kwargs):
"""Initializes the PerfCountNotifier on tests starting with test_name.
Args:
step_names: List of perf steps names. This is needed to know perf steps
from other steps especially when the step is successful.
minimum_count: The number of minimum consecutive (REGRESS|IMPROVE) needed
to notify.
combine_results: Combine summary results email for all builders in one.
"""
# Set defaults.
ChromiumNotifier.__init__(self, **kwargs)
self.minimum_count = minimum_count
self.combine_results = combine_results
self.step_names = step_names
self.recent_results = None
self.error_email = False
self.new_email_results = {}
self.recent_results = FailuresHistory(expiration_time=_EXPIRATION_TIME,
size_limit=1000)
def AddNewEmailResult(self, result):
"""Stores an email result for a builder.
Args:
result: A tuple of the form ('REGRESS|IMPROVE', 'value_name', 'builder').
"""
builder_name = result[2]
build_results = self.GetEmailResults(builder_name)
if not result[1] in build_results[result[0]]:
build_results[result[0]].append(result[1])
else:
PerfLog('(%s) email result has already been stored.' %
', '.join(result))
def GetEmailResults(self, builder_name):
"""Returns the email results for a builder."""
if not builder_name in self.new_email_results:
self.new_email_results[builder_name] = GetNewBuilderResult()
return self.new_email_results[builder_name]
def _UpdateResults(self, builder_name, results):
"""Updates the results by adding/removing from the history.
Args:
builder_name: Builder name the results belong to.
results: List of result tuples, each tuple is of the form
('REGRESS|IMPROVE', 'value_name', 'builder').
"""
new_results_ids = [' '.join(result) for result in results]
# Delete the old results if the new results do not have them.
to_delete = [
old_id for old_id in self.recent_results.failures
if (old_id not in new_results_ids and old_id.endswith(builder_name)
)
]
for old_id in to_delete:
self._DeleteResult(old_id)
# Update the new results history
for new_id in results:
self._StoreResult(new_id)
def _StoreResult(self, result):
"""Stores the result value and removes counter results.
Example: if this is a REGRESS result then it is stored and its counter
IMPROVE result, if any, is reset.
Args:
result: A tuple of the form ('REGRESS|IMPROVE', 'value_name', 'builder').
"""
self.recent_results.Put(' '.join(result))
if result[0] == REGRESS:
counter_id = IMPROVE + ' '.join(result[1:])
else:
counter_id = REGRESS + ' '.join(result[1:])
# Reset counter_id count since this breaks the consecutive count of it.
self._DeleteResult(counter_id)
def _DeleteResult(self, result_id):
"""Removes the history of results identified by result_id.
Args:
result_id: The id of the history entry (see _StoreResult() for details).
"""
num_results = self.recent_results.GetCount(result_id)
if num_results > 0:
# This is a hack into FailuresHistory since it does not allow to delete
# entries in its history unless they are expired.
# FailuresHistory.failures_count is the total number of entries in the
# history limitted by FailuresHistory.size_limit.
del self.recent_results.failures[result_id]
self.recent_results.failures_count -= num_results
def _DeleteAllForBuild(self, builder_name):
"""Deletes all test results related to a builder."""
to_delete = [
result for result in self.recent_results.failures
if result.endswith(builder_name)
]
for result in to_delete:
self._DeleteResult(result)
def _ResetResults(self, builder_name):
"""Reset pending email results for builder."""
builders = [builder_name]
if self.combine_results:
builders = self.new_email_results.keys()
for builder_name in builders:
self._DeleteAllForBuild(builder_name)
self.new_email_results[builder_name] = GetNewBuilderResult()
self.new_email_results[builder_name][EMAIL_TIME] = time.time()
def _IsPerfStep(self, step_status):
"""Checks if the step name is one of the defined perf tests names."""
return self.getName(step_status) in self.step_names
def isInterestingStep(self, build_status, step_status, results):
"""Ignore the step if it is not one of the perf results steps.
Returns:
True: - if a REGRESS|IMPROVE happens consecutive minimum number of times.
- if it is not a SUCCESS step and neither REGRESS|IMPROVE.
False: - if it is a SUCCESS step.
- if it is a notification which has already been notified.
"""
self.error_email = False
step_text = ' '.join(step_status.getText())
PerfLog('Analyzing failure text: %s.' % step_text)
if (not self._IsPerfStep(step_status)
or not self.isInterestingBuilder(build_status.getBuilder())):
return False
# In case of exceptions, sometimes results output is empty.
if not results:
results = [FAILURE]
builder_name = build_status.getBuilder().getName()
self.SetBuilderGraphURL(self.getName(step_status), build_status)
# If it is a success step, i.e. not interesting, then reset counters.
if results[0] == SUCCESS:
self._DeleteAllForBuild(builder_name)
return False
# step_text is similar to:
# media_tests_av_perf <div class="BuildResultInfo"> PERF_REGRESS:
# time/t (89.07%) PERF_IMPROVE: fps/video (5.40%) </div>
#
# regex would return tuples of the form:
# ('REGRESS', 'time/t', 'linux-rel')
# ('IMPROVE', 'fps/video', 'win-debug')
#
# It is important to put the builder name as the last element in the tuple
# since it is used to check tests that belong to same builder.
step_text = ' '.join(step_status.getText())
PerfLog('Analyzing failure text: %s.' % step_text)
perf_regress = perf_improve = ''
perf_results = []
if PERF_REGRESS in step_text:
perf_regress = step_text[step_text.find(PERF_REGRESS) +
len(PERF_REGRESS) +
1:step_text.find(PERF_IMPROVE)]
perf_results.extend([
(REGRESS, test_name, builder_name)
for test_name in re.findall(r'(\S+) (?=\(.+\))', perf_regress)
])
if PERF_IMPROVE in step_text:
# Based on log_parser/process_log.py PerformanceChangesAsText() function,
# we assume that PERF_REGRESS (if any) appears before PERF_IMPROVE.
perf_improve = step_text[step_text.find(PERF_IMPROVE) +
len(PERF_IMPROVE) + 1:]
perf_results.extend([
(IMPROVE, test_name, builder_name)
for test_name in re.findall(r'(\S+) (?=\(.+\))', perf_improve)
])
# If there is no regress or improve then this could be warning or exception.
if not perf_results:
if not self.recent_results.GetCount(step_text):
PerfLog(
'Unrecognized step status. Reporting status as interesting.'
)
# Force the build box to show in email
self.error_email = True
self.recent_results.Put(step_text)
return True
else:
PerfLog('This problem has already been notified.')
return False
update_list = []
for result in perf_results:
if len(result) != 3:
# We expect a tuple similar to ('REGRESS', 'time/t', 'linux-rel')
continue
result_id = ' '.join(result)
update_list.append(result)
PerfLog('Result: %s happened %d times in a row.' %
(result_id, self.recent_results.GetCount(result_id) + 1))
if self.recent_results.GetCount(
result_id) >= self.minimum_count - 1:
# This is an interesting result! We got the minimum consecutive count of
# this result. Store it in email results.
PerfLog(
'Result: %s happened enough consecutive times to be reported.'
% result_id)
self.AddNewEmailResult(result)
self._UpdateResults(builder_name, update_list)
# Final decision is made based on whether there are any notifications to
# email based on this and older build results.
return self.ShouldSendEmail(builder_name)
def buildMessage(self, builder_name, build_status, results, step_name):
"""Send an email about this interesting step.
Add the perf regressions/improvements that resulted in this email if any.
"""
PerfLog('About to send an email.')
email_subject = self.GetEmailSubject(builder_name, build_status,
results, step_name)
email_body = self.GetEmailBody(builder_name, build_status, results,
step_name)
html_content = (
'<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">'
'<html xmlns="http://www.w3.org/1999/xhtml"><body>%s</body></html>'
% email_body)
defered_object = self.BuildEmailObject(email_subject, html_content,
builder_name, build_status,
step_name)
self._ResetResults(builder_name)
return defered_object
def ShouldSendEmail(self, builder_name):
"""Returns if we should send a summary email at this moment.
Returns:
True if it has been at least minimum_delay_between_alert since the
last email sent. False otherwise.
"""
builders = [builder_name]
if self.combine_results:
builders = self.new_email_results.keys()
for builder_name in builders:
if self._ShouldSendEmail(builder_name):
return True
return False
def _ShouldSendEmail(self, builder_name):
results = self.GetEmailResults(builder_name)
last_time_mail_sent = results[EMAIL_TIME]
if (last_time_mail_sent and last_time_mail_sent >
time.time() - self.minimum_delay_between_alert):
# Rate limit tree alerts.
PerfLog(
'Time since last email is too short. Should not send email.')
return False
# Return True if there are any builder results to email about.
return results and (results[REGRESS] or results[IMPROVE])
def GetEmailSubject(self, builder_name, build_status, results, step_name):
"""Returns the subject of for an email based on perf results."""
project_name = self.master_status.getTitle()
latest_revision = build_utils.getLatestRevision(build_status)
result = 'changes'
builders = [builder_name]
if self.combine_results:
builders = self.new_email_results.keys()
return (
'%s %s on %s, revision %s' %
(project_name, result, ', '.join(builders), str(latest_revision)))
def GetEmailHeader(self, builder_name, build_status, results, step_name):
"""Returns a header message in an email.
Used for backward compatibility with chromium_notifier. It allows the
users to add text to every email from the master.cfg setup.
"""
status_text = self.status_header % {
'builder': builder_name,
'steps': step_name,
'results': results
}
return status_text
def GetEmailBody(self, builder_name, build_status, results, step_name):
"""Returns the main email body content."""
email_body = ''
builders = [builder_name]
if self.combine_results:
builders = self.new_email_results.keys()
for builder_name in builders:
email_body += '%s%s\n' % (self.GetEmailHeader(
builder_name, build_status, results,
step_name), self.GetPerfEmailBody(builder_name))
# Latest build box is not relevant with multiple builder results combined.
if not self.combine_results or self.error_email:
email_body += (
'\n\nLatest build results:%s' %
self.GenStepBox(builder_name, build_status, step_name))
PerfLog('Perf email body: %s' % email_body)
return email_body.replace('\n', '<br>')
def GetPerfEmailBody(self, builder_name):
builder_results = self.GetEmailResults(builder_name)
graph_url = builder_results[GRAPH_URL]
msg = ''
# Add regression HTML links.
if builder_results[REGRESS]:
test_urls = CreateHTMLTestURLList(graph_url,
builder_results[REGRESS])
msg += '<strong>%s</strong>: %s.\n' % (PERF_REGRESS,
', '.join(test_urls))
# Add improvement HTML links.
if builder_results[IMPROVE]:
test_urls = CreateHTMLTestURLList(graph_url,
builder_results[IMPROVE])
msg += '<strong>%s</strong>: %s.\n' % (PERF_IMPROVE,
', '.join(test_urls))
return msg or 'No perf results.\n'
def BuildEmailObject(self, email_subject, html_content, builder_name,
build_status, step_name):
"""Creates an email object ready to be sent."""
m = MIMEMultipart('alternative')
m.attach(MIMEText(html_content, 'html', 'iso-8859-1'))
m['Date'] = formatdate(localtime=True)
m['Subject'] = email_subject
m['From'] = self.fromaddr
if self.reply_to:
m['Reply-To'] = self.reply_to
recipients = list(self.extraRecipients[:])
dl = []
if self.sendToInterestedUsers and self.lookup:
for u in build_status.getInterestedUsers():
d = defer.maybeDeferred(self.lookup.getAddress, u)
d.addCallback(recipients.append)
dl.append(d)
defered_object = defer.DeferredList(dl)
defered_object.addCallback(self._gotRecipients, recipients, m)
defered_object.addCallback(self.getFinishedMessage, builder_name,
build_status, step_name)
return defered_object
def GenStepBox(self, builder_name, build_status, step_name):
"""Generates a HTML styled summary box for one step."""
waterfall_url = self.master_status.getBuildbotURL()
styles = dict(build_utils.DEFAULT_STYLES)
builder_results = self.GetEmailResults(builder_name)
if builder_results[IMPROVE] and not builder_results[REGRESS]:
styles['warnings'] = styles['success']
return build_utils.EmailableBuildTable_bb8(build_status,
waterfall_url,
styles=styles,
step_names=[step_name])
def SetBuilderGraphURL(self, step_name, build_status):
"""Stores the graph URL used in emails for this builder."""
builder_name = build_status.getBuilder().getName()
builder_results = self.GetEmailResults(builder_name)
graph_url = GetGraphURL(step_name, build_status)
latest_revision = build_utils.getLatestRevision(build_status)
if latest_revision:
graph_url = SetQueryParameter(graph_url, 'rev', latest_revision)
builder_results[GRAPH_URL] = graph_url
def test_maxsize(self): # Test that size_limit works as expected. # No failures should expire. h = FailuresHistory(expiration_time=3600, size_limit=4) self.assertEqual(h.GetCount(42), 0) h.Put(42) self.assertEqual(h.GetCount(42), 1) h.Put(42) self.assertEqual(h.GetCount(42), 2) h.Put(42) self.assertEqual(h.GetCount(42), 3) h.Put(42) self.assertEqual(h.GetCount(42), 4) h.Put(13) # 4+1 = 5 > size_limit -> drop one old '42' failure. self.assertEqual(h.GetCount(13), 1) self.assertEqual(h.GetCount(42), 3)
def _InitRecentResults(self):
"""Initializes a new failures history object to store results."""
self.recent_results = FailuresHistory(expiration_time=_EXPIRATION_TIME,
size_limit=1000)
class PerfCountNotifier(ChromiumNotifier):
"""This is a status notifier that only alerts on consecutive perf changes.
The notifier only notifies when a number of consecutive REGRESS or IMPROVE
perf results are recorded.
See builder.interfaces.IStatusReceiver for more information about
parameters type.
"""
def __init__(self, step_names, minimum_count=5, **kwargs):
"""Initializes the PerfCountNotifier on tests starting with test_name.
Args:
step_names: List of perf steps names. This is needed to know perf steps
from other steps especially when the step is successful.
minimum_count: The number of minimum consecutive (REGRESS|IMPROVE) needed
to notify.
"""
# Set defaults.
ChromiumNotifier.__init__(self, **kwargs)
self.minimum_count = minimum_count
self.step_names = step_names
self.recent_results = None
self._InitRecentResults()
self.notifications = FailuresHistory(expiration_time=_EXPIRATION_TIME,
size_limit=1000)
def _InitRecentResults(self):
"""Initializes a new failures history object to store results."""
self.recent_results = FailuresHistory(expiration_time=_EXPIRATION_TIME,
size_limit=1000)
def _UpdateResults(self, results):
"""Updates the results by adding/removing from the history.
Args:
results: List of result tuples, each tuple is of the form
('REGRESS|IMPROVE', 'value_name').
"""
new_results_ids = [' '.join(result) for result in results]
# Delete the old results if the new results do not have them.
to_delete = [
old_id for old_id in self.recent_results.failures
if old_id not in new_results_ids
]
for old_id in to_delete:
self._DeleteResult(old_id)
# Update the new results history
for new_id in results:
self._StoreResult(new_id)
def _StoreResult(self, result):
"""Stores the result value and removes counter results.
Example: if this is a REGRESS result then it is stored and its counter
IMPROVE result, if any, is reset.
Args:
result: A tuple of the form ('REGRESS|IMPROVE', 'value_name').
"""
self.recent_results.Put(' '.join(result))
if result[0] == 'REGRESS':
counter_id = 'IMPROVE ' + result[1]
else:
counter_id = 'REGRESS ' + result[1]
# Reset counter_id count since this breaks the consecutive count of it.
self._DeleteResult(counter_id)
def _DeleteResult(self, result_id):
"""Removes the history of results identified by result_id.
Args:
result_id: The id of the history entry (see _StoreResult() for details).
"""
num_results = self.recent_results.GetCount(result_id)
if num_results > 0:
# This is a hack into FailuresHistory since it does not allow to delete
# entries in its history unless they are expired.
# FailuresHistory.failures_count is the total number of entries in the
# history limitted by FailuresHistory.size_limit.
del self.recent_results.failures[result_id]
self.recent_results.failures_count -= num_results
def _IsPerfStep(self, step_status):
"""Checks if the step name is one of the defined perf tests names."""
return self.getName(step_status) in self.step_names
def isInterestingStep(self, build_status, step_status, results):
"""Ignore the step if it is not one of the perf results steps.
Returns:
True: - if a REGRESS|IMPROVE happens consecutive minimum number of times.
- if it is not a SUCCESS step and neither REGRESS|IMPROVE.
False: - if it is a SUCCESS step.
- if it is a notification which has already been notified.
"""
if not self._IsPerfStep(step_status):
return False
# In case of exceptions, sometimes results output is empty.
if not results:
results = [FAILURE]
# If it is a success step, i.e. not interesting, then reset counters.
if results[0] == SUCCESS:
self._InitRecentResults()
return False
# step_text is similar to:
# media_tests_av_perf <div class="BuildResultInfo"> PERF_REGRESS:
# time/t (89.07%) PERF_IMPROVE: fps/video (5.40%) </div>
#
# regex would return tuples of the form:
# ('REGRESS', 'time/t')
# ('IMPROVE', 'fps/video')
step_text = ' '.join(step_status.getText())
log.msg('[PerfCountNotifier] Analyzing failure text: %s.' % step_text)
perf_results = re.findall('PERF_(REGRESS|IMPROVE): (\S+)', step_text)
# If there is no regress or improve then this could be warning or exception.
if not perf_results:
if not self.notifications.GetCount(step_text):
log.msg(
'[PerfCountNotifier] Unrecognized step status encountered. '
'Reporting status as interesting.')
self.notifications.Put(step_text)
return True
else:
log.msg(
'[PerfCountNotifier] This problem has already been notified.'
)
return False
is_interesting = False
update_list = []
for result in perf_results:
if len(result) != 2:
# We expect a tuple similar to ('REGRESS', 'time/t')
continue
result_id = ' '.join(result)
update_list.append(result)
log.msg(
'[PerfCountNotifier] Result: %s happened %d times in a row.' %
(result_id, self.recent_results.GetCount(result_id) + 1))
if self.recent_results.GetCount(
result_id) >= self.minimum_count - 1:
# This is an interesting result! We got the minimum consecutive count of
# this result, however we still need to check if its been notified.
if not self.notifications.GetCount(result_id):
log.msg(
'[PerfCountNotifier] Result: %s happened enough consecutive '
'times to be reported.' % result_id)
self.notifications.Put(result_id)
is_interesting = True
else:
log.msg(
'[PerfCountNotifier] Result: %s has already been notified.'
% result_id)
self._UpdateResults(update_list)
return is_interesting