def mailfailure(jobname, job, message):
message_lines = [""]
message_lines.append("The following tests FAILED for this job")
message_lines.append("http://%s/results/%s" %
(socket.gethostname(), jobname))
message_lines.append("")
message_lines.append(format_failure_message("Job name", "Kernel",
"Test name", "FAIL/WARN",
"Failure reason"))
message_lines.append(format_failure_message("=" * 8, "=" * 6, "=" * 8,
"=" * 8, "=" * 14))
message_header = "\n".join(message_lines)
subject = "AUTOTEST: FAILED tests from job %s" % jobname
mail.send("", job.user, "", subject, message_header + message)
def mailfailure(jobname, job, message):
message_lines = [""]
message_lines.append("The following tests FAILED for this job")
message_lines.append("http://%s/results/%s" %
(socket.gethostname(), jobname))
message_lines.append("")
message_lines.append(format_failure_message("Job name", "Kernel",
"Test name", "FAIL/WARN",
"Failure reason"))
message_lines.append(format_failure_message("=" * 8, "=" * 6, "=" * 8,
"=" * 8, "=" * 14))
message_header = "\n".join(message_lines)
subject = "AUTOTEST: FAILED tests from job %s" % jobname
mail.send("", job.user, "", subject, message_header + message)
def email_about_test_failure(failed_tests, all_tests):
"""
Send an email about all the tests that have failed if there are any.
@param failed_tests: The list of failed tests. This will be sorted in this
function.
@param all_tests: All the names of tests that have been recently ran.
"""
if failed_tests:
failed_tests.sort()
mail.send(
_MAIL_RESULTS_FROM, [_MAIL_RESULTS_TO], [], 'Long Failing Tests',
'%d/%d tests have been failing for at least %d days.\n'
'They are the following:\n\n%s' %
(len(failed_tests), len(all_tests), _DAYS_TO_BE_FAILING_TOO_LONG,
'\n'.join(failed_tests)))
def test_email_count_of_total_number_of_tests(self):
"""Test that the email report displays total number of tests."""
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
mail.send(
'*****@*****.**',
['*****@*****.**'],
[],
'Long Failing Tests',
'1/2 tests have been failing for at least %d days.\n'
'They are the following:\n\ntest'
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
failures = ['test']
all_tests = set(failures) | {'not_failure'}
self.mox.ReplayAll()
complete_failures.email_about_test_failure(failures, all_tests)
def test_email_sent_about_all_failed_tests(self):
"""Test that the email report mentions all the failed_tests."""
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
mail.send(
'*****@*****.**',
['*****@*****.**'],
[],
'Long Failing Tests',
'1/1 tests have been failing for at least %d days.\n'
'They are the following:\n\ntest'
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
failures = ['test']
all_tests = set(failures)
self.mox.ReplayAll()
complete_failures.email_about_test_failure(failures, all_tests)
def test(self):
"""Does a basic test of as much of the system as possible."""
job = models.Job(job_idx=1)
kernel = models.Kernel(kernel_idx=1)
machine = models.Machine(machine_idx=1)
success_status = models.Status(status_idx=GOOD_STATUS_IDX)
fail_status = models.Status(status_idx=FAIL_STATUS_IDX)
old_passing_test = models.Test(job=job,
status=success_status,
kernel=kernel,
machine=machine,
test='test1',
started_time=self.datetime(2012, 1, 1))
old_passing_test.save()
failing_test = models.Test(job=job,
status=fail_status,
kernel=kernel,
machine=machine,
test='test2',
started_time=self.datetime(2012, 1, 1))
failing_test.save()
good_test = models.Test(job=job,
status=success_status,
kernel=kernel,
machine=machine,
test='test3',
started_time=self.datetime(2012, 1, 20))
good_test.save()
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 10
MockDatetime.today().AndReturn(self.datetime(2012, 1, 21))
MockDatetime.today().AndReturn(self.datetime(2012, 1, 21))
mail.send(
'*****@*****.**',
['*****@*****.**'], [],
'Long Failing Tests',
'2/3 tests have been failing for at least %d days.\n'
'They are the following:\n\ntest1\ntest2' %
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
self.mox.ReplayAll()
complete_failures.main()
def test_email_has_test_names_sorted_alphabetically(self):
"""Test that the email report has entries sorted alphabetically."""
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
mail.send(
'*****@*****.**',
['*****@*****.**'],
[],
'Long Failing Tests',
'2/2 tests have been failing for at least %d days.\n'
'They are the following:\n\ntest1\ntest2'
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
# We use an OrderedDict to gurantee that the elements would be out of
# order if we did a simple traversal.
failures = ['test2', 'test1']
all_tests = set(failures)
self.mox.ReplayAll()
complete_failures.email_about_test_failure(failures, all_tests)
def mailfailure(jobname, job, message):
"""Send an email about the failure.
@param jobname: String representing the job name.
@param job: A job object.
@param message: The message to mail.
"""
message_lines = [""]
message_lines.append("The following tests FAILED for this job")
message_lines.append("http://%s/results/%s" %
(socket.gethostname(), jobname))
message_lines.append("")
message_lines.append(
format_failure_message("Job name", "Kernel", "Test name", "FAIL/WARN",
"Failure reason"))
message_lines.append(
format_failure_message("=" * 8, "=" * 6, "=" * 8, "=" * 8, "=" * 14))
message_header = "\n".join(message_lines)
subject = "AUTOTEST: FAILED tests from job %s" % jobname
mail.send("", job.user, "", subject, message_header + message)
def flag_problem_test(machine):
"""
Notify people about the last job that ran on a machine.
This method is invoked everytime a machine fails to repair, and attempts
to identify the last test that ran on the machine. If successfull, it files
a bug, or sends out an email, or just logs the fact.
@param machine: The hostname (e.g. IP address) of the machine to find the
last job ran on it.
"""
rpc = frontend.AFE()
logger = MachineDeathLogger()
try:
problem_test = _find_problem_test(machine, rpc)
except (urllib2.URLError, xmlrpclib.ProtocolError):
logger.logger.error('%s | ERROR: Could not contact RPC server'
% machine)
return
if problem_test:
job_id = problem_test['job']['id']
job_name = problem_test['job']['name']
bug = reporting.MachineKillerBug(job_id=job_id,
job_name=job_name,
machine=machine)
reporter = reporting.Reporter()
bug_id = reporter.report(bug)[0]
if bug_id is None:
try:
email_prefix = ('The following test is killing a machine, '
'could not file a bug to report this:\n\n')
mail.send(_SENDER_ADDRESS, _NOTIFY_ADDRESS, '',
bug.title(), email_prefix + bug.summary())
except smtplib.SMTPDataError:
logger.logger.error('%s | %d | %s'
% (machine, job_id, job_name))
message += '%s\n%s\n\t%s' % (pool, host, '\n\t'.join(jobs))
jobs_to_analyze += relevent_hqes
else:
job_less.append(host.hostname)
if job_less:
message += (
'\nNo jobs found for the following hosts within cutoff %s\n\t' %
cutoff)
message += '\n\t'.join(job_less)
if pool_less:
message += '%s%s' % ('\nNo pools found on the following hosts:',
'\n\t'.join(pool_less))
if jobs_to_analyze:
message += '\n\n%s' % analyze_jobs(jobs_to_analyze)
if message:
return '%s\n%s' % ('Host information:', message)
return 'No hosts matching query %s from %s days back' % (payload,
days_back)
if __name__ == '__main__':
args = _parse_args(sys.argv[1:])
message = last_jobs_on_hosts(_parse_query(args.query), int(args.limit),
int(args.days_back))
if args.email:
mail.send('', args.email, '', 'Results from your sheirff script.',
message)
print message
