def mailfailure(jobname, job, message): message_lines = [""] message_lines.append("The following tests FAILED for this job") message_lines.append("http://%s/results/%s" % (socket.gethostname(), jobname)) message_lines.append("") message_lines.append(format_failure_message("Job name", "Kernel", "Test name", "FAIL/WARN", "Failure reason")) message_lines.append(format_failure_message("=" * 8, "=" * 6, "=" * 8, "=" * 8, "=" * 14)) message_header = "\n".join(message_lines) subject = "AUTOTEST: FAILED tests from job %s" % jobname mail.send("", job.user, "", subject, message_header + message)
def email_about_test_failure(failed_tests, all_tests): """ Send an email about all the tests that have failed if there are any. @param failed_tests: The list of failed tests. This will be sorted in this function. @param all_tests: All the names of tests that have been recently ran. """ if failed_tests: failed_tests.sort() mail.send( _MAIL_RESULTS_FROM, [_MAIL_RESULTS_TO], [], 'Long Failing Tests', '%d/%d tests have been failing for at least %d days.\n' 'They are the following:\n\n%s' % (len(failed_tests), len(all_tests), _DAYS_TO_BE_FAILING_TOO_LONG, '\n'.join(failed_tests)))
def test_email_count_of_total_number_of_tests(self): """Test that the email report displays total number of tests.""" complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60 mail.send( '*****@*****.**', ['*****@*****.**'], [], 'Long Failing Tests', '1/2 tests have been failing for at least %d days.\n' 'They are the following:\n\ntest' % complete_failures._DAYS_TO_BE_FAILING_TOO_LONG) failures = ['test'] all_tests = set(failures) | {'not_failure'} self.mox.ReplayAll() complete_failures.email_about_test_failure(failures, all_tests)
def test_email_sent_about_all_failed_tests(self): """Test that the email report mentions all the failed_tests.""" complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60 mail.send( '*****@*****.**', ['*****@*****.**'], [], 'Long Failing Tests', '1/1 tests have been failing for at least %d days.\n' 'They are the following:\n\ntest' % complete_failures._DAYS_TO_BE_FAILING_TOO_LONG) failures = ['test'] all_tests = set(failures) self.mox.ReplayAll() complete_failures.email_about_test_failure(failures, all_tests)
def test(self): """Does a basic test of as much of the system as possible.""" job = models.Job(job_idx=1) kernel = models.Kernel(kernel_idx=1) machine = models.Machine(machine_idx=1) success_status = models.Status(status_idx=GOOD_STATUS_IDX) fail_status = models.Status(status_idx=FAIL_STATUS_IDX) old_passing_test = models.Test(job=job, status=success_status, kernel=kernel, machine=machine, test='test1', started_time=self.datetime(2012, 1, 1)) old_passing_test.save() failing_test = models.Test(job=job, status=fail_status, kernel=kernel, machine=machine, test='test2', started_time=self.datetime(2012, 1, 1)) failing_test.save() good_test = models.Test(job=job, status=success_status, kernel=kernel, machine=machine, test='test3', started_time=self.datetime(2012, 1, 20)) good_test.save() complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 10 MockDatetime.today().AndReturn(self.datetime(2012, 1, 21)) MockDatetime.today().AndReturn(self.datetime(2012, 1, 21)) mail.send( '*****@*****.**', ['*****@*****.**'], [], 'Long Failing Tests', '2/3 tests have been failing for at least %d days.\n' 'They are the following:\n\ntest1\ntest2' % complete_failures._DAYS_TO_BE_FAILING_TOO_LONG) self.mox.ReplayAll() complete_failures.main()
def test_email_has_test_names_sorted_alphabetically(self): """Test that the email report has entries sorted alphabetically.""" complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60 mail.send( '*****@*****.**', ['*****@*****.**'], [], 'Long Failing Tests', '2/2 tests have been failing for at least %d days.\n' 'They are the following:\n\ntest1\ntest2' % complete_failures._DAYS_TO_BE_FAILING_TOO_LONG) # We use an OrderedDict to gurantee that the elements would be out of # order if we did a simple traversal. failures = ['test2', 'test1'] all_tests = set(failures) self.mox.ReplayAll() complete_failures.email_about_test_failure(failures, all_tests)
def mailfailure(jobname, job, message): """Send an email about the failure. @param jobname: String representing the job name. @param job: A job object. @param message: The message to mail. """ message_lines = [""] message_lines.append("The following tests FAILED for this job") message_lines.append("http://%s/results/%s" % (socket.gethostname(), jobname)) message_lines.append("") message_lines.append( format_failure_message("Job name", "Kernel", "Test name", "FAIL/WARN", "Failure reason")) message_lines.append( format_failure_message("=" * 8, "=" * 6, "=" * 8, "=" * 8, "=" * 14)) message_header = "\n".join(message_lines) subject = "AUTOTEST: FAILED tests from job %s" % jobname mail.send("", job.user, "", subject, message_header + message)
def flag_problem_test(machine): """ Notify people about the last job that ran on a machine. This method is invoked everytime a machine fails to repair, and attempts to identify the last test that ran on the machine. If successfull, it files a bug, or sends out an email, or just logs the fact. @param machine: The hostname (e.g. IP address) of the machine to find the last job ran on it. """ rpc = frontend.AFE() logger = MachineDeathLogger() try: problem_test = _find_problem_test(machine, rpc) except (urllib2.URLError, xmlrpclib.ProtocolError): logger.logger.error('%s | ERROR: Could not contact RPC server' % machine) return if problem_test: job_id = problem_test['job']['id'] job_name = problem_test['job']['name'] bug = reporting.MachineKillerBug(job_id=job_id, job_name=job_name, machine=machine) reporter = reporting.Reporter() bug_id = reporter.report(bug)[0] if bug_id is None: try: email_prefix = ('The following test is killing a machine, ' 'could not file a bug to report this:\n\n') mail.send(_SENDER_ADDRESS, _NOTIFY_ADDRESS, '', bug.title(), email_prefix + bug.summary()) except smtplib.SMTPDataError: logger.logger.error('%s | %d | %s' % (machine, job_id, job_name))
message += '%s\n%s\n\t%s' % (pool, host, '\n\t'.join(jobs)) jobs_to_analyze += relevent_hqes else: job_less.append(host.hostname) if job_less: message += ( '\nNo jobs found for the following hosts within cutoff %s\n\t' % cutoff) message += '\n\t'.join(job_less) if pool_less: message += '%s%s' % ('\nNo pools found on the following hosts:', '\n\t'.join(pool_less)) if jobs_to_analyze: message += '\n\n%s' % analyze_jobs(jobs_to_analyze) if message: return '%s\n%s' % ('Host information:', message) return 'No hosts matching query %s from %s days back' % (payload, days_back) if __name__ == '__main__': args = _parse_args(sys.argv[1:]) message = last_jobs_on_hosts(_parse_query(args.query), int(args.limit), int(args.days_back)) if args.email: mail.send('', args.email, '', 'Results from your sheirff script.', message) print message