def submit_errors(error_msg):
""" submit_errors is used for sending out notifications
of errors.
We can put in a variety of things here. For now, we have
email submission and a quick syslog sent over to verify that
we made it into here.
"""
# If there's no error messages, just get out of here
if not error_msg:
util.write_log('nothing to submit')
return True
myhost = util.get_differ_hostname()
# email out the message
if DIFFER_EMAIL_ERRORS:
subject = 'Differ ERRORS: %s' % myhost
util.mail_it(RCPT_TO, MAIL_FROM, subject, error_msg, '*****@*****.**')
# send to the syslog on DIFFERLOGHOST the fact that we sent out an error
# helpful for perhaps getting a quick look at how many servers
# were sending out errors
human_time = time.strftime('%Y%m%d %H:%M', time.localtime())
try:
syslog_msg = '%s errors submitted at %s' % (myhost, human_time)
c = syslog_client.syslog_client((DIFFERLOGHOST, 514))
c.log(syslog_msg, facility='local4', priority='info')
except:
pass
def main():
"""
Ok, the parser is abused a little bit here when I use set const. But, instead of
having some long if/elif/else tree to set our different variables, we just set it
inside the const and have everything get parsed and formatted for later on.
"""
parser = optparse.OptionParser()
parser.add_option('--min', help='summarize 1 min', dest='duration', \
action='store_const', const='60,Minutely')
parser.add_option('--5min', help='summarize 5 min', dest='duration', \
action='store_const', const='300,5Min')
parser.add_option('--hour', help='summarize 1 hour', dest='duration', \
action='store_const', const='3600,Hourly')
parser.add_option('--day', help='summarize 1 day', dest='duration', \
action='store_const', const='86400,Daily')
parser.add_option('--week', help='summarize 1 week', dest='duration', \
action='store_const', const='604800,Weekly')
parser.add_option('--nagios', help='output nagios data', dest='nagios', \
action='store_true', default=False)
parser.add_option('--email', help='email out to prod-alerts', dest='email', \
action='store_true', default=False)
parser.add_option('--product', help='product to summarize [default: ALL]', dest='product', \
action='store', default='%%')
options, args = parser.parse_args()
# we require a duration from the parser options
if options.duration is None:
parser.print_help()
sys.exit(-1)
start, nicename = options.duration.split(',')
start = time.time() - int(start)
total, product_totals, filederrors = SummarizeFiledErrors(start, options.product)
if options.nagios:
perfdata = 'cases=%s' % total
msg = 'OK: %s case(s) filed|%s' % (total, perfdata)
print msg
sys.exit(0)
elif total > 0 and options.email:
subject = 'Differ %s Summary' % nicename
msg = '%s\n Filed in Total: %s' % (filederrors, total)
emailto = lolfly.RCPT_TO
util.mail_it(emailto, lolfly.MAIL_FROM, subject, msg, lolfly.REPLY_TO)
else:
print filederrors
print 'Total : %s' % total
def main():
start = time.time()
util.write_log("starting %s" % sys.argv[0])
case_count, case_list, product_list, output = file_errors(limit=400)
if case_count > 0:
lolfly_rcpt_to = RCPT_TO
subject = "Fogbugz Submissions"
emailoutput = "%s\n\nSUMMARY:%s case(s) filed" % (output, case_count)
if LOLFLY_EMAIL_ERRORS:
util.mail_it(lolfly_rcpt_to, MAIL_FROM, subject, emailoutput, REPLY_TO)
util.write_log("%s cases were filed" % case_count)
end = time.time()
duration = end - start
util.write_log("finished in %s sec" % duration)
def file_errors(limit=1):
case_count = 0
case_list = []
product_list = []
yearmonth = time.strftime("%Y%m")
msg = ""
differ = differdb.DifferDB()
# Sometimes, fogbugz ain't the happiest, so let's try handling some
# of the exceptions that can arise from it
try:
fbz = fbz_filer.FBZ()
except fogbugz.FogBugzLogonError:
msg += util.write_log("unable to login to fogbugz, exiting...")
return case_count, case_list, product_list, msg
except:
# General exception catch-all. It appears some of the vendor libraries don't import things
# in a friendly way for things like pychecker
msg += util.write_log("problems talking to fogbugz, exiting...")
return case_count, case_list, product_list, msg
# First section, get the errors that have exceptions
for i in differ.get_grouped_unfiled_exceptions():
code_location = i.code_location
code_method = i.code_method
exception = i.exception
bug_count = i.count
error_message = i.error_message
host = i.host
logfile = i.logfile
log_location = "%s:%s" % (host, logfile)
if exception == "pkg_resources.ExtractionError" and code_method == "extraction_error":
product = "IGNORE_THIS_ERROR"
elif exception == "socket.error" and code_method == "bind":
product = "IGNORE_THIS_ERROR"
elif code_location and code_location.startswith("/var/www/"):
product = code_location.split("/")[3]
elif code_location and code_location.startswith("/usr"):
product = "ops"
else:
product = None
# Sometimes there's an error that happens and you're going
# to ignore it, so let's just keep on looping
if product == "IGNORE_THIS_ERROR":
differ.update_group_case_id(-1, code_location, code_method, exception)
differ.update_group_product(product, -1, code_location, code_method, exception)
msg += util.write_log("IGNORED: %s %s on %s" % (code_method, exception, host))
continue
error_message = util.smart_truncate(error_message, length=MAX_BODY_LEN)
# might as well strip the /var/www/ in the fbz title
fbz_code_location = code_location
if fbz_code_location and fbz_code_location.startswith("/var/www/"):
fbz_code_location = fbz_code_location[9:]
bug_title = "%s %s:%s" % (exception, fbz_code_location, code_method)
bug_text = "%s error(s)\n---- %s ----\n%s" % (bug_count, log_location, error_message)
# There is still some debugging to do as to why some output isn't getting submitted
# properly. Until then, wrap the case section in a try/except so that the general
# automation still works
try:
case, priority = fbz.file_case(product, bug_title, bug_text)
differ.update_group_case_id(case, code_location, code_method, exception)
differ.update_group_product(product, case, code_location, code_method, exception)
log_output = "%sx p%s %s %s %s:%s:%s" % (
bug_count,
priority,
product,
exception,
host,
code_location,
code_method,
)
util.write_log(log_output)
if priority <= 5:
email_output = "%s\n%s/?%s\n\n" % (log_output, FBZ_URL, case)
msg += email_output
case_count += 1
case_list.append(case)
product_list.append(product)
except fogbugz.FogBugzConnectionError:
# We've been seeing that with this error, if we truncate off some of the text after
# WSGI variables, we're good
bug_text = bug_text.split("WSGI Variables")[0]
case, priority = fbz.file_case(product, bug_title, bug_text)
differ.update_group_case_id(case, code_location, code_method, exception)
differ.update_group_product(product, case, code_location, code_method, exception)
log_output = "%sx p%s %s %s %s:%s:%s" % (
bug_count,
priority,
product,
exception,
host,
code_location,
code_method,
)
util.write_log(log_output)
if priority <= 5:
email_output = "%s\n%s/?%s\n\n" % (log_output, FBZ_URL, case)
msg += email_output
case_count += 1
case_list.append(case)
product_list.append(product)
except:
traceback.print_exc(sys.stdout)
msg += util.write_log(
"LOLFLY ERROR: submitting info for %s %s %s" % (code_location, code_method, exception)
)
util.mail_it(ERROR_TO, MAIL_FROM, "DIFFER GOT AN ERROR", msg, REPLY_TO)
# second section, get the errors without exceptions
for i in differ.get_unfiled_nonexceptions(limit=limit):
errorid = i.id
timestamp = i.timestamp
host = i.host
logfile = i.logfile
error_message = i.error_message
log_location = "%s:%s" % (host, logfile)
error_message = util.smart_truncate(error_message, length=MAX_BODY_LEN)
# the message isn't formatted in any way we recognize, so just grab the first line
bug_title = error_message.split("\n")[0]
bug_title = util.smart_truncate(bug_title, length=MAX_TITLE_LEN)
if logfile.startswith("/var/log/mysql"):
product = "ops"
elif LOL_ERROR.search(bug_title):
product = LOL_ERROR.search(bug_title).group(1)
else:
product = None
# attempt to strip out date information at the beginning of the title
for fmt in util.DATE_FMT:
bug_title = re.sub(fmt, "", bug_title)
bug_text = "1 error(s)\n--%s--\n%s" % (log_location, error_message)
try:
case, priority = fbz.file_case(product, bug_title, bug_text)
util.write_log("update_case_id(%s, %s)" % (errorid, case))
differ.update_case_id(errorid, case)
log_output = '1x p%s %s %s:"%s"' % (priority, product, host, bug_title)
util.write_log(log_output)
if priority <= 5:
email_output = "%s\n%s/?%s\n\n" % (log_output, FBZ_URL, case)
msg += email_output
case_count += 1
case_list.append(case)
product_list.append(product)
except fogbugz.FogBugzConnectionError:
bug_text = util.smart_truncate(bug_text, length=MAX_BODY_LEN)
case, priority = fbz.file_case(product, bug_title, bug_text)
differ.update_case_id(errorid, case)
log_output = '1x p%s %s %s:"%s"' % (priority, product, host, bug_title)
util.write_log(log_output)
if priority <= 5:
email_output = "%s\n%s/?%s\n\n" % (log_output, FBZ_URL, case)
msg += email_output
case_count += 1
case_list.append(case)
product_list.append(product)
except fogbugz.FogBugzAPIError, exc:
traceback.print_exc(sys.stdout)
errmsg = "LOLFLY ERROR: got %r when filing a case for (%r, %r, %s)" % (
exc,
product,
bug_title,
repr(bug_text)[:40] + "...",
)
util.write_log(errmsg)
msg += errmsg
msg += "\n\n"
