def execute_task(task_fxn, *arg, **kwargs):
try:
task_fxn(*arg, **kwargs)
except:
if gae_utils.Debug():
raise
else:
if 'AppEngine' not in gae_utils.GetUserAgent():
raise # not being run from bg queue
from backend.admin import send_admin_email
from utils.gae import error_msg
error, err_msg = error_msg()
retry_task = False
for retry_error in ['DownloadError', 'DeadlineExceededError']:
if retry_error in error:
retry_task = True
task_msg = 'Task: %s' % (err_msg)
subject = 'Task: %s' % (error)
if retry_task:
task_msg = 'Retrying Task - ' + task_msg
subject = 'Retrying Task - ' + subject
send_admin_email(subject, task_msg)
logging.critical(task_msg)
if retry_task:
raise
raise deferred.PermanentTaskFailure
def execute_task(task_fxn, *arg, **kwargs):
try:
task_fxn(*arg, **kwargs)
except:
if gae_utils.Debug():
raise
else:
if 'AppEngine' not in gae_utils.GetUserAgent():
raise # not being run from bg queue
from backend.admin import send_admin_email
from utils.gae import error_msg
error, err_msg = error_msg()
retry_task = False
for retry_error in ['DownloadError','DeadlineExceededError']:
if retry_error in error:
retry_task = True
task_msg = 'Task: %s' % (err_msg)
subject = 'Task: %s' % (error)
if retry_task:
task_msg = 'Retrying Task - ' + task_msg
subject = 'Retrying Task - ' + subject
send_admin_email(subject, task_msg)
logging.critical(task_msg)
if retry_task:
raise
raise deferred.PermanentTaskFailure
def send_error(self, *args, **kwargs):
self.set_status(500) # always 500?
from utils.gae import error_msg
error, err_msg = error_msg()
logging.error('%s - %s' % (error, err_msg))
if gae_utils.Debug():
logging.info('debug server, not sending admin error report')
return super(BaseHandler, self).send_error(*args, **kwargs)
else:
for bot_agent in ['google','appengine','alexa','yahoo','bot','bing']:
if bot_agent in gae_utils.GetUserAgent().lower():
return# self.error_output(error, err_msg)
if '405: Method Not Allowed' in err_msg:
return self.error_output(error, err_msg)
# TODO: include info about user and session
# TODO: print out 500 page. And also do the same for 400 page.
user = self.get_current_user()
if user: user = user.key().name()
deferred.defer(send_admin_email,
subject='Error - %s' % error, message='Error Traceback - %s' % err_msg,
user=user, user_agent=gae_utils.GetUserAgent(), ip=gae_utils.IPAddress(), url=gae_utils.GetUrl())
self._finished = False
if getattr(self, 'server_error',''):
self.server_error()
# TODO: Ajax/API views should have their own error output
else:
self.error_output(error, err_msg)
self._finished = True
return
def throttle_slack(form_fields, throttle_count):
# TODO - refactor as more generic throttler
cache_key = 'slack-notify-%s' % form_fields['channel']
message_cache_key = 'slack-notify-%s-%s' % (form_fields['channel'],
form_fields['text'])
t = arrow.now()
saved_message_notification = memcache.get(message_cache_key)
if saved_message_notification:
logging.warning(
'Duplicate message notification for %s. Throttling notification notification.'
% message_cache_key)
return True
saved_notifications = memcache.get(cache_key)
if not saved_notifications: saved_notifications = []
recent_notifications = []
for n in saved_notifications:
try:
if isinstance(n, list):
n = n[0] # because of a previous bug, remove after June 2017
notification_delta = t - arrow.get(n)
if notification_delta.seconds < THROTTLE_TIME: # from the last 60 seconds
recent_notifications.append(n)
except:
logging.critical('error: notification parsing error: %s' %
gae_utils.error_msg(trace=True))
continue
recent_notifications.append(t.timestamp)
if len(recent_notifications) > throttle_count:
logging.error(
'Saved notify count of %s. Throttling notification notification.' %
len(recent_notifications))
return True
memcache.set_multi({
cache_key: recent_notifications,
message_cache_key: 1
},
time=THROTTLE_TIME)
return False
def send_error(self, *args, **kwargs):
self.set_status(500) # always 500?
from utils.gae import error_msg
error, err_msg = error_msg()
logging.error('%s - %s' % (error, err_msg))
if gae_utils.Debug():
logging.info('debug server, not sending admin error report')
return super(BaseHandler, self).send_error(*args, **kwargs)
else:
for bot_agent in [
'google', 'appengine', 'alexa', 'yahoo', 'bot', 'bing'
]:
if bot_agent in gae_utils.GetUserAgent().lower():
return # self.error_output(error, err_msg)
if '405: Method Not Allowed' in err_msg:
return self.error_output(error, err_msg)
# TODO: include info about user and session
# TODO: print out 500 page. And also do the same for 400 page.
user = self.get_current_user()
if user: user = user.key().name()
deferred.defer(send_admin_email,
subject='Error - %s' % error,
message='Error Traceback - %s' % err_msg,
user=user,
user_agent=gae_utils.GetUserAgent(),
ip=gae_utils.IPAddress(),
url=gae_utils.GetUrl())
self._finished = False
if getattr(self, 'server_error', ''):
self.server_error()
# TODO: Ajax/API views should have their own error output
else:
self.error_output(error, err_msg)
self._finished = True
return
def error_reporter(self, *pars, **kpars):
try:
handler_method(self, *pars, **kpars)
return
except:
from utils import gae as gae_utils
from google.appengine.api import memcache
#from service.slack import slack_notification
import arrow
error_cls = gae_utils.error_class()
from utils import errors
for error_group in (errors.NonErrors, errors.BadRequestErrors,
errors.AccessDeniedErrors):
if error_cls in error_group: # TODO - 40x errors
logging.warning(
'Error %s (%s) for handler %s does not require admin report'
% (error_cls, gae_utils.error_msg(), self.__class__))
raise
cache_key = '%s - %s' % (self.__class__, error_cls)
try:
last_error = memcache.get('error-report')
except AssertionError:
logging.error(
'AssertionError: No api proxy found for service "memcache"'
)
last_error = None
if last_error:
seconds_ago = arrow.now().timestamp - last_error
if seconds_ago < TOTAL_ERROR_THROTTLE:
logging.info(
'last error was %s seconds ago. Not sending error notification.'
% seconds_ago)
raise
raise
try:
throttled = memcache.get(cache_key)
except AssertionError:
logging.error(
'AssertionError: No api proxy found for service "memcache"'
)
return
if throttled:
logging.info(
'throttle found for cache key %s. Not sending error notification.'
% cache_key)
raise
# otherwise set throttle and notify
throttle = kwargs.get('throttle') or DEFAULT_REPORT_THROTTLE
memcache.set_multi(
{
'error-report': arrow.now().timestamp,
cache_key: True
},
time=throttle)
logging.info('Sending error notification for cache key %s.' %
cache_key)
subject = 'Error: %s' % cache_key
body = 'path: %s | %s' % (self.request.path, gae_utils.error_msg())
# from service.admin import send_admin_email
# send_admin_email(subject=subject, message=body,
# user_agent=gae_utils.GetUserAgent(), ip=gae_utils.IPAddress(),
# url=gae_utils.GetUrl(), admins=['*****@*****.**'])
# body = "%s -- UA: %s; IP: %s" % (body, gae_utils.GetUserAgent(),
# gae_utils.IPAddress())
# slack_notification('%s | %s' % (subject, body), channel='#dev-notify')
raise