def operation(self):
current_time = time.time()
error_msg = ""
for job in self.jobs_list:
if job.active is None:
continue
execution_time = current_time - job.start_time
time_from_last_failed_check = current_time - job.last_monitor_check_failed
if (execution_time > job.monitor_interval
and time_from_last_failed_check > job.monitor_interval):
job.last_monitor_check_failed = current_time
if execution_time < 60:
error = "Job %s is taking more than %d seconds to execute" % (
job.name, execution_time)
elif execution_time < 3600:
minutes = int(execution_time / 60)
error = "Job %s is taking more than %d minutes to execute" % (
job.name, minutes)
else:
hours = int(execution_time / 3600)
error = "Job %s is taking more than %d hours to execute" % (
job.name, hours)
error_msg += error + '\n'
log.err(error)
if error_msg:
schedule_exception_email(error_msg)
def _wrap(self, function, *args, **kwargs):
"""
Wrap provided function calling it inside a thread and
passing the store to it.
"""
with transact_lock:
start_time = datetime.now()
store = Store(create_database(GLSettings.db_uri))
try:
if self.instance:
result = function(self.instance, store, *args, **kwargs)
else:
result = function(store, *args, **kwargs)
store.commit()
except:
store.rollback()
raise
else:
return result
finally:
store.reset()
store.close()
duration = timedelta_to_milliseconds(datetime.now() -
start_time)
msg = "Query [%s] executed in %.1fms" % (self.method.__name__,
duration)
if duration > self.timelimit:
log.err(msg)
schedule_exception_email(msg)
else:
log.debug(msg)
def _wrap(self, function, *args, **kwargs):
"""
Wrap provided function calling it inside a thread and
passing the store to it.
"""
with transact_lock: # pylint: disable=not-context-manager
start_time = datetime.now()
store = get_store()
try:
if self.instance:
result = function(self.instance, store, *args, **kwargs)
else:
result = function(store, *args, **kwargs)
store.commit()
except:
store.rollback()
raise
else:
return result
finally:
store.reset()
store.close()
duration = timedelta_to_milliseconds(datetime.now() -
start_time)
err_tup = "Query [%s] executed in %.1fms", self.method.__name__, duration
if duration > self.timelimit:
log.err(*err_tup)
schedule_exception_email(*err_tup)
else:
log.debug(*err_tup)
def post(self):
request = self.validate_message(self.request.content.read(),
requests.ExceptionDesc)
if not GLSettings.disable_client_exception_notification:
exception_email = "URL: %s\n\n" % request['errorUrl']
exception_email += "User Agent: %s\n\n" % request['agent']
exception_email += "Error Message: %s\n\n" % request['errorMessage']
exception_email += "Stacktrace:\n"
exception_email += json.dumps(request['stackTrace'], indent=2)
schedule_exception_email(exception_email)
log.debug("Received client exception and passed error to exception mail handler")
def execution_check(self):
self.request.execution_time = datetime.now() - self.request.start_time
if self.request.execution_time.seconds > self.handler_exec_time_threshold:
err_tup = ("Handler [%s] exceeded execution threshold (of %d secs) with an execution time of %.2f seconds",
self.name, self.handler_exec_time_threshold, self.request.execution_time.seconds)
log.err(*err_tup)
schedule_exception_email(*err_tup)
track_handler(self)
if self.uniform_answer_time:
needed_delay = (Settings.side_channels_guard - (self.request.execution_time.microseconds / 1000)) / 1000
if needed_delay > 0:
yield deferred_sleep(needed_delay)