def __call__(self, fn, instance, args, kwargs):
for attempt in xrange(self.count):
LOG.debug(
"Retry attempt #%d/%d to call %s:%s:%s",
attempt,
self.count,
fn.__class__.__name__,
fn.__module__,
fn.__name__,
)
try:
rc = fn(*args, **kwargs)
if self.refuse_rc_fn and self.refuse_rc_fn(rc):
LOG.debug("RC refused...retrying...: %s", rc)
continue
return rc
except Exception as e:
if self.log_faults:
logtool.log_fault(e, level=self.log_faults_level)
else:
LOG.info("Attempt #%d failed: %s", attempt, e)
if self.refuse_exc_fn and self.refuse_exc_fn(e):
LOG.debug("Exception refused: %s", e)
raise
if self.expose_last_exc and attempt == self.count - 1: # Last one
LOG.debug("Exposing last exception: %s", e)
raise
zzz = (
self.delay
+ ((-1 if random.random() < 0.5 else 1) * (self.jitter * random.random()))
+ attempt * self.backoff
)
LOG.debug("Retryp delay: %d seconds", zzz)
time.sleep(zzz)
raise FailedTooOften
def load_belief_file (self, target, fname):
self.debug (" belief: %s" % fname)
try:
target.update (json.loads (file (fname).read ()))
except Exception as e:
logtool.log_fault (e)
self.error ("Error loading belief: %s -- %s" % (fname, e))
raise ValueError
def load_belief_file (self, target, fname):
self.debug (" belief: %s" % fname)
try:
n = json.loads (open (fname, encoding = "utf-8").read ())
target = self._mergedicts (target, n)
except Exception as e:
logtool.log_fault (e)
self.error ("Error loading belief: %s -- %s" % (fname, e))
raise ValueError
def main():
try:
APP.run()
except KeyboardInterrupt:
pass
except clip.ClipExit as e:
sys.exit(e.status)
except Exception as e:
logtool.log_fault(e)
sys.exit(1)
def sentry_handler (e, task, message = None, local_settings = None):
"""Yes, this eats exceptions"""
try:
app_name = _settings_value (local_settings, "APPLICATION_NAME",
settings.APPLICATION_NAME)
app_ver = _settings_value (
local_settings, "APPLICATION_VERSION",
settings.APPLICATION_VERSION if app_name == settings.APPLICATION_NAME
else "unknown")
app_dsn = _settings_value (
local_settings, "SENTRY_DSN",
settings.RAVEN_CONFIG["dsn"]) # pylint: disable=no-member
app_tags = {"component": app_name,
"version": app_ver}
# Localise the Sentry for the plugin
sentry = raven.Client (app_dsn,
auto_log_stacks = True,
release = "%s: %s" % (app_name, app_ver),
transport = raven.transport.http.HTTPTransport)
logtool.log_fault (e, message = message, level = logging.INFO)
data = {
"tags": app_tags,
"task": task,
"request": {
"args": task.request.args,
"callbacks": task.request.callbacks,
"called_directly": task.request.called_directly,
"correlation_id": task.request.correlation_id,
"delivery_info": task.request.delivery_info,
"errbacks": task.request.errbacks,
"eta": task.request.eta,
"expires": task.request.expires,
"headers": task.request.headers,
"hostname": task.request.hostname,
"id": task.request.id,
"is_eager": task.request.is_eager,
"kwargs": task.request.kwargs,
"reply_to": task.request.reply_to,
"retries": task.request.retries,
"task": task.request.task,
"taskset": task.request.taskset,
"timelimit": task.request.timelimit,
"utc": task.request.utc,
},
}
if message:
data["message"] = message
sentry.extra_context (data)
einfo = sys.exc_info ()
rc = sentry.captureException (einfo, **app_tags)
del einfo
LOG.error ("Sentry filed: %s", rc)
except Exception as ee:
logtool.log_fault (ee, message = "FAULT: Problem logging exception.",
level = logging.INFO)
def sentry_handler(e, task, message=None, local_settings=None):
"""Yes, this eats exceptions"""
try:
app_name = _settings_value(local_settings, "APPLICATION_NAME",
settings.APPLICATION_NAME)
app_ver = _settings_value(
local_settings, "APPLICATION_VERSION", settings.APPLICATION_VERSION
if app_name == settings.APPLICATION_NAME else "unknown")
app_dsn = _settings_value(local_settings, "SENTRY_DSN",
settings.RAVEN_CONFIG["dsn"]) # pylint: disable=no-member
app_tags = {"component": app_name, "version": app_ver}
# Localise the Sentry for the plugin
sentry = raven.Client(app_dsn,
auto_log_stacks=True,
release="%s: %s" % (app_name, app_ver),
transport=raven.transport.http.HTTPTransport)
logtool.log_fault(e, message=message, level=logging.INFO)
data = {
"tags": app_tags,
"task": task,
"request": {
"args": task.request.args,
"callbacks": task.request.callbacks,
"called_directly": task.request.called_directly,
"correlation_id": task.request.correlation_id,
"delivery_info": task.request.delivery_info,
"errbacks": task.request.errbacks,
"eta": task.request.eta,
"expires": task.request.expires,
"headers": task.request.headers,
"hostname": task.request.hostname,
"id": task.request.id,
"is_eager": task.request.is_eager,
"kwargs": task.request.kwargs,
"reply_to": task.request.reply_to,
"retries": task.request.retries,
"task": task.request.task,
"taskset": task.request.taskset,
"timelimit": task.request.timelimit,
"utc": task.request.utc,
},
}
if message:
data["message"] = message
sentry.extra_context(data)
einfo = sys.exc_info()
rc = sentry.captureException(einfo, **app_tags)
del einfo
LOG.error("Sentry filed: %s", rc)
except Exception as ee:
logtool.log_fault(ee,
message="FAULT: Problem logging exception.",
level=logging.INFO)
def main():
try:
APP.run()
except KeyboardInterrupt:
pass
except clip.ClipExit:
sys.exit(0)
except Exception as e:
logtool.log_fault(e)
print >> sys.stderr, ("Something broke!")
client = raven.Client(
"https://250e838eaff24eee9461682bc7160904"
":[email protected]/127918")
h = client.captureException()
print >> sys.stderr, "\t Sentry filed: %s" % h
sys.exit(1)
def main ():
try:
APP.run ()
except KeyboardInterrupt:
pass
except clip.ClipExit:
sys.exit (0)
except Exception as e:
logtool.log_fault (e)
print >> sys.stderr, ("Something broke!")
if not (getattr (Config, "_nosentry", False)
or Config.get ("user/nosentry", params = {"default": True})):
client = raven.Client (
"https://250e838eaff24eee9461682bc7160904"
":[email protected]/127918")
h = client.captureException ()
print >> sys.stderr, "\t Sentry filed: %s" % h
sys.exit (1)
def main ():
try:
cfg = ConfigObj (DEFAULT_WORKDIR + DEFAULT_CONFIGFILE,
interpolation = False)
CONFIG.update (dict (cfg))
except: # pylint: disable = W0702
print ("Unable to parse config file: %s" % DEFAULT_CONFIGFILE,
file = sys.stderr)
sys.exit (9)
try:
APP.run ()
except clip.ClipExit as e:
sys.exit (1 if e.status else 0)
except (CmdError, KeyboardInterrupt):
sys.exit (2)
except Exception as e:
logtool.log_fault (e)
sys.exit (3)
def main ():
try:
logging.config.fileConfig (DEFAULT_LOGCONF,
disable_existing_loggers = False)
setproctitle.setproctitle (DEFAULT_PROCNAME)
conf = ConfigObj (DEFAULT_CONF, interpolation = False)
app_main (conf)
except InitialiseException as e:
print >> sys.stderr, "Configuration problem. See logs for details."
sys.exit (1)
except KeyboardInterrupt:
sys.exit (0)
except RuntimeException:
LOG.error ("Exiting...")
sys.exit (1)
except Exception as e:
logtool.log_fault (e)
LOG.error ("Exiting...")
sys.exit (1)
def sentry_exception (conf, stats, e, message = None):
sentry = raven.Client (conf["sentry_dsn"],
auto_log_stacks = True,
release = get_versions ()["version"])
sentry_tags = {"component": "coneyeye"}
logtool.log_fault (e, message = message)
data = {
"conf": conf,
"stats": stats,
}
if message:
data["message"] = message
sentry.extra_context (data)
try:
exc_info = sys.exc_info()
rc = sentry.captureException (exc_info, **sentry_tags)
LOG.error ("Sentry filed: %s", rc)
finally:
del exc_info
def sentry_exception (e, request, message = None):
"""Yes, this eats exceptions"""
try:
logtool.log_fault (e, message = message, level = logging.INFO)
data = {
"task": task,
"request": {
"args": task.request.args,
"callbacks": task.request.callbacks,
"called_directly": task.request.called_directly,
"correlation_id": task.request.correlation_id,
"delivery_info": task.request.delivery_info,
"errbacks": task.request.errbacks,
"eta": task.request.eta,
"expires": task.request.expires,
"headers": task.request.headers,
"hostname": task.request.hostname,
"id": task.request.id,
"is_eager": task.request.is_eager,
"kwargs": task.request.kwargs,
"origin": task.request.origin,
"parent_id": task.request.parent_id,
"reply_to": task.request.reply_to,
"retries": task.request.retries,
"root_id": task.request.root_id,
"task": task.request.task,
"taskset": task.request.taskset,
"timelimit": task.request.timelimit,
"utc": task.request.utc,
},
}
if message:
data["message"] = message
SENTRY.extra_context (data)
einfo = sys.exc_info ()
rc = SENTRY.captureException (einfo)
del einfo
LOG.error ("Sentry filed: %s", rc)
except Exception as ee:
logtool.log_fault (ee, message = "FAULT: Problem logging exception.",
level = logging.INFO)
def sentry_exception (e, request, message = None):
sentry_tags = {"component": "qcameravalues"}
try:
sentry = raven.Client (settings.RAVEN_CONFIG["dsn"],
auto_log_stacks = True)
logtool.log_fault (e, message = message)
data = {
"job": request,
}
if message:
data["message"] = message
sentry.extra_context (data)
if e is not None:
einfo = sys.exc_info ()
rc = sentry.captureException (einfo, **sentry_tags)
del einfo
else:
rc = sentry.capture (**sentry_tags)
LOG.error ("Sentry filed: %s", rc)
except Exception as ee:
logtool.log_fault (ee, message = "FAULT: Problem logging exception.")
def main ():
try:
(options, args) = parse_options () # pylint: disable-msg = W0612
if options.rc:
sys.exit (options.rc)
actions = {
"check": do_check,
"clean": do_clean,
"sample": do_sample,
"status": do_status,
"write": do_write,
}
rc = actions.get (options.action, do_unknown) (options)
if rc:
print_error (options,
"ERROR: Failure during %s process." % options.action)
sys.exit (rc)
except Exception as e: # pylint: disable-msg = W0702
logtool.log_fault (e, message = "FAULT: Irrecoverable error.",
level = logging.DEBUG)
sys.exit (99)
def app_main (conf):
try:
if not conf.get ("sentry_dsn"):
raise InitialiseException ("Missing configuration: sentry_dsn")
mq_conn = {
"url": conf.get ("rabbitmq_adminapi_url"),
"user": conf.get ("rabbitmq_adminapi_user"),
"passwd": conf.get ("rabbitmq_adminapi_passwd"),
}
statsd_conn = {
"host": conf.get ("statsd_host", "127.0.0.1"),
"port": conf.get ("statsd_port", 8125),
"prefix": conf.get ("statsd_prefix", "rabbitmq"),
}
delay = int (conf.get ("delay", 5)) * 60
# ignore_queues = ["*.celery@*", "*.celeryev.*", "*pidbox"]
ignore_queues = [s.strip ()
for s in conf.get ("suppress_queues", "").split(",")]
except Exception as e:
logtool.log_fault (e)
raise InitialiseException
stats = None
while True:
try:
with statsd.StatsClient (**statsd_conn).pipeline() as pipe:
stats = mqreport (mq_conn, ignore_queues = ignore_queues)
LOG.info (json.dumps (stats, indent = 2))
for k, v in stats.items ():
pipe.gauge (k, v)
pipe.send ()
LOG.info ("Sent...")
time.sleep (delay)
except KeyboardInterrupt:
raise
except Exception as e:
sentry_exception (conf, stats, e)
LOG.info ("Sleeping for a minute...")
time.sleep (60)
