def __call__(self, fn, instance, args, kwargs): for attempt in xrange(self.count): LOG.debug( "Retry attempt #%d/%d to call %s:%s:%s", attempt, self.count, fn.__class__.__name__, fn.__module__, fn.__name__, ) try: rc = fn(*args, **kwargs) if self.refuse_rc_fn and self.refuse_rc_fn(rc): LOG.debug("RC refused...retrying...: %s", rc) continue return rc except Exception as e: if self.log_faults: logtool.log_fault(e, level=self.log_faults_level) else: LOG.info("Attempt #%d failed: %s", attempt, e) if self.refuse_exc_fn and self.refuse_exc_fn(e): LOG.debug("Exception refused: %s", e) raise if self.expose_last_exc and attempt == self.count - 1: # Last one LOG.debug("Exposing last exception: %s", e) raise zzz = ( self.delay + ((-1 if random.random() < 0.5 else 1) * (self.jitter * random.random())) + attempt * self.backoff ) LOG.debug("Retryp delay: %d seconds", zzz) time.sleep(zzz) raise FailedTooOften
def load_belief_file (self, target, fname): self.debug (" belief: %s" % fname) try: target.update (json.loads (file (fname).read ())) except Exception as e: logtool.log_fault (e) self.error ("Error loading belief: %s -- %s" % (fname, e)) raise ValueError
def load_belief_file (self, target, fname): self.debug (" belief: %s" % fname) try: n = json.loads (open (fname, encoding = "utf-8").read ()) target = self._mergedicts (target, n) except Exception as e: logtool.log_fault (e) self.error ("Error loading belief: %s -- %s" % (fname, e)) raise ValueError
def main(): try: APP.run() except KeyboardInterrupt: pass except clip.ClipExit as e: sys.exit(e.status) except Exception as e: logtool.log_fault(e) sys.exit(1)
def sentry_handler (e, task, message = None, local_settings = None): """Yes, this eats exceptions""" try: app_name = _settings_value (local_settings, "APPLICATION_NAME", settings.APPLICATION_NAME) app_ver = _settings_value ( local_settings, "APPLICATION_VERSION", settings.APPLICATION_VERSION if app_name == settings.APPLICATION_NAME else "unknown") app_dsn = _settings_value ( local_settings, "SENTRY_DSN", settings.RAVEN_CONFIG["dsn"]) # pylint: disable=no-member app_tags = {"component": app_name, "version": app_ver} # Localise the Sentry for the plugin sentry = raven.Client (app_dsn, auto_log_stacks = True, release = "%s: %s" % (app_name, app_ver), transport = raven.transport.http.HTTPTransport) logtool.log_fault (e, message = message, level = logging.INFO) data = { "tags": app_tags, "task": task, "request": { "args": task.request.args, "callbacks": task.request.callbacks, "called_directly": task.request.called_directly, "correlation_id": task.request.correlation_id, "delivery_info": task.request.delivery_info, "errbacks": task.request.errbacks, "eta": task.request.eta, "expires": task.request.expires, "headers": task.request.headers, "hostname": task.request.hostname, "id": task.request.id, "is_eager": task.request.is_eager, "kwargs": task.request.kwargs, "reply_to": task.request.reply_to, "retries": task.request.retries, "task": task.request.task, "taskset": task.request.taskset, "timelimit": task.request.timelimit, "utc": task.request.utc, }, } if message: data["message"] = message sentry.extra_context (data) einfo = sys.exc_info () rc = sentry.captureException (einfo, **app_tags) del einfo LOG.error ("Sentry filed: %s", rc) except Exception as ee: logtool.log_fault (ee, message = "FAULT: Problem logging exception.", level = logging.INFO)
def sentry_handler(e, task, message=None, local_settings=None): """Yes, this eats exceptions""" try: app_name = _settings_value(local_settings, "APPLICATION_NAME", settings.APPLICATION_NAME) app_ver = _settings_value( local_settings, "APPLICATION_VERSION", settings.APPLICATION_VERSION if app_name == settings.APPLICATION_NAME else "unknown") app_dsn = _settings_value(local_settings, "SENTRY_DSN", settings.RAVEN_CONFIG["dsn"]) # pylint: disable=no-member app_tags = {"component": app_name, "version": app_ver} # Localise the Sentry for the plugin sentry = raven.Client(app_dsn, auto_log_stacks=True, release="%s: %s" % (app_name, app_ver), transport=raven.transport.http.HTTPTransport) logtool.log_fault(e, message=message, level=logging.INFO) data = { "tags": app_tags, "task": task, "request": { "args": task.request.args, "callbacks": task.request.callbacks, "called_directly": task.request.called_directly, "correlation_id": task.request.correlation_id, "delivery_info": task.request.delivery_info, "errbacks": task.request.errbacks, "eta": task.request.eta, "expires": task.request.expires, "headers": task.request.headers, "hostname": task.request.hostname, "id": task.request.id, "is_eager": task.request.is_eager, "kwargs": task.request.kwargs, "reply_to": task.request.reply_to, "retries": task.request.retries, "task": task.request.task, "taskset": task.request.taskset, "timelimit": task.request.timelimit, "utc": task.request.utc, }, } if message: data["message"] = message sentry.extra_context(data) einfo = sys.exc_info() rc = sentry.captureException(einfo, **app_tags) del einfo LOG.error("Sentry filed: %s", rc) except Exception as ee: logtool.log_fault(ee, message="FAULT: Problem logging exception.", level=logging.INFO)
def main(): try: APP.run() except KeyboardInterrupt: pass except clip.ClipExit: sys.exit(0) except Exception as e: logtool.log_fault(e) print >> sys.stderr, ("Something broke!") client = raven.Client( "https://250e838eaff24eee9461682bc7160904" ":[email protected]/127918") h = client.captureException() print >> sys.stderr, "\t Sentry filed: %s" % h sys.exit(1)
def main (): try: APP.run () except KeyboardInterrupt: pass except clip.ClipExit: sys.exit (0) except Exception as e: logtool.log_fault (e) print >> sys.stderr, ("Something broke!") if not (getattr (Config, "_nosentry", False) or Config.get ("user/nosentry", params = {"default": True})): client = raven.Client ( "https://250e838eaff24eee9461682bc7160904" ":[email protected]/127918") h = client.captureException () print >> sys.stderr, "\t Sentry filed: %s" % h sys.exit (1)
def main (): try: cfg = ConfigObj (DEFAULT_WORKDIR + DEFAULT_CONFIGFILE, interpolation = False) CONFIG.update (dict (cfg)) except: # pylint: disable = W0702 print ("Unable to parse config file: %s" % DEFAULT_CONFIGFILE, file = sys.stderr) sys.exit (9) try: APP.run () except clip.ClipExit as e: sys.exit (1 if e.status else 0) except (CmdError, KeyboardInterrupt): sys.exit (2) except Exception as e: logtool.log_fault (e) sys.exit (3)
def main (): try: logging.config.fileConfig (DEFAULT_LOGCONF, disable_existing_loggers = False) setproctitle.setproctitle (DEFAULT_PROCNAME) conf = ConfigObj (DEFAULT_CONF, interpolation = False) app_main (conf) except InitialiseException as e: print >> sys.stderr, "Configuration problem. See logs for details." sys.exit (1) except KeyboardInterrupt: sys.exit (0) except RuntimeException: LOG.error ("Exiting...") sys.exit (1) except Exception as e: logtool.log_fault (e) LOG.error ("Exiting...") sys.exit (1)
def sentry_exception (conf, stats, e, message = None): sentry = raven.Client (conf["sentry_dsn"], auto_log_stacks = True, release = get_versions ()["version"]) sentry_tags = {"component": "coneyeye"} logtool.log_fault (e, message = message) data = { "conf": conf, "stats": stats, } if message: data["message"] = message sentry.extra_context (data) try: exc_info = sys.exc_info() rc = sentry.captureException (exc_info, **sentry_tags) LOG.error ("Sentry filed: %s", rc) finally: del exc_info
def sentry_exception (e, request, message = None): """Yes, this eats exceptions""" try: logtool.log_fault (e, message = message, level = logging.INFO) data = { "task": task, "request": { "args": task.request.args, "callbacks": task.request.callbacks, "called_directly": task.request.called_directly, "correlation_id": task.request.correlation_id, "delivery_info": task.request.delivery_info, "errbacks": task.request.errbacks, "eta": task.request.eta, "expires": task.request.expires, "headers": task.request.headers, "hostname": task.request.hostname, "id": task.request.id, "is_eager": task.request.is_eager, "kwargs": task.request.kwargs, "origin": task.request.origin, "parent_id": task.request.parent_id, "reply_to": task.request.reply_to, "retries": task.request.retries, "root_id": task.request.root_id, "task": task.request.task, "taskset": task.request.taskset, "timelimit": task.request.timelimit, "utc": task.request.utc, }, } if message: data["message"] = message SENTRY.extra_context (data) einfo = sys.exc_info () rc = SENTRY.captureException (einfo) del einfo LOG.error ("Sentry filed: %s", rc) except Exception as ee: logtool.log_fault (ee, message = "FAULT: Problem logging exception.", level = logging.INFO)
def sentry_exception (e, request, message = None): sentry_tags = {"component": "qcameravalues"} try: sentry = raven.Client (settings.RAVEN_CONFIG["dsn"], auto_log_stacks = True) logtool.log_fault (e, message = message) data = { "job": request, } if message: data["message"] = message sentry.extra_context (data) if e is not None: einfo = sys.exc_info () rc = sentry.captureException (einfo, **sentry_tags) del einfo else: rc = sentry.capture (**sentry_tags) LOG.error ("Sentry filed: %s", rc) except Exception as ee: logtool.log_fault (ee, message = "FAULT: Problem logging exception.")
def main (): try: (options, args) = parse_options () # pylint: disable-msg = W0612 if options.rc: sys.exit (options.rc) actions = { "check": do_check, "clean": do_clean, "sample": do_sample, "status": do_status, "write": do_write, } rc = actions.get (options.action, do_unknown) (options) if rc: print_error (options, "ERROR: Failure during %s process." % options.action) sys.exit (rc) except Exception as e: # pylint: disable-msg = W0702 logtool.log_fault (e, message = "FAULT: Irrecoverable error.", level = logging.DEBUG) sys.exit (99)
def app_main (conf): try: if not conf.get ("sentry_dsn"): raise InitialiseException ("Missing configuration: sentry_dsn") mq_conn = { "url": conf.get ("rabbitmq_adminapi_url"), "user": conf.get ("rabbitmq_adminapi_user"), "passwd": conf.get ("rabbitmq_adminapi_passwd"), } statsd_conn = { "host": conf.get ("statsd_host", "127.0.0.1"), "port": conf.get ("statsd_port", 8125), "prefix": conf.get ("statsd_prefix", "rabbitmq"), } delay = int (conf.get ("delay", 5)) * 60 # ignore_queues = ["*.celery@*", "*.celeryev.*", "*pidbox"] ignore_queues = [s.strip () for s in conf.get ("suppress_queues", "").split(",")] except Exception as e: logtool.log_fault (e) raise InitialiseException stats = None while True: try: with statsd.StatsClient (**statsd_conn).pipeline() as pipe: stats = mqreport (mq_conn, ignore_queues = ignore_queues) LOG.info (json.dumps (stats, indent = 2)) for k, v in stats.items (): pipe.gauge (k, v) pipe.send () LOG.info ("Sent...") time.sleep (delay) except KeyboardInterrupt: raise except Exception as e: sentry_exception (conf, stats, e) LOG.info ("Sleeping for a minute...") time.sleep (60)