def init_sentry(app):
sentry = IndicoSentry(wrap_wsgi=False, register_signal=True, logging=False)
sentry.init_app(app)
# setup logging manually and exclude uncaught indico exceptions.
# these are logged manually in the flask error handler logic so
# we get the X-Sentry-ID header which is not populated in the
# logging handlers
handler = SentryHandler(sentry.client,
level=getattr(logging,
config.SENTRY_LOGGING_LEVEL))
handler.addFilter(BlacklistFilter({'indico.flask', 'celery.redirected'}))
setup_logging(handler)
# connect to the celery logger
register_logger_signal(sentry.client)
register_signal(sentry.client)
def __init__(self, config_file, logger):
self.config = config_from_file(config_file)
self.info = config_from_file("info.json")
self.logger = logger
super(Bot, self).__init__(command_prefix=get_prefix,
shard_count=self.config.shard_count)
self.init_databases()
self.modlog = ModLog(self)
if self.config.sentry_dsn != None:
self.sentry = Client(self.config.sentry_dsn)
sentry_handler = SentryHandler(self.sentry)
sentry_handler.setLevel(logging.ERROR)
setup_logging(sentry_handler)
else:
self.sentry = None
def scheduler_sentry(self):
sentry_dsn_dict = SentryWrapper.get_config('sentry_dsn')
if sentry_dsn_dict is None:
return
scheduler_dsn = sentry_dsn_dict.get('scheduler')
if scheduler_dsn is None:
return
from raven import Client
from raven.handlers.logging import SentryHandler
from raven.conf import setup_logging
self.client = Client(dsn=scheduler_dsn)
self.handler = SentryHandler(self.client)
self.handler.setLevel(logging.ERROR)
# finish the job
setup_logging(self.handler)
def init_plugins():
"""初始化日志、错误追踪插件，并将当前配置信息打 log"""
from everyclass.common.flask import print_config
# Sentry
if __app.config['CONFIG_NAME'] in __app.config['SENTRY_AVAILABLE_IN']:
sentry.init_app(app=__app)
sentry_handler = SentryHandler(sentry.client)
sentry_handler.setLevel(logging.WARNING)
logging.getLogger().addHandler(sentry_handler)
logger.info('Sentry is inited because you are in {} mode.'.format(
__app.config['CONFIG_NAME']))
print_config(__app, logger)
def lambda_handler(event, context):
logger = logging.getLogger()
logger.debug(event)
raven_client = Raven_client() # Needs SENTRY_DSN env var
sentry_handler = SentryHandler(raven_client)
sentry_handler.setLevel(logging.WARNING)
setup_logging(sentry_handler)
logger.setLevel(logging.INFO)
enable_xray()
try:
RecordProcesser().process(event)
except Exception as e:
logger.error('error processing event'.format(event))
raise e
def setup_logging():
logger = app.logger
file_handler = logging.FileHandler(CONF.get("logging", "file"))
file_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
log_level = getattr(logging, CONF.get("logging", "level"))
logger.setLevel(log_level)
logger.addHandler(file_handler)
raven_dsn = CONF.get("logging", "sentry")
if raven_dsn:
raven_log_level = getattr(logging, CONF.get("logging", "sentry_level"))
sentry_handler = SentryHandler(raven_dsn)
sentry_handler.setLevel(raven_log_level)
logger.addHandler(sentry_handler)
def main(bot_class, logger, config):
# from here the environment is supposed to be set (daemon / non daemon,
# config.py in the python path )
from .utils import PLUGINS_SUBDIR
from . import holder
from .errBot import bot_config_defaults
bot_config_defaults(config)
if config.BOT_LOG_FILE:
hdlr = logging.FileHandler(config.BOT_LOG_FILE)
hdlr.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(hdlr)
if config.BOT_LOG_SENTRY:
try:
from raven.handlers.logging import SentryHandler
except ImportError as _:
log.exception(
"You have BOT_LOG_SENTRY enabled, but I couldn't import modules "
"needed for Sentry integration. Did you install raven? "
"(See http://raven.readthedocs.org/en/latest/install/index.html "
"for installation instructions)")
exit(-1)
sentryhandler = SentryHandler(config.SENTRY_DSN,
level=config.SENTRY_LOGLEVEL)
logger.addHandler(sentryhandler)
logger.setLevel(config.BOT_LOG_LEVEL)
# make the plugins subdir to store the plugin shelves
d = path.join(config.BOT_DATA_DIR, PLUGINS_SUBDIR)
if not path.exists(d):
makedirs(d, mode=0o755)
try:
holder.bot = bot_class(config)
except Exception:
log.exception(
"Unable to configure the backend, please check if your config.py is correct."
)
exit(-1)
errors = holder.bot.update_dynamic_plugins()
if errors:
log.error('Some plugins failed to load:\n' + '\n'.join(errors))
log.debug('serve from %s' % holder.bot)
holder.bot.serve_forever()
def register_logger_signal(client, logger=None, loglevel=logging.ERROR):
filter_ = CeleryFilter()
handler = SentryHandler(client)
handler.setLevel(loglevel)
handler.addFilter(filter_)
def process_logger_event(sender, logger, loglevel, logfile, format,
colorize, **kw):
for h in logger.handlers:
if type(h) == SentryHandler:
h.addFilter(filter_)
return False
logger.addHandler(handler)
after_setup_logger.connect(process_logger_event, weak=False)
def main():
import logging
logging.basicConfig(format=LOG_FORMAT)
logging.getLogger('werkzeug').setLevel(logging.INFO)
from hambar.app import create_app, create_manager
app = create_app()
SENTRY_DSN = app.config.get('SENTRY_DSN')
if SENTRY_DSN:
from raven.conf import setup_logging
from raven.handlers.logging import SentryHandler
setup_logging(SentryHandler(SENTRY_DSN, level=logging.WARN))
manager = create_manager(app)
manager.run()
def create_app():
app = Flask(__name__)
if config.is_prod_mode():
app.config.from_object('server.config.ProductionConfig')
else:
app.config.from_object('server.config.DevelopmentConfig')
# Set up sentry logging
if app.config['SENTRY_DSN'] is not None:
handler = SentryHandler(app.config['SENTRY_DSN'])
handler.setLevel(logging.ERROR)
setup_logging(handler)
Sentry(app, dsn=app.config['SENTRY_DSN'])
logger.info('Logging to Sentry')
else:
logger.info('No Sentry logging')
return app
def setup_sentry_client(app, loop):
"""设置 sentry client, 用于发送错误信息
如果在 DEBUG 状态下, 不使用 sentry
"""
if not app.config.DEBUG:
app.sentry = raven.Client(
dsn=app.config.SENTRY_DSN,
transport=raven_aiohttp.AioHttpTransport,
enable_breadcrumbs=False,
)
handler = SentryHandler(
client=app.sentry,
level=app.config.get('SENTRY_LEVEL', logging.ERROR))
logging.getLogger('asyncio').addHandler(handler)
logger.info('Sentry client prepared')
def setup_sentry():
sentry_url = get_ship_config().get('sentry_url', '')
tags = {'ship_IP': get_external_ip()}
sentry_client = Client(sentry_url,
include_paths=sentry_include_path,
release=__version__,
auto_log_stacks=True,
ignore_exceptions=sentry_ignore_exceptions,
tags=tags)
handler = SentryHandler(sentry_client, level=logging.WARNING)
setup_logging(handler)
return sentry_client
def check_args(self, args):
# check allow_other and allow_root
if args.allow_other:
args.allow_root = False
else:
args.allow_root = True
# check log_level
if args.debug:
args.log_level = 'debug'
# setup logging
if args.log != "syslog":
handler = TimedRotatingFileHandler(args.log, when="midnight")
handler.setFormatter(
Formatter(fmt='%(asctime)s %(threadName)s: '
'%(message)s',
datefmt='%B-%d-%Y %H:%M:%S'))
else:
if sys.platform == 'darwin':
handler = SysLogHandler(address="/var/run/syslog")
else:
handler = SysLogHandler(address="/dev/log")
logger_fmt = 'GitFS on {mount_point} [%(process)d]: %(threadName)s: '\
'%(message)s'.format(mount_point=args.mount_point)
handler.setFormatter(Formatter(fmt=logger_fmt))
if args.sentry_dsn != '':
from raven.conf import setup_logging
from raven.handlers.logging import SentryHandler
sentry_handler = SentryHandler(args.sentry_dsn)
sentry_handler.setLevel("ERROR")
setup_logging(sentry_handler)
log.addHandler(sentry_handler)
handler.setLevel(args.log_level.upper())
log.setLevel(args.log_level.upper())
log.addHandler(handler)
# set cache size
lru_cache.maxsize = args.cache_size
# return absolute repository's path
args.repo_path = os.path.abspath(args.repo_path)
return args
def init_app(self,
app,
dsn=None,
logging=None,
level=None,
wrap_wsgi=None,
register_signal=None):
if dsn is not None:
self.dsn = dsn
if level is not None:
self.level = level
if wrap_wsgi is not None:
self.wrap_wsgi = wrap_wsgi
else:
# Fix https://github.com/getsentry/raven-python/issues/412
# the gist is that we get errors twice in debug mode if we don't do this
if app and app.debug:
self.wrap_wsgi = False
else:
self.wrap_wsgi = True
if register_signal is not None:
self.register_signal = register_signal
if logging is not None:
self.logging = logging
if not self.client:
self.client = make_client(self.client_cls, app, self.dsn)
if self.logging:
setup_logging(SentryHandler(self.client, level=self.level))
if self.wrap_wsgi:
app.wsgi_app = SentryMiddleware(app.wsgi_app, self.client)
app.before_request(self.before_request)
if self.register_signal:
got_request_exception.connect(self.handle_exception, sender=app)
request_finished.connect(self.add_sentry_id_header, sender=app)
if not hasattr(app, 'extensions'):
app.extensions = {}
app.extensions['sentry'] = self
def setup_logger():
"""Set up our logger with sentry support.
"""
logger = logging.getLogger('osm-reporter')
logger.setLevel(logging.DEBUG)
default_handler_level = logging.DEBUG
# create formatter that will be added to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
temp_dir = '/tmp'
# so e.g. jenkins can override log dir.
if 'OSM_REPORTER_LOGFILE' in os.environ:
file_name = os.environ['OSM_REPORTER_LOGFILE']
else:
file_name = os.path.join(temp_dir, 'reporter.log')
file_handler = logging.FileHandler(file_name)
file_handler.setLevel(default_handler_level)
# create console handler with a higher log level
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.DEBUG)
try:
# pylint: disable=F0401
# noinspection PyUnresolvedReferences
from raven.handlers.logging import SentryHandler
# noinspection PyUnresolvedReferences
from raven import Client
# pylint: enable=F0401
client = Client(
'http://*****:*****@sentry.kartoza.com/25')
sentry_handler = SentryHandler(client)
sentry_handler.setFormatter(formatter)
sentry_handler.setLevel(logging.ERROR)
add_handler_once(logger, sentry_handler)
logger.debug('Sentry logging enabled')
except ImportError:
logger.debug('Sentry logging disabled. Try pip install raven')
# Set formatters
file_handler.setFormatter(formatter)
console_handler.setFormatter(formatter)
# add the handlers to the logger
add_handler_once(logger, file_handler)
add_handler_once(logger, console_handler)
def __init__(self, config=None):
self.config = {
'capture_timeout_warnings':
boolval(os.environ.get('SENTRY_CAPTURE_TIMEOUTS', True)),
'timeout_warning_threshold':
float(os.environ.get('SENTRY_TIMEOUT_THRESHOLD', 0.50)),
'capture_memory_warnings':
boolval(os.environ.get('SENTRY_CAPTURE_MEMORY', True)),
'memory_warning_threshold':
float(os.environ.get('SENTRY_MEMORY_THRESHOLD', 0.75)),
'capture_unhandled_exceptions':
boolval(os.environ.get('SENTRY_CAPTURE_UNHANDLED', True)),
'auto_bread_crumbs':
boolval(os.environ.get('SENTRY_AUTO_BREADCRUMBS', True)),
'capture_errors':
boolval(os.environ.get('SENTRY_CAPTURE_ERRORS', True)),
'filter_local':
boolval(os.environ.get('SENTRY_FILTER_LOCAL', True)),
'is_local':
os.environ.get('IS_OFFLINE', False)
or os.environ.get('IS_LOCAL', False),
'logging':
boolval(os.environ.get('SENTRY_CAPTURE_LOGS', True)),
'log_level':
extract_log_level_from_environment('SENTRY_LOG_LEVEL',
logging.WARNING),
'enabled':
boolval(os.environ.get('SENTRY_ENABLED', True)),
}
self.config.update(config or {})
# check for local environment
if self.config['filter_local'] and self.config['is_local']:
logger.warning('Sentry is disabled in local environment')
self.config["enabled"] = False
return
if self.config.get('raven_client'):
assert self.config.get('raven_client') and not isinstance(
self.config.get('raven_client'), Client)
else:
self.config['raven_client'] = configure_raven_client(self.config)
if self.config['logging'] and self.config['raven_client']:
handler = SentryHandler(self.config['raven_client'])
handler.setLevel(self.config['log_level'])
setup_logging(handler)
def setup_logger():
"""Set up our logger with sentry support.
"""
logger = logging.getLogger('osm-reporter')
logger.setLevel(logging.DEBUG)
default_handler_level = logging.DEBUG
# create formatter that will be added to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
temp_dir = '/tmp'
# so e.g. jenkins can override log dir.
if 'OSM_REPORTER_LOGFILE' in os.environ:
file_name = os.environ['OSM_REPORTER_LOGFILE']
else:
file_name = os.path.join(temp_dir, 'reporter.log')
file_handler = logging.FileHandler(file_name)
file_handler.setLevel(default_handler_level)
# create console handler with a higher log level
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.ERROR)
try:
# pylint: disable=F0401
# noinspection PyUnresolvedReferences
from raven.handlers.logging import SentryHandler
# noinspection PyUnresolvedReferences
from raven import Client
# pylint: enable=F0401
client = Client(
'http://*****:*****@sentry.linfiniti.com/6')
sentry_handler = SentryHandler(client)
sentry_handler.setFormatter(formatter)
sentry_handler.setLevel(logging.ERROR)
add_handler_once(logger, sentry_handler)
logger.debug('Sentry logging enabled')
except ImportError:
logger.debug('Sentry logging disabled. Try pip install raven')
# Set formatters
file_handler.setFormatter(formatter)
console_handler.setFormatter(formatter)
# add the handlers to the logger
add_handler_once(logger, file_handler)
add_handler_once(logger, console_handler)
def install_handler(self, app):
"""Install log handler."""
from raven.contrib.celery import register_logger_signal, \
register_signal
from raven.contrib.flask import Sentry
from raven.handlers.logging import SentryHandler
# Installs sentry in app.extensions['sentry']
level = getattr(logging, app.config['LOGGING_SENTRY_LEVEL'])
# Get the Sentry class.
cls = app.config['LOGGING_SENTRY_CLASS']
if cls:
if isinstance(cls, six.string_types):
cls = import_string(cls)
else:
cls = Sentry
sentry = cls(
app,
logging=True,
level=level
)
# Capture warnings from warnings module
if app.config['LOGGING_SENTRY_PYWARNINGS']:
self.capture_pywarnings(
SentryHandler(sentry.client))
# Setup Celery logging to Sentry
if app.config['LOGGING_SENTRY_CELERY']:
try:
register_logger_signal(sentry.client, loglevel=level)
except TypeError:
# Compatibility mode for Raven<=5.1.0
register_logger_signal(sentry.client)
register_signal(sentry.client)
# Werkzeug only adds a stream handler if there's no other handlers
# defined, so when Sentry adds a log handler no output is
# received from Werkzeug unless we install a console handler
# here on the werkzeug logger.
if app.debug:
logger = logging.getLogger('werkzeug')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())
def init_app(self, app, dsn=None):
self.app = app
if dsn is not None:
self.dsn = dsn
if not self.client:
self.client = make_client(self.client_cls, app, self.dsn)
if self.logging:
setup_logging(SentryHandler(self.client, level=self.level))
got_request_exception.connect(self.handle_exception, sender=app)
if not hasattr(app, 'extensions'):
app.extensions = {}
app.extensions['sentry'] = self
def init_unity(level, sentry_dsn, release):
root = logging.getLogger()
root.setLevel(0)
if sentry_dsn:
hdlr = SentryHandler(
raven.Client(sentry_dsn,
transport=GeventedHTTPTransport,
release=release))
hdlr.setLevel(logging.ERROR)
root.addHandler(hdlr)
hdlr = UnityLogHandler()
hdlr.setLevel(level)
root.addHandler(hdlr)
root.info(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
root.info('==============================================')
def setup_logging():
config = pluginapi.get_config('mediagoblin.plugins.raven')
conf_setup_logging = False
if config.get('setup_logging'):
conf_setup_logging = bool(int(config.get('setup_logging')))
if not conf_setup_logging:
return
from raven.handlers.logging import SentryHandler
from raven.conf import setup_logging
client = get_client()
_log.info('Setting up raven logging handler')
setup_logging(SentryHandler(client))
def run():
# Log everything to stderr.
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.INFO)
# Add Sentry (no-op if no configured).
handler = SentryHandler(sentry)
handler.setLevel(logging.ERROR)
logger.addHandler(handler)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main(loop))
except Exception:
logger.exception('Aborted.')
raise
finally:
loop.close()
def configure_logging(logger="respa_exchange", level=logging.INFO, handler=None):
logger = logging.getLogger(logger)
logger.setLevel(level)
if not handler:
handler = logging.StreamHandler(stream=sys.stdout)
handler.setFormatter(logging.Formatter(
fmt="%(asctime)s - %(name)s - %(levelname)s: %(message)s",
datefmt=logging.Formatter.default_time_format
))
logger.addHandler(handler)
if hasattr(settings, 'RAVEN_CONFIG') and 'dsn' in settings.RAVEN_CONFIG:
from raven.handlers.logging import SentryHandler
from raven.conf import setup_logging
sentry_handler = SentryHandler(settings.RAVEN_CONFIG['dsn'])
sentry_handler.setLevel(logging.ERROR)
logger.addHandler(sentry_handler)
setup_logging(sentry_handler)
def setup_sentry(app: Flask):
sentry_dsn = app.config.get('SENTRY_DSN')
if not sentry_dsn:
client = DummyClient()
else:
client = Client(
sentry_dsn,
transport=GeventedHTTPTransport,
release=app.config.get('CURRENT_RELEASE'),
environment=app.config.get('CURRENT_ENVIRONMENT'),
)
sentry = Sentry(client=client, logging=True, level=logging.WARNING)
handler = SentryHandler(client, level=logging.WARNING)
setup_sentry_logging(handler)
sentry.init_app(app)
return sentry, client
def create_app(load_admin=True):
from redash import handlers
from redash.admin import init_admin
from redash.models import db
from redash.authentication import setup_authentication
from redash.metrics.request import provision_app
app = Flask(__name__,
template_folder=settings.STATIC_ASSETS_PATHS[-1],
static_folder=settings.STATIC_ASSETS_PATHS[-1],
static_path='/static')
# Make sure we get the right referral address even behind proxies like nginx.
app.wsgi_app = ProxyFix(app.wsgi_app, settings.PROXIES_COUNT)
app.url_map.converters['org_slug'] = SlugConverter
if settings.ENFORCE_HTTPS:
SSLify(app, skips=['ping'])
if settings.SENTRY_DSN:
from raven.contrib.flask import Sentry
from raven.handlers.logging import SentryHandler
sentry = Sentry(app, dsn=settings.SENTRY_DSN)
sentry.client.release = __version__
sentry_handler = SentryHandler(settings.SENTRY_DSN)
sentry_handler.setLevel(logging.ERROR)
logging.getLogger().addHandler(sentry_handler)
# configure our database
app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI
app.config.update(settings.all_settings())
provision_app(app)
db.init_app(app)
migrate.init_app(app, db)
if load_admin:
init_admin(app)
mail.init_app(app)
setup_authentication(app)
limiter.init_app(app)
handlers.init_app(app)
return app
def configure_logging(config):
"""
Allow some basic logging configuration to be read from the cherrpy
config.
"""
# PasteScript will have already configured the logger if the appropriate
# sections were found in the config file, so we do nothing if the
# config has a loggers section, otherwise we do some simple setup
# using the 'log_*' values from the config.
if config.global_conf_parser.has_section("loggers"):
return
format = config.get("log_format",
"%(name)s %(levelname)s %(asctime)s %(message)s")
level = logging._levelNames[config.get("log_level", "DEBUG")]
destination = config.get("log_destination", "stdout")
log.info("Logging at '%s' level to '%s'" % (level, destination))
# Get root logger
root = logging.getLogger()
# Set level
root.setLevel(level)
# Turn down paste httpserver logging
if level <= logging.DEBUG:
logging.getLogger("paste.httpserver.ThreadPool").setLevel(logging.WARN)
# Remove old handlers
for h in root.handlers[:]:
root.removeHandler(h)
# Create handler
if destination == "stdout":
handler = logging.StreamHandler(sys.stdout)
else:
handler = logging.FileHandler(destination)
# Create formatter
formatter = logging.Formatter(format)
# Hook everything up
handler.setFormatter(formatter)
root.addHandler(handler)
# If sentry is configured, also log to it
if config.sentry_dsn:
eggs.require("raven")
from raven.handlers.logging import SentryHandler
sentry_handler = SentryHandler(config.sentry_dsn)
sentry_handler.setLevel(logging.WARN)
root.addHandler(sentry_handler)
def init_logging():
root = logging.getLogger()
root.setLevel(0)
if State.config['sentry']:
patch_gevent_hub_print_exception()
hdlr = SentryHandler(
raven.Client(State.config['sentry'],
transport=GeventedHTTPTransport))
hdlr.setLevel(logging.ERROR)
root.addHandler(hdlr)
hdlr = logging.StreamHandler(sys.stdout)
hdlr.setLevel(getattr(logging, logging.DEBUG))
root.addHandler(hdlr)
root.info(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"))
root.info('==============================================')
def lambda_handler(event, context):
# Log everything to stderr.
logger.addHandler(logging.StreamHandler(stream=sys.stdout))
logger.setLevel(logging.DEBUG)
# Add Sentry (no-op if no configured).
handler = SentryHandler(sentry)
handler.setLevel(logging.ERROR)
logger.addHandler(handler)
loop = asyncio.get_event_loop_policy().new_event_loop()
try:
loop.run_until_complete(main(loop, event))
except Exception:
logger.exception('Aborted.')
raise
finally:
loop.close()
def get_app():
app = Flask("SwissVoice")
log.debug("Loading config")
app.config.from_object(config.default)
app.config.from_object(config.environment)
log.debug("setting up Sentry...")
sentry_client = raven.Client(app.config["SENTRY_DSN"], release=__version__)
Sentry(app, sentry_client)
sentry_handler = SentryHandler(sentry_client)
sentry_handler.setLevel(logging.ERROR)
setup_logging(sentry_handler)
register_converters(app)
app.json_encoder = SwissJsonEncoder
log.info("app setup!")
return app
def lambda_handler(event=None, context=None):
# Log everything to stderr.
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.DEBUG)
# Add Sentry (no-op if no configured).
handler = SentryHandler(sentry)
handler.setLevel(logging.ERROR)
logger.addHandler(handler)
loop = asyncio.get_event_loop()
futures = [main(loop, inventory) for inventory in ('firefox', 'archive')]
try:
loop.run_until_complete(asyncio.gather(*futures))
except Exception:
logger.exception('Aborted.')
raise
finally:
loop.close()
