def init_logging_syslog(log_level='notset', facility='local0', bubble=True):
handler = logbook.SyslogHandler('postgresql-metrics',
facility=facility,
level=figure_out_log_level(log_level),
bubble=bubble)
handler.push_application()
get_logger().debug("syslog logging initialized")
def test_syslog_handler(logger, activation_strategy, sock_family, socktype, address, app_name):
delimiter = DELIMITERS[sock_family]
with closing(socket.socket(sock_family, socktype)) as inc:
inc.bind(address)
if socktype == socket.SOCK_STREAM:
inc.listen(0)
inc.settimeout(1)
if UNIX_SOCKET_AVAILABLE and sock_family == socket.AF_UNIX:
expected = (r'^<12>%stestlogger: Syslog is weird%s$' % (app_name + ':' if app_name else '', delimiter))
else:
expected = (r'^<12>1 \d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?Z %s %s %d - - %sSyslog is weird%s$' % (
socket.gethostname(),
app_name if app_name else 'testlogger',
os.getpid(), 'testlogger: ' if app_name else '',
delimiter))
handler = logbook.SyslogHandler(app_name, inc.getsockname(), socktype=socktype)
with activation_strategy(handler):
logger.warn('Syslog is weird')
if socktype == socket.SOCK_STREAM:
with closing(inc.accept()[0]) as inc2:
rv = inc2.recv(1024)
else:
rv = inc.recvfrom(1024)[0]
rv = rv.decode('utf-8')
assert re.match(expected, rv), \
'expected {}, got {}'.format(expected, rv)
def setup_log(**args):
logbook.SyslogHandler().push_application()
logbook.StreamHandler(sys.stderr, bubble=True).push_application()
redirect_stdouts_to_logger(args['logger']) # logs to local syslog
if os.path.exists('/dev/log'):
h = logging.handlers.SysLogHandler('/dev/log')
else:
h = logging.handlers.SysLogHandler()
h.setLevel(args['loglevel'])
formatter = logging.Formatter(logging.BASIC_FORMAT)
h.setFormatter(formatter)
args['logger'].addHandler(h)
def setup_papertrail(project_name, channel, PAPERTRAIL_HOST, PAPERTRAIL_PORT):
"""
Setup papertrail account using taskcluster secrets
"""
# Setup papertrail
papertrail = logbook.SyslogHandler(
application_name=f"code-review/{channel}/{project_name}",
address=(PAPERTRAIL_HOST, int(PAPERTRAIL_PORT)),
level=logbook.INFO,
format_string="{record.time} {record.channel}: {record.message}",
bubble=True,
)
papertrail.push_application()
def setup_papertrail(project_name, channel, PAPERTRAIL_HOST, PAPERTRAIL_PORT):
'''
Setup papertrail account using taskcluster secrets
'''
# Setup papertrail
papertrail = logbook.SyslogHandler(
application_name='mozilla/release-services/{}/{}'.format(
channel, project_name),
address=(PAPERTRAIL_HOST, int(PAPERTRAIL_PORT)),
format_string='{record.time} {record.channel}: {record.message}',
bubble=True,
)
papertrail.push_application()
def run(config_file):
error_handler = logbook.SyslogHandler('pushjournal', level='INFO')
error_handler.push_application()
app_config = config.load(config_file)
app_notifiers = notifiers.create_notifiers(app_config)
reader = journal.Reader()
boot_file = os.path.join(tempfile.gettempdir(), ".pushjournal-boot")
if app_config.get("notify_boot", False) and not os.path.isfile(boot_file):
body = ""
if app_config.get("show_public_ip", False):
try:
body += "Public IP: {}\n\n".format(_get_public_ip())
except Exception:
logbook.error("Could not get the public IP: {}",
traceback.format_exc())
body += "Failed to get the public IP\n"
if app_config.get("show_local_ips", False):
try:
body += "Local IPs:\n{}\n\n".format("\n".join(
"- {}: {}".format(*addr) for addr in _get_local_ips()))
except Exception:
logbook.error("Could not get local IPs: {}",
traceback.format_exc())
body += "Failed to get local IPs\n"
_notify(app_notifiers, "System booted", body, True)
with open(boot_file, "wb"):
pass
for f in app_config['filters']:
f['match'] = re.compile(f['match'])
reader.seek_tail()
while True:
reader.wait()
for entry in reader:
for f in app_config['filters']:
m = f['match'].search(entry['MESSAGE'])
if not m:
continue
_notify(app_notifiers, f['title'].format(*m.groups()),
f['body'].format(*m.groups()), False)
def test_syslog_handler(logger, activation_strategy, unix_sock_path):
to_test = [
(socket.AF_INET, ('127.0.0.1', 0)),
]
if hasattr(socket, 'AF_UNIX'):
to_test.append((socket.AF_UNIX, unix_sock_path))
for sock_family, address in to_test:
with closing(socket.socket(sock_family, socket.SOCK_DGRAM)) as inc:
inc.bind(address)
inc.settimeout(1)
for app_name in [None, 'Testing']:
handler = logbook.SyslogHandler(app_name, inc.getsockname())
with activation_strategy(handler):
logger.warn('Syslog is weird')
try:
rv = inc.recvfrom(1024)[0]
except socket.error:
assert False, 'got timeout on socket'
assert rv == (u('<12>%stestlogger: Syslog is weird\x00') %
((app_name and (app_name + u(':'))) or u(''))).encode('utf-8')
def test_syslog_handler(self):
to_test = [
(socket.AF_INET, ('127.0.0.1', 0)),
]
if hasattr(socket, 'AF_UNIX'):
to_test.append((socket.AF_UNIX, self.filename))
for sock_family, address in to_test:
inc = socket.socket(sock_family, socket.SOCK_DGRAM)
inc.bind(address)
inc.settimeout(1)
for app_name in [None, 'Testing']:
handler = logbook.SyslogHandler(app_name, inc.getsockname())
with handler:
self.log.warn('Syslog is weird')
try:
rv = inc.recvfrom(1024)[0]
except socket.error:
self.fail('got timeout on socket')
self.assertEqual(rv, (
u'<12>%stestlogger: Syslog is weird\x00' %
(app_name and app_name + u':' or u'')).encode('utf-8'))
def main(args):
if not args.debug:
for handler in [logbook.SyslogHandler(level=logbook.INFO)]:
handler.format_string = '{record.thread}|{record.channel}: {record.message}'
handler.push_application()
with closing(socket.socket()) as s:
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(("0.0.0.0", args.port))
logbook.debug("Listening on port {}", args.port)
s.listen(socket.SOMAXCONN)
message_sink = DatabaseMessageSink()
semaphore = get_semaphore()
while True:
try:
p, a = s.accept()
except KeyboardInterrupt:
break
logbook.debug("Incoming connection from {}", a)
thread = SMTPServerThread(p, message_sink, semaphore)
thread.daemon = True
thread.start()
def configure_log_handler(application_name, loglevel, output):
if isinstance(loglevel, (str, unicode)):
loglevel = getattr(logbook, loglevel.upper())
if not isinstance(loglevel, int):
raise TypeError(
"configure_log_handler expects loglevel to be either an integer or a string corresponding to an integer attribute of the logbook module."
)
if output == 'syslog':
log_handler = logbook.SyslogHandler(application_name=application_name,
facility='user',
bubble=False,
level=loglevel)
elif output == '-' or not output:
log_handler = logbook.StderrHandler(level=loglevel, bubble=False)
else:
log_handler = logbook.FileHandler(filename=output,
encoding='utf-8',
level=loglevel,
bubble=False)
return log_handler
def test_syslog_handler(logger, activation_strategy, sock_family, socktype,
address):
delimiter = {socket.AF_UNIX: '\x00', socket.AF_INET: '\n'}[sock_family]
with closing(socket.socket(sock_family, socktype)) as inc:
inc.bind(address)
if socktype == socket.SOCK_STREAM:
inc.listen(0)
inc.settimeout(1)
for app_name in [None, 'Testing']:
if sock_family == socket.AF_UNIX:
expected = (r'^<12>%stestlogger: Syslog is weird%s$' %
(app_name + ':' if app_name else '', delimiter))
else:
expected = (
r'^<12>1 \d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?Z %s %s %d '
'- - %sSyslog is weird%s$' %
(socket.gethostname(),
app_name if app_name else 'testlogger', os.getpid(),
'testlogger: ' if app_name else '', delimiter))
handler = logbook.SyslogHandler(app_name,
inc.getsockname(),
socktype=socktype)
with activation_strategy(handler):
logger.warn('Syslog is weird')
try:
if socktype == socket.SOCK_STREAM:
with closing(inc.accept()[0]) as inc2:
rv = inc2.recv(1024)
else:
rv = inc.recvfrom(1024)[0]
except socket.error:
assert False, 'got timeout on socket'
rv = rv.decode('utf-8')
assert re.match(expected, rv), \
'expected {}, got {}'.format(expected, rv)
def create_app(config=None):
if config is None:
config = {}
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
app = flask.Flask(__name__,
static_folder=os.path.join(ROOT_DIR, "..", "static"))
_CONF_D_PATH = os.environ.get('CONFIG_DIRECTORY',
os.path.join(ROOT_DIR, "..", "..", "conf.d"))
configs = [os.path.join(ROOT_DIR, "app.yml")]
if os.path.isdir(_CONF_D_PATH):
configs.extend(
sorted(
os.path.join(_CONF_D_PATH, x) for x in os.listdir(_CONF_D_PATH)
if x.endswith(".yml")))
configs.append(os.path.expanduser('~/.config/backslash/devconfig.yml'))
for yaml_path in configs:
if os.path.isfile(yaml_path):
with open(yaml_path) as yaml_file:
app.config.update(yaml.load(yaml_file))
app.config.update(config)
if 'SQLALCHEMY_DATABASE_URI' not in app.config:
app.config['SQLALCHEMY_DATABASE_URI'] = os.path.expandvars(
os.environ.get(
'SQLALCHEMY_DATABASE_URI',
'postgresql://localhost/{0}'.format(app.config['app_name'])))
if os.path.exists("/dev/log"):
syslog_handler = logbook.SyslogHandler(app.config['app_name'],
"/dev/log")
syslog_handler.push_application()
del app.logger.handlers[:]
redirect_logging()
if app.config['TESTING']:
app.config['TRACEBACK_DIR'] = '/tmp/backslash_tracebacks'
else:
_disable_logs(['dogpile.lock'])
app.logger.info("Started")
Mail(app)
from . import models
from .blueprints import rest, views, runtoken
from .blueprints.api.main import blueprint as api_blueprint
models.db.init_app(app)
from . import auth
Security(app, auth.user_datastore, register_blueprint=False)
blueprints = [
auth.auth, views.blueprint, api_blueprint, rest.blueprint,
runtoken.blueprint
]
from .errors import errors
for blueprint in blueprints:
app.register_blueprint(blueprint)
for code in errors:
app.errorhandler(code)(errors[code])
return app
# Logging configuration
try:
import logbook
import logbook.compat
from client_logging import util as cl_util
logbook.compat.redirect_logging()
null_handler = logbook.NullHandler()
if DEBUG:
log_handler = logbook.StderrHandler(level=logbook.WARNING)
if CLIENT_EXCEPTION_LOGGING:
js_log_handler = logbook.StderrHandler(filter=cl_util.js_error_log_filter, level=logbook.INFO)
js_log_handler.formatter = cl_util.js_error_formatter_stderr
else:
log_handler = logbook.SyslogHandler(level=logbook.WARNING)
if CLIENT_EXCEPTION_LOGGING:
js_log_handler = logbook.SyslogHandler(filter=cl_util.js_error_log_filter, level=logbook.INFO)
js_log_handler.formatter = cl_util.js_error_formatter_syslog
null_handler.push_application()
log_handler.push_application()
if CLIENT_EXCEPTION_LOGGING:
js_log_handler.push_application()
except ImportError:
pass
CLIENT_EXCEPTION_RATE = env('CLIENT_EXCEPTION_RATE', "20/hour")
logging.getLogger('suds.client').setLevel(logging.ERROR)
import logbook
import os
import requests
from RPi.GPIO import cleanup
from lib.scale import BeeHiveScale
from lib.dht import DHTSensorController
import settings
secrets = json.load(
open(os.path.join(os.path.dirname(__file__), "secrets.json"), "r"))
mail_log_settings = secrets["logging"]
logger = logbook.NestedSetup([
logbook.NullHandler(),
logbook.SyslogHandler(level=logbook.INFO),
logbook.MailHandler(from_addr=mail_log_settings["sender"],
recipients=mail_log_settings["recipients"],
level=logbook.ERROR,
credentials=(mail_log_settings["username"],
mail_log_settings["password"]),
server_addr=(mail_log_settings["server"],
mail_log_settings["port"]),
secure=mail_log_settings["secure"],
bubble=True)
])
def acquire_weights(scale_obj,
sleep_time=settings.WEIGHT_MEASURE_WAIT_TIME,
measure_count=settings.WEIGHT_MEASURE_COUNT):
host = value
if option in ("-p", "--port"):
port = int(value)
if option in ("-d", "--delay"):
delay = float(value)
if option in ("-t", "--tubes"):
tubes = re.split(r'\s*,\s*', value)
except Usage, err:
print >> sys.stderr, sys.argv[0].split("/")[-1] + ": " + str(err.msg)
print >> sys.stderr, "\t for help use --help"
return 2
if output == 'syslog':
log_handler = logbook.SyslogHandler(
application_name='beanstalk-dumper',
bubble=False,
level=loglevel)
elif output == '-' or not output:
log_handler = logbook.StderrHandler(
level=loglevel,
bubble=False)
else:
log_handler = logbook.FileHandler(
filename=output,
encoding='utf-8',
level=loglevel,
bubble=False)
with logbook.NullHandler():
with log_handler.applicationbound():
log.info("Starting Beanstalk dumper...")
def create_app(config=None):
if config is None:
config = {}
# Defaults
config.update({
'MAX_MAILBOX_AGE_SECONDS': 7 * 24 * 60 * 60, # 1 week
})
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
app = flask.Flask(__name__,
static_folder=os.path.join(ROOT_DIR, "..", "static"))
_CONF_D_PATH = os.environ.get('CONFIG_DIRECTORY',
os.path.join(ROOT_DIR, "..", "..", "conf.d"))
configs = [os.path.join(ROOT_DIR, "app.yml")]
if os.path.isdir(_CONF_D_PATH):
configs.extend(
sorted(
os.path.join(_CONF_D_PATH, x) for x in os.listdir(_CONF_D_PATH)
if x.endswith(".yml")))
for yaml_path in configs:
if os.path.isfile(yaml_path):
with open(yaml_path) as yaml_file:
app.config.update(yaml.load(yaml_file))
app.config.update(config)
if 'SQLALCHEMY_DATABASE_URI' not in app.config:
app.config['SQLALCHEMY_DATABASE_URI'] = os.path.expandvars(
os.environ.get(
'SQLALCHEMY_DATABASE_URI',
'postgresql://localhost/{0}'.format(app.config['app_name'])))
if os.path.exists("/dev/log"):
syslog_handler = logbook.SyslogHandler(app.config['app_name'],
"/dev/log")
syslog_handler.push_application()
del app.logger.handlers[:]
redirect_logging()
Mail(app)
from . import models
from . import errors
models.db.init_app(app)
from . import auth
Security(app, auth.user_datastore, register_blueprint=False)
from .errors import errors
from .auth import auth
from .views import views
from .setup import setup
blueprints = [auth, views, setup]
for blueprint in blueprints:
app.register_blueprint(blueprint)
from .api import blueprint as api_blueprint
app.register_blueprint(api_blueprint, url_prefix="/v2")
for code in errors:
app.errorhandler(code)(errors[code])
return app
def create_app(config=None):
if config is None:
config = {}
ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
app = flask.Flask(__name__,
static_folder=os.path.join(ROOT_DIR, "..", "static"))
_CONF_D_PATH = os.environ.get('CONFIG_DIRECTORY',
os.path.join(ROOT_DIR, "..", "..", "conf.d"))
configs = [os.path.join(ROOT_DIR, "app.yml")]
if os.path.isdir(_CONF_D_PATH):
configs.extend(
sorted(
os.path.join(_CONF_D_PATH, x) for x in os.listdir(_CONF_D_PATH)
if x.endswith(".yml")))
for yaml_path in configs:
if os.path.isfile(yaml_path):
with open(yaml_path) as yaml_file:
app.config.update(yaml.load(yaml_file))
app.config.update(config)
if 'SQLALCHEMY_DATABASE_URI' not in app.config:
app.config['SQLALCHEMY_DATABASE_URI'] = os.path.expandvars(
os.environ.get(
'SQLALCHEMY_DATABASE_URI',
'postgresql://localhost/{0}'.format(app.config['app_name'])))
if os.path.exists("/dev/log"):
syslog_handler = logbook.SyslogHandler(app.config['app_name'],
"/dev/log")
syslog_handler.push_application()
del app.logger.handlers[:]
redirect_logging()
app.logger.info("Started")
Mail(app)
from . import models
models.db.init_app(app)
from . import auth
from .blueprints import investigations, entities, events, roles, users, event_types
Security(app, auth.user_datastore, register_blueprint=False)
from .auth import auth
from .views import views
from .setup import setup
app.register_blueprint(auth)
app.register_blueprint(views)
app.register_blueprint(setup)
app.register_blueprint(investigations, url_prefix="/investigations")
app.register_blueprint(entities, url_prefix="/entities")
app.register_blueprint(events, url_prefix="/events")
app.register_blueprint(roles, url_prefix="/roles")
app.register_blueprint(users, url_prefix="/users")
app.register_blueprint(event_types, url_prefix="/event-types")
from .errors import errors
for code in errors:
app.errorhandler(code)(errors[code])
return app
