def setup_logging():
os.makedirs(LOG_DIR, exist_ok=True)
format_string = "[{record.time:%H:%M:%S}] {record.level_name}: {record.channel}:{record.extra[strat_id]} {record.message}"
handlers = [logbook.NullHandler()]
if CLOUD_LOGGING:
cloud_handler = GoogleCloudHandler(level="DEBUG",
bubble=True,
format_string=format_string)
handlers.append(cloud_handler)
file_handler = logbook.RotatingFileHandler(APP_LOG,
level="DEBUG",
bubble=True,
format_string=format_string)
stream_handler = logbook.StreamHandler(sys.stdout,
level="INFO",
bubble=True)
stream_handler.format_string = format_string
error_file_handler = logbook.RotatingFileHandler(ERROR_LOG,
level="ERROR",
bubble=True)
error_file_handler.format_string = """
----------------------------------------------------------------------------------
{record.time:%H:%M:%S} KRYPTOS:{record.channel}:{record.level_name}:
{record.message}
Module: {record.module}:{record.lineno}
Function: {record.func_name}
Channel: {record.channel}
Trade Date: {record.extra[strat_date]}
Exception: {record.formatted_exception}
----------------------------------------------------------------------------------
"""
handlers.extend([file_handler, stream_handler, error_file_handler])
setup = logbook.NestedSetup(handlers)
setup.push_thread()
def _setup_file_handler(self):
"""
This helper method sets up a rotating file logger based on the base
path supplied in the configuration. No file logging is performed if
the base path is not supplied
"""
log_path_base = self.app.config.get('LOGGER_PATH_BASE')
if log_path_base is None:
return
log_path_base = os.path.expanduser(log_path_base)
if not os.path.exists(log_path_base):
os.makedirs(log_path_base)
log_file_name = self.app.config.get('LOGGER_FILE_NAME')
if log_file_name is None:
log_file_name = os.path.basename(sys.argv[0])
level_name = self.app.config.get('LOGGER_LEVEL', 'INFO')
logbook.RotatingFileHandler(
os.path.join(log_path_base, log_file_name + '.log'),
level=logbook.lookup_level(level_name),
bubble=True,
format_string=self.app.config.get('LOGGER_FORMAT'),
).push_application()
def test_rotating_file_handler(logfile, activation_strategy, logger):
basename = os.path.basename(logfile)
handler = logbook.RotatingFileHandler(
logfile,
max_size=2048,
backup_count=3,
)
handler.format_string = '{record.message}'
with activation_strategy(handler):
for c, x in zip(LETTERS, xrange(32)):
logger.warn(c * 256)
files = [
x for x in os.listdir(os.path.dirname(logfile))
if x.startswith(basename)
]
files.sort()
assert files == [
basename, basename + '.1', basename + '.2', basename + '.3'
]
with open(logfile) as f:
assert f.readline().rstrip() == ('C' * 256)
assert f.readline().rstrip() == ('D' * 256)
assert f.readline().rstrip() == ('E' * 256)
assert f.readline().rstrip() == ('F' * 256)
def init_logging_file(filename, log_level='notset', rotate_log=True, rotate_max_size=10485760):
log_dir = os.path.dirname(filename)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if rotate_log is True:
handler = logbook.RotatingFileHandler(filename, level=figure_out_log_level(log_level),
max_size=int(rotate_max_size), bubble=True)
else:
handler = logbook.FileHandler(filename, level=figure_out_log_level(log_level), bubble=True)
handler.push_application()
get_logger().debug("file based logging initialized in directory: " + log_dir)
def _get_log_handlers():
"""
Initializes all relevant log handlers.
:return: A list of log handlers.
"""
return [
logbook.NullHandler(),
logbook.StreamHandler(sys.stdout, level=logbook.DEBUG, bubble=True),
logbook.RotatingFileHandler(config.LOGFILE, level=logbook.DEBUG, max_size=5 * 1024 * 1024, bubble=True)
]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', default='0.0.0.0')
parser.add_argument('--port', default=8080)
parser.add_argument('--database', default='rss.db')
parser.add_argument('--logfile', default='webapp.log')
parser.add_argument('-d', '--debug', action='store_true')
args = parser.parse_args()
log_handler = logbook.RotatingFileHandler(
args.logfile, level=logbook.DEBUG if args.debug else logbook.INFO)
with log_handler.applicationbound():
webapp = WebApp(args.hostname, args.port, args.database, args.debug)
webapp.start()
def get_logger(perform_rollover=False):
"""
Push to the app stack the needed handlers and return a Logger object.
:rtype: logbook.Logger
"""
# NOTE: make sure that the folder exists, the logger is created before
# saving settings on the first run.
_base = os.path.join(get_path_prefix(), "leap")
mkdir_p(_base)
bitmask_log_file = os.path.join(_base, 'bitmask.log')
level = logbook.WARNING
if flags.DEBUG:
level = logbook.NOTSET
# This handler consumes logs not handled by the others
null_handler = logbook.NullHandler()
null_handler.push_application()
silencer = SelectiveSilencerFilter()
zmq_handler = SafeZMQHandler('tcp://127.0.0.1:5000', multi=True,
level=level, filter=silencer.filter)
zmq_handler.push_application()
file_handler = logbook.RotatingFileHandler(
bitmask_log_file, format_string=LOG_FORMAT, bubble=True,
filter=silencer.filter, max_size=sys.maxint)
if perform_rollover:
file_handler.perform_rollover()
file_handler.push_application()
# don't use simple stream, go for colored log handler instead
# stream_handler = logbook.StreamHandler(sys.stdout,
# format_string=LOG_FORMAT,
# bubble=True)
# stream_handler.push_application()
stream_handler = ColorizedStderrHandler(
level=level, format_string=LOG_FORMAT, bubble=True,
filter=silencer.filter)
stream_handler.push_application()
logger = logbook.Logger('leap')
return logger
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--loop', action='store_true')
parser.add_argument('--interval', default=60 * 30, type=float)
parser.add_argument('--database', default='rss.db')
parser.add_argument('--logfile', default='crawler.log')
parser.add_argument('-d', '--debug', action='store_true')
args = parser.parse_args()
log_handler = logbook.RotatingFileHandler(
args.logfile, level=logbook.DEBUG if args.debug else logbook.INFO)
with log_handler.applicationbound():
if args.loop:
crawler = LoopCrawler(args.interval, args.database, args.debug)
else:
crawler = Crawler(args.database, args.debug)
crawler.start()
def _get_log_handlers():
"""
Initializes all relevant log handlers.
:return: A list of log handlers.
"""
handlers = [
logbook.StreamHandler(sys.stdout, level=logbook.INFO, bubble=True),
]
if config.LOG_FILE_PATH:
handlers.append(
logbook.RotatingFileHandler(config.LOG_FILE_PATH,
level=logbook.DEBUG,
backup_count=1,
max_size=5 * 1024 * 1024,
bubble=True))
return handlers
def test_rotating_file_handler(self):
basename = os.path.join(self.dirname, 'rot.log')
handler = logbook.RotatingFileHandler(basename, max_size=2048,
backup_count=3,
)
handler.format_string = '{record.message}'
with handler:
for c, x in izip(LETTERS, xrange(32)):
self.log.warn(c * 256)
files = [x for x in os.listdir(self.dirname)
if x.startswith('rot.log')]
files.sort()
self.assertEqual(files, ['rot.log', 'rot.log.1', 'rot.log.2',
'rot.log.3'])
with open(basename) as f:
self.assertEqual(f.readline().rstrip(), 'C' * 256)
self.assertEqual(f.readline().rstrip(), 'D' * 256)
self.assertEqual(f.readline().rstrip(), 'E' * 256)
self.assertEqual(f.readline().rstrip(), 'F' * 256)
def _get_log_handlers(logs_directory_path=None):
"""
Returns a list of the nested log handlers setup.
"""
handlers_list = list()
handlers_list.append(logbook.NullHandler())
# Add the rotating file handler, if a logs directory path was supplied.
if logs_directory_path is not None:
if not os.path.exists(logs_directory_path):
os.makedirs(logs_directory_path)
handlers_list.append(
logbook.RotatingFileHandler(os.path.join(logs_directory_path,
LOG_FILE_NAME),
max_size=1024 * 1024,
backup_count=5,
bubble=True))
handlers_list.append(
logbook.StreamHandler(sys.stdout, level='DEBUG', bubble=True))
handlers_list.append(
logbook.StreamHandler(sys.stderr, level='ERROR', bubble=True))
return handlers_list
def post_fork(server, worker):
server.log.info('Worker spawned (pid: %s)', worker.pid)
logging_rotating_file_handler = logging.handlers.RotatingFileHandler(
config.LOG_FILE_PATH.replace('.log', f'.{worker.pid}.flask.log'),
maxBytes=5 * 1024 * 1024,
backupCount=5)
root_logger = logging.getLogger()
root_logger.addHandler(logging_rotating_file_handler)
root_logger.setLevel(logging.CRITICAL)
logger_setup = logbook.NestedSetup([
logbook.StreamHandler(sys.stdout, level=logbook.INFO, bubble=True),
logbook.RotatingFileHandler(config.LOG_FILE_PATH.replace(
'.log', f'.{worker.pid}.log'),
level=logbook.INFO,
max_size=5 * 1024 * 1024,
bubble=True)
])
logger_setup.push_application()
def get_logger(perform_rollover=False):
"""
Push to the app stack the needed handlers and return a Logger object.
:rtype: logbook.Logger
"""
# NOTE: make sure that the folder exists, the logger is created before
# saving settings on the first run.
_base = os.path.join(get_path_prefix(), "leap")
mkdir_p(_base)
bitmask_log_file = os.path.join(_base, 'bitmask.log')
# level = logbook.WARNING
# if flags.DEBUG:
# level = logbook.NOTSET
level = logbook.NOTSET
# This handler consumes logs not handled by the others
null_handler = logbook.NullHandler()
null_handler.push_application()
file_handler = logbook.RotatingFileHandler(bitmask_log_file,
format_string=LOG_FORMAT,
bubble=True,
max_size=sys.maxint)
if perform_rollover:
file_handler.perform_rollover()
file_handler.push_application()
stream_handler = ColorizedStderrHandler(level=level,
format_string=LOG_FORMAT,
bubble=True)
stream_handler.push_application()
logger = logbook.Logger('leap')
return logger
import sys
import logbook
from logbook import Logger
from pyexpander import config
file_handler = logbook.RotatingFileHandler(config.LOGFILE, level=logbook.DEBUG)
console_handler = logbook.StreamHandler(sys.stdout, level=logbook.INFO, bubble=True)
file_handler.push_application()
console_handler.push_application()
def get_logger(name):
"""
Return the logger for the given name.
:param name: The name of the logger.
:return: A logbook Logger.
"""
logger = Logger(name, level=logbook.DEBUG)
return logger
# -*- coding: utf-8 -*-
""" local logger """
import logbook
from conf import Conf
conf = Conf()
logbook.set_datetime_format(conf.datetime_format)
log = logbook.RotatingFileHandler(conf.logfile, max_size = conf.max_size, \
backup_count = conf.backup_count,\
level = conf.level, \
bubble=True
)
#print dir(log)
log.format_string = "[{record.time:%Y-%m-%d %H:%M:%S.%f}][{record.thread},{record.module},{record.func_name},{record.lineno}] {record.level_name}: {record.channel}: {record.message}"
log.default_format_string = "[{record.time:%Y-%m-%d %H:%M:%S.%f}][{record.thread},{record.module},{record.func_name},{record.lineno}] {record.level_name}: {record.channel}: {record.message}"
log.push_application()
def get_logger(logger_name):
""" get logger """
logger = logbook.Logger(logger_name)
#print dir(logger)
return logger
def main():
with spin_consumers(), \
logbook.RotatingFileHandler(DEFAULT_LOG_PATH).applicationbound(), \
logbook.StreamHandler(sys.stdout).applicationbound():
app.run()
"""
import logbook
import socket
import gevent
import toml
logbook.set_datetime_format("local")
logger = logbook.Logger('hekac')
#log = logbook.FileHandler('heka_tcp.log')
log = logbook.RotatingFileHandler('heka_tcp.log', max_size=1024, backup_count=5)
log.push_application()
def get_conf(conf_fn):
""" get configuration from .toml"""
with open(conf_fn) as conf_fh:
conf = toml.loads(conf_fh.read())
#print(config)
return conf
import api
import logbook
from webbrowser import open as web_open
logbook.RotatingFileHandler('my_search_api',
level=logbook.TRACE).push_application()
logger = logbook.Logger("Main")
def main():
logger.info('Taking user input...')
user_input = input('Enter search criteria: ')
logger.info(f'Searching for {user_input}')
articles = api.search(user_input)
print(f'Found {len(articles)} articles')
id_to_article_dict = {}
for r in articles:
id_to_article_dict[r.id] = r
print(f"[{r.id}] {r.title}: {r.category}")
print("*" * 20)
user_input = input(
'For which article do you need more information [q for quit]: ')
if user_input != 'q':
web_open("http://talkpython.fm" +
id_to_article_dict[int(user_input)].url,
new=2)
logger.info('End of execution')
from tornado import tcpserver
from tornado import gen
from configs import db as m_configs_db
from configs import logs as m_configs_logs
from configs.codec import *
import drivers
from drivers import redis as m_d_redis
from drivers import mysql as m_d_mysql
logbook.set_datetime_format("local")
logbook.RotatingFileHandler(
m_configs_logs.logpath,
level=m_configs_logs.loglevel,
backup_count=100,
max_size=1024 * 1024 * 10,
format_string=
'[{record.time}] {record.level_name} {record.filename} {record.lineno}: {record.message}'
).push_application()
LOG = logbook.Logger("***Main***")
__id__ = 0
__registered_dbs__ = {
# handle id: {
# dbname: db name
# tables: {
# tablename: table name
# columns: {
# colname: (bare/hash/list/set/sset, pos, int/text)
# }
from flask import jsonify
from gevent.pywsgi import LoggingLogAdapter
from gevent.pywsgi import WSGIServer
from sqlalchemy_utils import create_database
from sqlalchemy_utils import database_exists
import config
from api import users_blueprint
from app import app
from app import db
# Create log directory
pathlib.Path(config.LOG_PATH).mkdir(parents=True, exist_ok=True)
logbook.StreamHandler(sys.stdout).push_application()
logbook.RotatingFileHandler(config.LOGFILE, max_size=52428800,
bubble=True).push_application()
logbook.compat.redirect_logging(set_root_logger_level=False)
logging.root.setLevel(config.LOGLEVEL)
logger = logbook.Logger('[SERVER]', getattr(logbook, config.LOGLEVEL))
wsgi_logger = LoggingLogAdapter(logging.getLogger('wsgi'), level=logging.DEBUG)
wsgi_server = WSGIServer((config.API_IP, config.API_PORT),
app,
log=wsgi_logger,
error_log=wsgi_logger)
app.register_blueprint(users_blueprint)
@app.errorhandler(404)
import logbook
h1 = logbook.StderrHandler(bubble=True)
h1.push_application()
h2 = logbook.RotatingFileHandler("a.log", bubble=True)
h2.push_application()
logger = logbook.Logger("cli")
logger.debug("info")
def get_logger(setting_getter, name, fail_to_local=False, filter=None):
global got_logger
if got_logger:
# Yes, this means that if you try to change your logging within an
# application, it won't work. This is intentional. You shouldn't do
# that.
return got_logger
if filter:
def log_filter(r, h):
if server_pipe_log_filter_re.search(r.message):
return False
return filter(r, h)
else:
def log_filter(r, h):
return not server_pipe_log_filter_re.search(r.message)
logger_name = 'penguindome-' + name
file_safe_logger_name = logger_name.replace(os.sep, '_')
logger = logbook.Logger(logger_name)
internal_log_dir = os.path.join(var_dir, 'log')
internal_log_file = os.path.join(internal_log_dir,
file_safe_logger_name + '.log')
os.makedirs(internal_log_dir, 0o0700, exist_ok=True)
# We always do local debug logging, regardless of whether we're also
# logging elsewhere.
logbook.RotatingFileHandler(internal_log_file,
bubble=True,
filter=log_filter).push_application()
handler_name = setting_getter('logging:handler')
if handler_name:
handler_name = handler_name.lower()
handler_name += 'handler'
handler_name = next(d for d in dir(logbook)
if d.lower() == handler_name)
handler = logbook.__dict__[handler_name]
kwargs = {'bubble': True, 'filter': log_filter}
level = setting_getter('logging:level')
kwargs['level'] = logbook.__dict__[level.upper()]
if handler_name == 'SyslogHandler':
kwargs['facility'] = setting_getter('logging:syslog:facility')
hostname = setting_getter('logging:syslog:host')
if hostname:
port = setting_getter('logging:syslog:port')
try:
addrinfo = socket.getaddrinfo(hostname, port,
socket.AF_INET,
socket.SOCK_STREAM)[0]
except:
if not fail_to_local:
raise
logger.warn(
'Failed to resolve {}:{}, falling back to '
'local-only logging', hostname, port)
handler = None
else:
kwargs['socktype'] = addrinfo[1]
kwargs['address'] = addrinfo[4]
if handler:
if fail_to_local:
try:
with ThreadingTimeout(5, swallow_exc=False):
handler = handler(**kwargs)
except:
logger.warn(
'Failed to create {}, falling back to '
'local-only logging', handler_name)
else:
handler.push_application()
else:
handler(**kwargs).push_application()
logbook.compat.redirect_logging()
got_logger = logger
return got_logger
import feedparser
import logbook
logbook.RotatingFileHandler('parser.log').push_application()
logger = logbook.Logger('feed_parser')
def print_feed(feed, selected_tag):
logger.debug("Trying to print 'published', 'title' and 'link'...")
selected_feeds = [
entry for entry in feed.entries if selected_tag in entry.tags[0].term
]
for entry in selected_feeds:
try:
print(
f'[{entry.tags[0].term}] {entry.published} - {entry.title}: {entry.link}'
)
except AttributeError as error:
logger.error(
f'Could not print some of the attributes of the feed.')
print(f'Cannot find some of the attributes: {error}')
def main():
logger.info('Starting the application...')
selected_tag = input('Preferred tag name:')
# day1_file = 'my_file_day1.xml'
# feed = feedparser.parse(day1_file)
day2_file = 'my_file_day2.xml'
feed = feedparser.parse(day2_file)
print_feed(feed, selected_tag)
def main(): # pragma: no cover
config = SafeConfigParser()
dirs = ('.', '/etc', '/usr/local/etc')
if not config.read([os.path.join(dir, config_file) for dir in dirs]):
sys.exit('Could not find {} in {}'.format(config_file, dirs))
try:
logfile = config.get('logging', 'file')
rotating = config.getboolean('logging', 'rotate', fallback=False)
if rotating:
max_size = config.get('logging', 'max_size', fallback=1048576)
backup_count = config.get('logging', 'backup_count', fallback=5)
handler = logbook.RotatingFileHandler(logfile,
max_size=max_size,
backup_count=backup_count)
else:
handler = logbook.FileHandler(logfile)
handler.push_application()
except Exception:
logbook.StderrHandler().push_application()
try:
kwargs = dict(config.items('mongodb'))
except NoSectionError:
sys.exit('No "mongodb" section in config file')
args = []
for arg in ('hosts', 'database', 'username', 'password'):
try:
args.append(config.get('mongodb', arg))
except NoOptionError:
sys.exit(
'No "{}" setting in "mongodb" section of config file'.format(
arg))
kwargs.pop(arg)
args[0] = [s.strip() for s in args[0].split(',')]
store = MongoStore(*args, **kwargs)
try:
email_sender = config.get('email', 'sender')
except NoSectionError:
sys.exit('No "email" section in config file')
except NoOptionError:
sys.exit('No "sender" setting in "email" section of config file')
business_logic = BusinessLogic(store, email_sender)
try:
listen_port = int(config.get('wsgi', 'port'))
log.info('Binding to port {}'.format(listen_port))
except Exception:
listen_port = 80
log.info('Binding to default port {}'.format(listen_port))
try:
auth_key = config.get('wsgi', 'auth_key')
log.info('Server authentication enabled')
except Exception:
log.warning('Server authentication DISABLED')
auth_key = None
httpd = make_server('',
listen_port,
partial(application, business_logic, auth_key),
handler_class=LogbookWSGIRequestHandler)
business_logic.schedule_next_deadline()
httpd.serve_forever()
import struct
import logbook
#import gevent
from time import strftime, localtime
from utils import get_conf
logbook.set_datetime_format("local")
logger = logbook.Logger('AKSrv')
#log = logbook.FileHandler('heka_tcp.log')
log = logbook.RotatingFileHandler('logs/ak_server.log', max_size=102400, backup_count=5, \
bubble=True)
log.push_application()
# static
STX = 0x02
ETX = 0x03
BLANK = 0x20 #
def pack(cmd):
""" pack """
clen = len(cmd)
dt = strftime("%Y-%m-%d %H:%M:%S", localtime())
import logbook
import api
logbook.RotatingFileHandler('sportal_scraper.log').push_application()
logger = logbook.Logger('Main')
def main():
logger.info('Starting application...')
api.download_site('https://www.sportal.bg')
top_news = api.get_top_news()
api.look_for_pattern(top_news, 'Мадрид')
api.look_for_pattern(top_news, 'Барселона')
if __name__ == '__main__':
main()
import re
from pprint import pprint
import logbook
import requests
import bs4
logbook.RotatingFileHandler('scrapper.log',
level=logbook.TRACE).push_application()
logger = logbook.Logger('scraper')
def get_site(URL):
logger.debug(f'Getting url {URL}')
response = requests.get(URL)
response.raise_for_status()
logger.debug(f'Successfully retrieved url {URL}')
return response.text
def main():
logger.info('Start scraping...')
site = get_site('http://codechalleng.es/challenges/')
soup = bs4.BeautifulSoup(site, 'html.parser')
css_class = '.challengeTitle'
logger.debug('Looking for class {css_class}')
challenges = soup.select(css_class)
challenges_names = []
logger.info('Iterating over challenges...')
