def setup_logging() -> None: """ Sets up the logging module to have a verbose option and formats the Console handler and File handler """ logging.addLevelName(VERBOSE, "VERBOSE") logging.Logger.verbose = logger_verbose logging.verbose = logging_verbose logging.VERBOSE = VERBOSE # define console handler console_handler = ConsoleHandler(sys.stderr) if hasattr(console_handler, "_column_color"): # noinspection PyProtectedMember # pylint: disable=protected-access console_handler._column_color['%(message)s'][logging.VERBOSE] = ('cyan', None, False) console_formatter = logging.Formatter("[%(asctime)s] %(levelname)s : %(message)s", "%H:%M:%S") console_handler.setFormatter(console_formatter) console_handler.setLevel(logging.VERBOSE) # define file handler if not os.path.exists(os.path.dirname(get_global_conf().getdir("install", "log_file"))): os.makedirs(os.path.dirname(get_global_conf().getdir("install", "log_file"))) file_handler = logging.FileHandler(get_global_conf().getdir("install", "log_file")) # add handlers logging.getLogger().addHandler(console_handler) logging.getLogger().addHandler(file_handler) logging.getLogger().setLevel(get_global_conf().getint("install", "log_level"))
def setup_logging(logfile=None, verbose=False): if logfile: if os.path.exists(os.path.dirname(logfile)): # Setup default file logging and set the handler to recieve everything fh = logging.FileHandler(logfile) fh.setFormatter(logging.Formatter(CONST_LOG_FORMAT_FILE)) fh.setLevel(logging.INFO) log.addHandler(fh) else: raise ("log directory does not exist (" + os.path.dirname(logfile) + ")") clean_up(-1) # Add a log handler for stdout and set the handler to recieve everything csh = RainbowLoggingHandler(sys.stderr, color_funcName=('black', 'yellow', True)) csh.setFormatter(logging.Formatter(CONST_LOG_FORMAT_CONSOLE)) csh.setLevel(logging.DEBUG) log.addHandler(csh) # Now set the root logger to INFO log.setLevel(logging.INFO) # Check for verbose logging enabled if verbose is True: log.setLevel(logging.DEBUG) if logfile: fh.setLevel(logging.DEBUG) csh.setLevel(logging.DEBUG) log.debug('Debug logging enabled') return log
def _createLogger(self, name): formatter = logging.Formatter( fmt= '(%(levelname)s) [%(asctime)s,%(msecs)03d] %(name)s %(filename)s:%(lineno)d\t%(message)s', datefmt='%H:%M:%S') logger = logging.getLogger(name) logger.setLevel(self.level) handler = RainbowLoggingHandler(sys.stdout, datefmt='%H:%M:%S') handler.setLevel(self.level) handler.setFormatter(formatter) logger.addHandler(handler) return logger
def __init__(self, context=LOG_CONTEXT, level=_DEFAULT_LOG_LEVEL, log_format_string=_DEFAULT_LOG_FORMAT_FILE, console_format_string=_DEFAULT_LOG_FORMAT_CONSOLE): self.context = os.path.basename(context) self.logger = logging.getLogger(LOG_CONTEXT) self.file_handler = None self.console_handler = None self.cloud_handler = None AWSH_ROOT = os.getenv('AWSH_ROOT', '/tmp') AWSH_LOG_ROOT = os.getenv('HOME', '/tmp') log_filename = '{}/.awsh/log/{}.log'.format(AWSH_LOG_ROOT, self.context) if os.path.exists(os.path.dirname(log_filename)): if self.file_handler is None: # Setup default file logging and set the handler to recieve everything fh = logging.FileHandler(log_filename) fh.setFormatter(logging.Formatter(log_format_string)) fh.setLevel(level) self.logger.addHandler(fh) self.file_handler = fh if self.console_handler is None: # Add a log handler for stdout and set the handler to recieve everything csh = RainbowLoggingHandler(sys.stderr, color_funcName=('black', 'yellow', True)) csh.setFormatter(logging.Formatter(console_format_string)) csh.setLevel(level) self.logger.addHandler(csh) self.console_handler = csh # Now set the root logger to the specified level logging.getLogger(LOG_CONTEXT).setLevel(level) else: # If logging can't be initialised we need to exit on the assumption # that we should not run without audit logging at the very least m = 'log directory does not exist {}'.format( os.path.dirname(log_filename)) raise IOError(m) exit(1)
def get_logger(name): logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) handler = logging.FileHandler(os.path.join(config.logs_dir, '{}.log'.format(name))) handler.setFormatter(log_formatter) handler.setLevel(logging.DEBUG) handler2 = RainbowLoggingHandler( sys.stdout, '%Y-%m-%d %H:%M:%S', color_asctime=('white', None, False) ) handler2.setFormatter(log_formatter) handler2.setLevel(logging.DEBUG) logger.addHandler(handler) logger.addHandler(handler2) return logger
def enable_logging(filename: str, level: str): class Squelch(logging.Filter): def filter(self, record): if record.levelno == logging.DEBUG: return not record.name.startswith('asyncio') and \ not record.name.startswith('websockets') and \ not record.name.startswith('aiohttp') elif record.levelno == logging.INFO and record.name.startswith('asyncio'): return False return True formatter = logging.Formatter(fmt='%(asctime)s.%(msecs)03d:%(levelname)s:%(name)s:%(message)s') # File logger captures everything. file_handler = logging.FileHandler(filename) file_handler.setLevel(logging.DEBUG) # Console output level is configurable. stream_handler = RainbowLoggingHandler( sys.stdout, color_asctime=('cyan', None, False), color_msecs=('cyan', None, False), color_levelname=('gray', None, False), color_module=('yellow', None, False), color_name=('blue', None, False), color_lineno=('green', None, False), datefmt="%Y-%m-%d %H:%M:%S" ) stream_handler.setLevel(getattr(logging, level)) stream_handler.addFilter(Squelch()) # Set an output format. stream_handler.setFormatter(formatter) file_handler.setFormatter(formatter) file_handler.addFilter(Squelch()) # Add handlers to root. root = logging.getLogger('') root.setLevel(logging.DEBUG) root.addHandler(stream_handler) root.addHandler(file_handler)
def main(instruments, mode, log_level, start_date=None, end_date=None): """ Algorithmic trading tool. """ # Make urllib3 logger more calm urllib3_logger = logging.getLogger('urllib3') urllib3_logger.setLevel(logging.CRITICAL) numeric_level = getattr(logging, log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % log_level) log_filename = "logs/pyfx_debug_{}-{}.log".format(strftime("%Y_%m_%d-%H_%M"), settings.ENVIRONMENT) logging.basicConfig(filename=log_filename, level=logging.DEBUG) logger = logging.getLogger('pyFx') formatter = logging.Formatter("[%(asctime)s/%(levelname)s] %(funcName)s():%(lineno)d\t%(message)s") handler = RainbowLoggingHandler(sys.stdout, color_funcName=('black', 'yellow', True)) handler.setFormatter(formatter) handler.setLevel(numeric_level) logger.addHandler(handler) start_date_ = start_date if start_date else settings.BACKTEST_START end_date_ = end_date if end_date else settings.BACKTEST_END BACKTEST_START = parser.parse(start_date_).replace(tzinfo=pytz.utc) BACKTEST_END = parser.parse(end_date_).replace(tzinfo=pytz.utc) if mode == 'backtest': api = oandapy.API( environment=settings.ENVIRONMENT, access_token=settings.ACCESS_TOKEN, ) broker = OandaBacktestBroker( api=api, account_id=settings.ACCOUNT_ID, initial_balance=decimal.Decimal(5000)) # Oanda 20:00, Local: 22:00, DailyFx: 16:00 clock = SimulatedClock( start=BACKTEST_START, stop=BACKTEST_END, interval=settings.CLOCK_INTERVAL, ) elif mode == 'live': api = oandapy.API( environment=settings.ENVIRONMENT, access_token=settings.ACCESS_TOKEN, ) clock = IntervalClock(interval=settings.CLOCK_INTERVAL) broker = OandaRealtimeBroker(api=api, account_id=settings.ACCOUNT_ID) else: raise NotImplementedError() # TODO Optimize load of instrument info instrument_list = set(instruments) for inst in instrument_list: inst.load(broker) # TODO We have to be able to instantiate strategies with custom args strategies = [settings.STRATEGY(instrument) for instrument in instrument_list] if mode == 'backtest': broker.init_backtest(BACKTEST_START, BACKTEST_END, strategies) pf = Portfolio(broker, mode='backtest') else: pf = Portfolio(broker, mode='live') controller = Controller(clock, broker, pf, strategies) controller.run_until_stopped()
# log everything to the file import time, os logging.basicConfig( # A level of the logger level=logging.INFO, # Specify format string for the handler format='%(asctime)s %(filename)-15s:%(lineno)-4d %(levelname)-8s %(message)s', datefmt='%m-%d-%Y %H:%M:%S', filename='/tmp/hlp-{}-{}.log'.format(int(time.time()), os.getpid()), filemode='w' ) # log specific things to the console console = RainbowLoggingHandler(sys.stderr) console.setLevel(logging.DEBUG) console.setFormatter(logging.Formatter('%(filename)s:%(lineno)d\t%(levelname)-8s: %(message)s')) logging.getLogger('planning.interface').addHandler(console) logging.getLogger('integration.master').addHandler(console) logging.getLogger('__main__').addHandler(console) # log all warnings or higher consoleError = RainbowLoggingHandler(sys.stderr) consoleError.setLevel(logging.WARN) consoleError.setFormatter(logging.Formatter('%(filename)s:%(lineno)d\t%(levelname)-8s: %(message)s')) logging.getLogger().addHandler(consoleError) logging.getLogger('OpenGL').setLevel(99) # logging.getLogger('integration.jobs').setLevel(logging.WARNING) # logging.getLogger('integration.visualization').setLevel(logging.WARNING) # logging.getLogger('integration.interface').setLevel(logging.WARNING)
def log_handler_colored_console(level=logging.WARNING): handler = RainbowLoggingHandler(None, color_funcName=('black', 'yellow', True)) handler.setLevel(level) return handler
""" # Standard lib imports import logging.handlers import time import os import sys # Third Party imports from rainbow_logging_handler import RainbowLoggingHandler # BITSON imports from config import Config console_logger = logging.getLogger('trest') console_logger.setLevel(logging.DEBUG) console_handler = RainbowLoggingHandler(sys.stderr) console_handler.setLevel(logging.DEBUG) console_handler.setFormatter(logging.Formatter(fmt="%(message)s")) console_logger.addHandler(console_handler) log_format = "".join( ["[%(asctime)s] %(name)20s - %(levelname)8s: ", "%(threadName)15s-%(funcName)15s() - %(message)s"] ) formatter = logging.Formatter(fmt=log_format) # Format UTC Time formatter.converter = time.gmtime if not os.path.isdir(Config.LOG_FOLDER): os.mkdir(Config.LOG_FOLDER)
#! /usr/bin/env python # -*- coding:utf-8 -*- # Standard lib imports import logging.handlers import sys import time # Third Party imports from rainbow_logging_handler import RainbowLoggingHandler # BITSON imports logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) log_format = "".join(["[%(asctime)s] %(levelname)8s: ", "%(message)s"]) formatter = logging.Formatter(fmt=log_format) # Format UTC Time formatter.converter = time.gmtime # Console Handler ch = RainbowLoggingHandler(sys.stderr) ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) logger.addHandler(ch)
log_level = logging.CRITICAL # set up logging to file logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(name)s %(funcName)s():%(lineno)d\t%(message)s', datefmt='%m-%d %H:%M', filename='spam.log', filemode='w') # define a Handler which writes INFO messages or higher to the sys.stderr console = RainbowLoggingHandler(sys.stdout) #logging.StreamHandler() # handler = RainbowLoggingHandler(sys.stdout) # get level from command input console.setLevel(log_level) formatter = logging.Formatter("[%(asctime)s] %(name)s %(funcName)s():%(lineno)d\t%(message)s") # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger logging.getLogger('').addHandler(console) logger = logging.getLogger('nrk2013') # tell requests lib to chill out requests_log = logging.getLogger("requests") requests_log.setLevel(logging.WARNING) # Dette skulle hjelpe? # En ekte glob! request = False
try: # Rainbow Logging import sys from rainbow_logging_handler import RainbowLoggingHandler color_msecs = ('black', None, True) default_handler = RainbowLoggingHandler(sys.stdout, color_msecs=color_msecs, datefmt=datefmt) # HACK for msecs color default_handler._column_color['.'] = color_msecs default_handler._column_color['%(msecs)03d'] = color_msecs except Exception: default_handler = logging.StreamHandler() default_handler.setFormatter(default_fmt) default_handler.setLevel(logging.DEBUG) # setup root logger logger = logging.getLogger() logger.addHandler(default_handler) logger.setLevel(logging.DEBUG) def set_fmt(fmt=default_fmt): global defaut_handler default_handler.setFormatter(fmt) def set_root_level(level): global logger logger.setLevel(level)
def main(instruments, mode, log_level, debug, step, start_date=None, end_date=None): """ Algorithmic trading tool. """ _start_time = time() if debug: def exception_handler(type, value, traceback): click.secho( '\nAn exception occurred while executing the requested ' 'command:', fg='red' ) hr(fg='red') sys.__excepthook__(type, value, traceback) click.secho('\nStarting interactive debugging session:', fg='red') hr(fg='red') pdb.post_mortem(traceback) sys.excepthook = exception_handler if step: pdb.set_trace() # Make urllib3 logger more calm urllib3_logger = logging.getLogger('urllib3') urllib3_logger.setLevel(logging.CRITICAL) try: os.makedirs('logs') except OSError as exception: if exception.errno != errno.EEXIST: raise numeric_level = getattr(logging, log_level.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % log_level) log_filename = "logs/pyfx_debug_{}-{}.log".format( strftime("%Y_%m_%d-%H_%M"), settings.ENVIRONMENT) logging.basicConfig(filename=log_filename, level=logging.DEBUG) formatter = logging.Formatter( "[%(asctime)s/%(levelname)s] %(funcName)s():%(lineno)d\t%(message)s") handler = RainbowLoggingHandler( sys.stdout, color_funcName=('black', 'yellow', True)) handler.setFormatter(formatter) handler.setLevel(numeric_level) log.addHandler(handler) start_date_ = start_date if start_date else settings.BACKTEST_START end_date_ = end_date if end_date else settings.BACKTEST_END BACKTEST_START = parser.parse(start_date_).replace(tzinfo=pytz.utc) BACKTEST_END = parser.parse(end_date_).replace(tzinfo=pytz.utc) if mode == 'backtest': api = oandapy.API( environment=settings.ENVIRONMENT, access_token=settings.ACCESS_TOKEN, ) broker = OandaBacktestBroker( api=api, account_id=settings.ACCOUNT_ID, initial_balance=decimal.Decimal(5000)) # Oanda 20:00, Local: 22:00, DailyFx: 16:00 clock_interval = settings.CLOCK_INTERVAL log.info('Starting simulated clock with interval {} seconds'.format(clock_interval)) clock = SimulatedClock( start=BACKTEST_START, stop=BACKTEST_END, interval=clock_interval, ) elif mode == 'live': api = oandapy.API( environment=settings.ENVIRONMENT, access_token=settings.ACCESS_TOKEN, ) clock = IntervalClock(interval=settings.CLOCK_INTERVAL) broker = OandaRealtimeBroker(api=api, account_id=settings.ACCOUNT_ID) else: raise NotImplementedError() # TODO Optimize load of instrument info instrument_list = set(instruments) for inst in instrument_list: inst.load(broker) # TODO We have to be able to instantiate strategies with custom args strategies = [settings.STRATEGY(instrument) for instrument in instrument_list] if mode == 'backtest': broker.init_backtest(BACKTEST_START, BACKTEST_END, strategies) pf = Portfolio(broker, mode='backtest') else: pf = Portfolio(broker, mode='live') controller = Controller(clock, broker, pf, strategies) controller.run_until_stopped() log.info('script duration: {:.2f}s'.format(time() - _start_time))
# default log format default_fmt = logging.Formatter('[%(asctime)s] %(levelname)s ' '(%(process)d) %(name)s : %(message)s', datefmt='%Y/%m/%d %H:%M:%S') # set up handler try: # Rainbow Logging import sys from rainbow_logging_handler import RainbowLoggingHandler default_handler = RainbowLoggingHandler(sys.stdout) except: default_handler = logging.StreamHandler() default_handler.setFormatter(default_fmt) default_handler.setLevel(logging.DEBUG) # setup root logger logger = logging.getLogger() logger.addHandler(default_handler) def setFmt(fmt=default_fmt): global defaut_handler default_handler.setFormatter(fmt) def setRootLevel(level): global logger logger.setLevel(level)
# set up stream handler if not disable_stream_handler: try: # Rainbow Logging from rainbow_logging_handler import RainbowLoggingHandler # noqa: E402 color_msecs = ('green', None, True) stream_handler = RainbowLoggingHandler( sys.stdout, color_msecs=color_msecs, datefmt=datefmt ) # msecs color stream_handler._column_color['.'] = color_msecs stream_handler._column_color['%(asctime)s'] = color_msecs stream_handler._column_color['%(msecs)03d'] = color_msecs except Exception: stream_handler = StreamHandler() # the level of output logging stream_handler.setLevel(DEBUG) stream_handler.setFormatter(default_fmt) logger.addHandler(stream_handler) if not disable_file_handler: file_handler = FileHandler(filename=save_filename) # the level of output logging file_handler.setLevel(DEBUG) file_handler.setFormatter(default_fmt) logger.addHandler(file_handler)
def initialize_logger(args, script_name, syslog_server=None): """ Initialize the logging instance with the values provided by the user and return the logging object. """ LOGGER = logging.getLogger(script_name) FORMATTER = logging.Formatter( ""+script_name+" : "+str(getpass.getuser())+" : %(asctime)s : %(levelname)s : %(message)s", datefmt='%m/%d/%Y %I:%M:%S %p') # create console handler and set level to info logging.StreamHandler() # setup `RainbowLoggingHandler` handler = RainbowLoggingHandler(sys.stderr, color_funcName=('black', 'white', True)) handler.setFormatter(FORMATTER) LOGGER.addHandler(handler) # setup log level from arguments if args.log: if contains('DEBUG', args.log.upper()): handler.setLevel(logging.DEBUG) LOGGER.setLevel(logging.DEBUG) elif contains('WARNING', args.log.upper()): handler.setLevel(logging.WARNING) LOGGER.setLevel(logging.WARNING) elif contains('ERROR', args.log.upper()): handler.setLevel(logging.ERROR) LOGGER.setLevel(logging.ERROR) else: handler.setLevel(logging.INFO) LOGGER.setLevel(logging.INFO) LOGGER.info('Logging level has been set to {}'.format(args.log.upper())) # setup logging directory to store log files if args.log_dir: if not os.path.isdir(args.log_dir): os.makedirs(args.log_dir) output_dir = args.log_dir else: output_dir = create_log_dir() LOGGER.info('Logging directory has been set to {}'.format(output_dir)) # create optional syslog handler, if the argument has been supplied to support it if args.syslog: DESIG_SYSLOG_SERVER = syslog_server handler = logging.handlers.SysLogHandler(address=(DESIG_SYSLOG_SERVER, logging.handlers.SYSLOG_UDP_PORT)) syslog_format = logging.Formatter( '[appname]: %(name)s: [alias]: '+str(getpass.getuser())+' %(message)s') handler.setFormatter(syslog_format) if args.syslog_level: # Set syslog level to the user specified level if args.syslog_level == 'info': handler.setLevel(logging.INFO) elif args.syslog_level == 'warning': handler.setLevel(logging.WARNING) elif args.syslog_level == 'error': handler.setLevel(logging.ERROR) LOGGER.info( 'Syslog has been enabled for [{}] logging level, sent to syslog server [{}]'.format( args.syslog_level, DESIG_SYSLOG_SERVER)) # Add syslog handler to logging object LOGGER.addHandler(handler) # create error file handler and set level to error handler = logging.FileHandler(os.path.join( output_dir, "error.log"),"a", encoding=None, delay="true") handler.setLevel(logging.ERROR) handler.setFormatter(FORMATTER) LOGGER.addHandler(handler) # create debug file handler and set level to debug handler = logging.FileHandler(os.path.join(output_dir, "all.log"),"a") handler.setLevel(logging.DEBUG) handler.setFormatter(FORMATTER) LOGGER.addHandler(handler) if args.logfile_rotate: ''' User has specified they would like to create a daily rotating logfile. Create an instance and DO NOT create a single serving logfile. ''' LOGGER.info('Logfiles are now set to auto-rotate at midnight UTC.') filepath_and_name_format = ('{}/{}'.format(output_dir, script_name.replace('.py',''))) LOGGER.info('Logfiles are now being written at {}'.format(filepath_and_name_format)) log_filename=filepath_and_name_format # maxBytes takes the max file size in MB and bit-shift converts to bytes handler=SizedTimedRotatingFileHandler( log_filename, maxBytes=args.logsize<<20) handler.setLevel(logging.DEBUG) handler.setFormatter(FORMATTER) handler.suffix = "%Y-%m-%d.log" LOGGER.addHandler(handler) else: # create individual execution file handler and set level to debug now = datetime.datetime.now() datetime_stamp = now.strftime("%Y-%m-%d_%H-%M-%S") datetime_stamp = ('{}_{}.log'.format(datetime_stamp, script_name.replace('.py',''))) filename = os.path.join(output_dir, datetime_stamp) handler = logging.FileHandler(filename, "a") if contains('DEBUG', args.log.upper()): handler.setLevel(logging.DEBUG) elif contains('WARNING', args.log.upper()): handler.setLevel(logging.WARNING) elif contains('ERROR', args.log.upper()): handler.setLevel(logging.ERROR) else: handler.setLevel(logging.INFO) handler.setFormatter(FORMATTER) LOGGER.addHandler(handler) # display the exact file name / path to the user LOGGER.info('Logs for this session now being written to {}'.format(filename)) # Attach filepath / filename string to logger LOGGER.filename = filename return LOGGER
import sys import logging from rainbow_logging_handler import RainbowLoggingHandler formatter = logging.Formatter( fmt= '(%(levelname)s) [%(asctime)s,%(msecs)03d] %(name)s %(module)s:%(lineno)d\t%(message)s' ) # formatter = logging.Formatter(fmt='%(asctime)s,%(msecs)03d [%(levelname)s] | %(name)s[%(process)d] | %(message)s') logger = logging.getLogger('hehe') logger.setLevel(logging.DEBUG) handler = RainbowLoggingHandler(sys.stdout, datefmt='%H:%M:%S') handler.setLevel(logging.DEBUG) handler.setFormatter(formatter) logger.addHandler(handler) logger.debug("yoohoo") logger.debug("debug msg") logger.info("info msg") logger.warn("warn msg") logger.error("error msg") logger.critical("critical msg") Lrecord = logging.makeLogRecord({ 'name': 'hehe', 'levelno': 10, 'lineno': 10000, 'msg': 'hehehehhehehehehhe',