def log_all_levels(logger): for level, name in sorted(logging._levelToName.items()): logger.log(level, 'A sample {} message'.format(name.lower())) if logging_tree: print('\nLogging Tree:') logging_tree.printout()
def test_term_formatter_no_args(): formatter = color_bucket_logger.TermFormatter() logger = logging.getLogger(__name__ + '.test_logger') logger.disabled = False logger.setLevel(logging.DEBUG) handler = BufHandler(level=logging.DEBUG) handler.setFormatter(formatter) logger.addHandler(handler) logger.debug('default termFormatter no args %s', 'blip') other_logger() logger.critical('I think you chew too loudly') logging_tree.printout() # The default is to colorize by logger name, so setup two # loggers, and assert they got expected color therefore different other_exp = '\x1b[38;5;53m' too_loudly_exp = '\x1b[38;5;70m' for logged_item in handler.buf: testlog.debug('logged_item: %s', logged_item) if 'other_logger' in logged_item: assert other_exp in logged_item if 'too loudly' in logged_item: assert too_loudly_exp in logged_item
def main(): parser = argparse.ArgumentParser() parser.add_argument("-c", "--colors", action="store_true") parser.add_argument("-f", "--log-file") parser.add_argument("--file-level") parser.add_argument("--console-level") parser.add_argument("-j", "--json", action="store_true") parser.add_argument("-l", "--long-levels", action="store_true") parser.add_argument("-t", "--tree", action="store_true") args = parser.parse_args() with global_logconf(shorten_levels=not args.long_levels, colors=args.colors) as cfg: if args.log_file: cfg.log_to_file(args.log_file, args.file_level, json=args.json) cfg.log_to_console_if_interactive(args.console_level) log.debug("debug test msg with extra", extra=dict(foo="bar")) log.debug("debug test msg") log.info("info test msg") log.warning("warning test msg") log.error("error test msg") log.critical("critical test msg") try: raise Exception("test exception") except: log.exception("exception test msg") if args.tree: import logging_tree print() logging_tree.printout()
def main(): q = Manager().Queue() workers = [] for i in range(5): wp = Process(target=worker_process, name='worker %d' % (i + 1), args=(q, )) workers.append(wp) wp.start() logging.config.dictConfig(d) # Messages to the logger need to come after you add their handlers # to the root foo = logging.getLogger('foo') spam = logging.getLogger('spam') spam.info('spam info in the main') foo.info('foo info in the main') lp = logging.handlers.QueueListener(q, _SimpleHandler(), respect_handler_level=False) lp.start() # At this point, the main process could do some useful work of its own # Once it's done that, it can wait for the workers to terminate... for wp in workers: wp.join() # And now tell the logging thread to finish up, too # q.put(None) # lp.join() lp.stop() printout()
def check_logging(self): from snout.core.config import LOGCFG_FILE as loggingcfg if os.path.exists(loggingcfg): click.secho(f"- Logging configuration:{loggingcfg}", nl=False) if self.verbose: from logging_tree import printout printout()
def main(): parser = argparse.ArgumentParser() parser.add_argument('-c', '--colors', action='store_true') parser.add_argument('-f', '--log-file') parser.add_argument('--file-level') parser.add_argument('--console-level') parser.add_argument('-j', '--json', action='store_true') parser.add_argument('-l', '--long-levels', action='store_true') parser.add_argument('-t', '--tree', action='store_true') args = parser.parse_args() ls = LogSetup( shorten_levels=not args.long_levels, colors=args.colors, ) if args.log_file: ls.add_file(args.log_file, args.file_level, json=args.json) ls.add_console(args.console_level) ls.finish() log.debug('debug test msg with extra', extra=dict(foo='bar')) log.debug('debug test msg') log.info('info test msg') log.warning('warning test msg') log.error('error test msg') log.critical('critical test msg') try: raise Exception('test exception') except: log.exception('exception test msg') if args.tree: import logging_tree print() logging_tree.printout()
def show_setup(): if not LOGGING_TREE: return if not os.environ.get('CBL_LOGGING_TREE'): return logging_tree.printout()
def setup_logging(args): handler = get_logging_handler(args) LOG.addHandler(handler) try: import logging_tree print('logging_tree:') logging_tree.printout() except ImportError: pass del_logging_flags(args)
def test_jsonlogging_with_partial_compiler(all_attr_fmt, benchmark, kwargs, level, null_handler, null_logger): # Given null_handler.setFormatter(jsonlogging.Formatter(fmt=all_attr_fmt)) logging_tree.printout() # Then benchmark.pedantic(null_logger.log, args=(level, 'message'), kwargs=kwargs, rounds=100, iterations=100)
def setup(self, tree=None) -> None: if tree is None: tree = logging_tree.tree() if self.level_by_logger.get("logging_tree", "info").upper() == "DEBUG": logging_tree.printout() for n, l in self.level_by_logger.items(): if n == tree[0]: print( f"\033[33m setting logger \"{n}\" ({tree[1]}) at level {l}\033[0m" ) tree[1].setLevel(l.upper()) for child in tree[2]: self.setup(child)
def main(): m = Manager() results_q = m.Queue() exc_q = m.Queue() lp = threading.Thread(target=logger_thread, args=(exc_q,)) lp.start() with Pool(processes=4) as pool: pool.apply_async(listener, args=(results_q, )) jobs = [pool.apply_async(worker, args=(func, [x], results_q, exc_q)) for x in range(10)] for job in jobs: job.get() results_q.put(None) exc_q.put(None) lp.join() printout()
def test_stuff(formatter_class): teardown_config() logging_tree.printout() # All of the default log record attributes fmt_string = 'logger %(levelname)s %(asctime)s %(created)f %(msecs)d %(relativeCreated)d %(filename)s %(funcName)s ' + \ '%(levelno)s %(module)s %(pathname)s %(process)d %(processName)s' + \ '%(thread)d %(threadName)s %(name)s %(message)s' logger, handler, formatter = setup_logger(color_groups=[ ('name', ['name', 'levelname']) ], fmt=fmt_string, auto_color=True, formatter_class=formatter_class) sfmt = 'slogger %(levelname)s %(filename)s %(process)d' + \ '%(thread)d %(name)s %(message)s' slogger, shandler, formatter = setup_logger( color_groups=[('name', ['name', 'message']), ('thread', ['process', 'filename'])], fmt=sfmt, auto_color=False, formatter_class=formatter_class) logging_tree.printout() logger.debug('D: %s', '_debug') logger.info('I: %s', '_info') logger.warning('W: %s', '_warn') slogger.debug('foo1') slogger.info('foo2') slogger.error('foo4') # for record in handler.record_buf: # print(record.message) for logged_item in handler.buf: sys.stdout.write('%s\n' % logged_item) for logged_item in shandler.buf: sys.stdout.write('%s\n' % logged_item)
def main(): lfn = 'logging-cfg.json' if os.path.exists(lfn): logging.config.dictConfig(json.load(open(lfn))) else: logging.basicConfig() logging.getLogger().setLevel(logging.NOTSET) try: import logging_tree logging_tree.printout() except ImportError: pass log.debug('DEBUG LOGGING ON') log.warning('Available via env: JOB_TYPE={}, SLEEP_S={}'.format( JOB_TYPE, SLEEP_S)) exitOnFailure=False concurrent_jobs=2 #Workflow = pypeflow.controller.PypeThreadWorkflow Workflow = PypeProcWatcherWorkflow Workflow.setNumThreadAllowed(concurrent_jobs, concurrent_jobs) wf = Workflow(job_type=JOB_TYPE) par = dict(sleep_s=SLEEP_S) DIR ='mytmp' makedirs(DIR) f0 = makePypeLocalFile('mytmp/f0') f1 = makePypeLocalFile('mytmp/f1') make_task = PypeTask( #inputs = {'f': f}, outputs = {'f0': f0}, parameters = par, TaskType = MyFakePypeThreadTaskBase) task = make_task(taskrun0) wf.addTasks([task]) make_task = PypeTask( inputs = {'f0': f0}, outputs = {'f1': f1}, parameters = par, TaskType = MyFakePypeThreadTaskBase) task = make_task(taskrun1) wf.addTasks([task]) wf.refreshTargets([task])
def setup_task_logfiles(**kwargs): root_logfile_path = fps_app.conf.CELERY_LOG_PATHSTEM task_logfile_path = fps_app.conf.CELERY_TASK_LOG_PATHSTEM rootlogger = logging.getLogger() rootlogger.handlers[0].addFilter(NoFpsFilter()) log_config.setup_email_errorhandler(rootlogger) log_config.setup_logfile_handlers(rootlogger, logfile_pathstem=root_logfile_path) for pkg_name in ('voeventdb', 'fourpisky'): pkglogger = logging.getLogger(pkg_name) # # Don't intermix package output with celery output: # pkglogger.propagate = False pkglogger.setLevel(logging.DEBUG) log_config.setup_logfile_handlers(pkglogger, logfile_pathstem=task_logfile_path) from logging_tree import printout printout()
def main(): # use some reasonable defaults for setting up logging. # - log to stderr # - use a default format: # """%(asctime)s,%(msecs)03d %(levelname)-0.1s %(name)s %(processName)s:%(process)d %(funcName)s:%(lineno)d - %(message)s""" # main_log = alogging.default_setup(name='example.main') main_log = alogging.app_setup('example') main_log.debug('Log to logging "example.main"') example_module.do_startup_stuff() try: example_module.do_work() except Exception as exc: # gruntle a bit and continue log.exception(exc) log.debug('end of main') import logging_tree logging_tree.printout()
def initialize(self): self.stream_handler = StdlogStreamHandler(playbook_context=self.context) stream_formatter = self._choose_stdout_formatter(self.stream_formatter) self.stream_handler.setFormatter(stream_formatter) self.file_handler = StdlogFileHandler(self.file_formatter_file, playbook_context=self.context) file_formatter = self._choose_file_formatter(self.file_formatter) self.file_handler.setFormatter(file_formatter) self.logger = logging.getLogger(self.logger_name) if not any([isinstance(handler, (StdlogFileHandler, StdlogStreamHandler)) for handler in self.logger.handlers]): self.logger.addHandler(self.stream_handler) self.logger.addHandler(self.file_handler) self.logger.setLevel(self.logger_level) import logging_tree logging_tree.printout()
def main(): lfn = 'logging-cfg.json' if os.path.exists(lfn): logging.config.dictConfig(json.load(open(lfn))) else: logging.basicConfig() logging.getLogger().setLevel(logging.NOTSET) try: import logging_tree logging_tree.printout() except ImportError: pass log.debug('DEBUG LOGGING ON') log.warning('Available via env: JOB_TYPE={}, SLEEP_S={}'.format( JOB_TYPE, SLEEP_S)) exitOnFailure = False concurrent_jobs = 2 #Workflow = pypeflow.controller.PypeThreadWorkflow Workflow = PypeProcWatcherWorkflow Workflow.setNumThreadAllowed(concurrent_jobs, concurrent_jobs) wf = Workflow(job_type=JOB_TYPE) par = dict(sleep_s=SLEEP_S) DIR = 'mytmp' makedirs(DIR) f0 = makePypeLocalFile('mytmp/f0') f1 = makePypeLocalFile('mytmp/f1') make_task = PypeTask( #inputs = {'f': f}, outputs={'f0': f0}, parameters=par, TaskType=MyFakePypeThreadTaskBase) task = make_task(taskrun0) wf.addTasks([task]) make_task = PypeTask(inputs={'f0': f0}, outputs={'f1': f1}, parameters=par, TaskType=MyFakePypeThreadTaskBase) task = make_task(taskrun1) wf.addTasks([task]) wf.refreshTargets([task])
def handle(self, **options): try: import logging_tree except ImportError as ie: raise django.core.management.base.CommandError( 'Need to pip install logging_tree first') # logging_tree only prints the config for loggers that have been created # some may not get created just by running a management command logging.getLogger('debug') logging.getLogger('django') logging.getLogger('django.db') logging.getLogger('django.db.backends.schema') logging.getLogger('django.request') logging.getLogger('django.security.csrf') logging.getLogger('django.server') logging.getLogger('django.template') logging.getLogger('net') logging.getLogger('py.warnings') # only print config if these modules are present try: import rules logging.getLogger('rules') except ImportError: pass try: import werkzeug logging.getLogger('werkzeug') except ImportError: pass # logging_tree always goes to stdout; need to capture it and redirect to django management stdout orig_stdout = sys.stdout try: sys.stdout = self.stdout logging_tree.printout() finally: sys.stdout = orig_stdout
def print_diagram(): import logging_tree logging_tree.printout()
def debug(): """Debug Logger by printing logging hierarchy.""" from logging_tree import printout printout()
def logging(): '''Show loggers tree''' printout()
def main(): parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', help="Configuration", default=None, required=True) parser.add_argument( '-l', '--log-level', help='Log level for modules (by default for wca) in [module:]level form,' 'where level can be one of: CRITICAL,ERROR,WARNING,INFO,DEBUG,TRACE' 'Example -l debug -l example:debug. Defaults to wca:INFO.' 'Can be overridden at runtime with config.yaml "loggers" section.', default=[], action='append', dest='levels', ) parser.add_argument('-r', '--register', action='append', dest='components', help="Register additional components in config", default=[]) parser.add_argument('-v', '--version', action='version', version=platforms.get_wca_version(), help="Show version") parser.add_argument('-0', '--root', help="Allow WCA process to be run using root account", dest='is_root_allowed', action='store_true') args = parser.parse_args() # Do not allow to run WCA with root privileges unless user indicates that it is intended. uid = os.geteuid() if uid == 0 and not args.is_root_allowed: log.fatal( "Do not run WCA with root privileges. Consult documentation " "to understand what capabilities are required. If root account " "has to be used then set --root/-0 argument to override.") exit(2) # Initialize logging subsystem from command line options. log_levels = logger.parse_loggers_from_list(args.levels) log_levels_copy_with_default = dict(**log_levels) log_levels_copy_with_default.setdefault(logger.DEFAULT_MODULE, 'info') logger.configure_loggers_from_dict(log_levels_copy_with_default) log.warning( 'This software is pre-production and should not be deployed to production servers.' ) log.debug('started PID=%r', os.getpid()) log.info('Version wca: %s', platforms.get_wca_version()) # Register internal & external components. components.register_components(extra_components=args.components) valid_config_file(args.config) # Initialize all necessary objects. try: configuration = config.load_config(args.config) except config.ConfigLoadError as e: log.error('Error: Cannot load config file! : %s', e) if log.getEffectiveLevel() <= logging.DEBUG: log.exception('Detailed exception:') exit(1) for key in configuration: if key != 'loggers' and key != 'runner': log.error('Error: Unknown fields in configuration ' 'file! Possible are: \'loggers\', ' '\'runner\'') exit(1) assure_type(configuration, dict) assert 'runner' in configuration, 'Improper config - missing runner instance!' # Configure loggers using configuration file. if 'loggers' in configuration: log_levels_config = configuration['loggers'] if not isinstance(log_levels, dict): log.error( 'Loggers configuration error: log levels are mapping from logger name to' 'log level!') exit(1) # Merge config from cmd line and config file. # Overwrite config file values with values provided from command line. log_levels = dict(log_levels_config, **log_levels) logger.configure_loggers_from_dict(log_levels) # Dump loggers configurations to debug issues with loggers. if os.environ.get('WCA_DUMP_LOGGERS') == 'True': print( '------------------------------------ Logging tree ---------------------' ) import logging_tree logging_tree.printout() print( '------------------------------------ Logging tree END------------------' ) # Extract main loop component. runner = configuration['runner'] assure_type(runner, Runner) # Prepare and run the "main loop". exit_code = runner.run() exit(exit_code)
def logging(): '''Show loggers tree''' printout()
def logtree(): log.debug('logtree') import logging_tree logging_tree.printout()
"pbs=True given, but not running in a PBS job (pbs_jobid empty)." ) return logging.config.dictConfig(config_dict) if __name__ == "__main__": import logging import cdsapi from logging_tree import printout c = cdsapi.Client() logging.config.dictConfig(LOGGING) logger0 = logging.getLogger("") logger1 = logging.getLogger(__name__) logger2 = logging.getLogger("testing") printout() for level, level_str in ( (logging.DEBUG, "DEBUG"), (logging.INFO, "INFO"), (logging.WARNING, "WARNING"), ): for logger, logger_name in ( (logger0, "root logger"), (logger1, "{} logger".format(__name__)), (logger2, "testing logger"), ): logger.log(level, "{} {}".format(level_str, logger_name + " test"))
def test_n_recentscw(cdciplatform, timestamp=None, osaversion="osa10.2", n_scw=2, e_offset=0, source=None, *a, **aa): import requests import time logging.getLogger('oda_api').setLevel(logging.DEBUG) logging.getLogger('oda_api').addHandler(logging.StreamHandler()) from logging_tree import printout printout() if timestamp is None: timestamp = time.time() else: timestamp = float(time.time()) if source is None: ra = 0 dec = 0 radius = 180 t1 = timestamp - 24 * 3600 * 580 t2 = timestamp - 24 * 3600 * 530 elif source == "Crab": ra = 83 dec = 22 radius = 5 t1 = timestamp - 24 * 3600 * 980 t2 = timestamp - 24 * 3600 * 530 elif source in ["Sco X-1", "Cyg X-1"]: c = get_named_source_coord(source) ra = c.ra.deg dec = c.dec.deg print(c) radius = 5 t1 = timestamp - 24 * 3600 * 365 * 15 t2 = timestamp - 24 * 3600 * 365 * 1.5 else: raise NotImplementedError r = get_scw_list(ra_obj=ra, dec_obj=dec, radius=radius, start_date=time.strftime("%Y-%m-%dT%H:00:00", time.gmtime(t1)), end_date=time.strftime("%Y-%m-%dT%H:00:00", time.gmtime(t2))) random.seed(0) scwpick = random.sample(r, n_scw) print("picked:", scwpick) assert len(scwpick) > 0 print("running test image at ", cdciplatform) from oda_api.api import DispatcherAPI from oda_api.plot_tools import OdaImage, OdaLightCurve from oda_api.data_products import BinaryData import os from astropy.io import fits import numpy as np from numpy import sqrt endpoint = platform_endpoint(cdciplatform) disp = DispatcherAPI(host=endpoint) disp.set_custom_progress_formatter(custom_progress_formatter) print(disp) if '10.2' in osaversion: osa_version = 'OSA10.2' elif '11.0' in osaversion: osa_version = 'OSA11.0' else: osa_version = 'OSA10.2' # default data = disp.get_product(instrument='isgri', product='isgri_image', scw_list=[str(s) for s in scwpick], E1_keV=25 + e_offset, E2_keV=80 + e_offset, osa_version=osa_version, RA=0, DEC=0, detection_threshold=15, product_type='Real') printout() print(data) print(dir(data)) catalog_table = data.dispatcher_catalog_1.table m = catalog_table['significance'] >= 0.0 print(catalog_table[m]) if source is not None: print(f"\033[31m source check requested for {source}\033[0m") print(f"found sources:", catalog_table['src_names']) t = catalog_table[catalog_table['src_names'] == source] print(t) assert len(t) == 1
def handle(self, *args, **options): import logging_tree logging_tree.printout()
if __name__ == '__main__': q = Queue() root = logging.getLogger() root.setLevel(logging.INFO) root.info('main') #h1 = logging.StreamHandler() #h1.setLevel(logging.ERROR) h2 = logging.FileHandler('process_logging.log') h2.setLevel(logging.DEBUG) #root.addHandler(h1) #root.addHandler(h2) workers = [] for i in range(5): wp = Process(target=worker_process, name='worker %d' % (i + 1), args=(func, i, q,)) workers.append(wp) wp.start() # logging.config.dictConfig(d) lp = threading.Thread(target=logger_thread, args=(q,)) lp.start() # At this point, the main process could do some useful work of its own # Once it's done that, it can wait for the workers to terminate... for wp in workers: wp.join() # And now tell the logging thread to finish up, too q.put(None) lp.join() lt.printout()
from pprint import pprint from config import Config import logging, logging.config from logging_tree import printout import utils cfg = Config(file("private_config_new.cfg")) logging.config.dictConfig(utils.get_log_dict()) printout()
newhandler = mlp.handlers['sockethand'] mlp.loggers['watchedLog'].addHandler(newhandler) for x in range(10): lvl = choice(levels) lvlname = logging.getLevelName(lvl) mlp.loggers['watchedLog'].log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters') # myls = mlog.Log_Server() # myls.start_log_server(8080) # myls.stop_log_server() print("Current logging tree after tests") logging_tree.printout(node=None) print() print('stopping que') mlp.stopQueue() print('Main Test Done!') context = dict(vars(logging)) context['handlers'] = mlp.handlers evaluator = eVal.Evaluator(context, True) try: while True: line = input('Enter source to evaluate or enter to exit:').strip() if not line: break
def _logging(): printout()
import json; import logging; import logging.config import logging_tree logging.config.dictConfig(json.load(open("conf_dict.json"))) log = logging.getLogger("") child_A = logging.getLogger("A") child_B = logging.getLogger("B") child_B_A = logging.getLogger("B.A") log.info("Now this is %s logging!","good") child_A.info("Now this is more logging!") log.warning("Now this is %s logging!","worrisome") logging_tree.printout()
def main(reloadp=False, debug=False, showlogconf=False): if showlogconf: logging_tree.printout() LOG.warn('upload.main() running') app.run(debug=debug, use_reloader=reloadp, host='0.0.0.0', port=5050)
from rq import Queue import RedisConfigHelper import json import logging import boto import os from utils import * import logging_tree from redis import StrictRedis logging.config.fileConfig("%s/%s" % (os.getcwd(),"logging.cfg")) logger = logging.getLogger() logging_tree.printout() parser = argparse.ArgumentParser(description='Subscribes to a Redis PubSub Channel and Manage the MDM on Request') parser.add_argument('-c','--channel', help='Channel Name', required=True) parser.add_argument('-q','--queue', help='RedisQueue queue name to dispatch to', required=True) args = vars(parser.parse_args()) ready = PubSub(redis.Redis(), args['channel']) mdmqueue = Queue("MDM",connection=redis.Redis()) config = RedisConfigHelper.RedisConfigHelper() r = StrictRedis()
def debug(): logging_tree.printout()