def log_all_levels(logger):
for level, name in sorted(logging._levelToName.items()):
logger.log(level, 'A sample {} message'.format(name.lower()))
if logging_tree:
print('\nLogging Tree:')
logging_tree.printout()
def test_term_formatter_no_args():
formatter = color_bucket_logger.TermFormatter()
logger = logging.getLogger(__name__ + '.test_logger')
logger.disabled = False
logger.setLevel(logging.DEBUG)
handler = BufHandler(level=logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.debug('default termFormatter no args %s', 'blip')
other_logger()
logger.critical('I think you chew too loudly')
logging_tree.printout()
# The default is to colorize by logger name, so setup two
# loggers, and assert they got expected color therefore different
other_exp = '\x1b[38;5;53m'
too_loudly_exp = '\x1b[38;5;70m'
for logged_item in handler.buf:
testlog.debug('logged_item: %s', logged_item)
if 'other_logger' in logged_item:
assert other_exp in logged_item
if 'too loudly' in logged_item:
assert too_loudly_exp in logged_item
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--colors", action="store_true")
parser.add_argument("-f", "--log-file")
parser.add_argument("--file-level")
parser.add_argument("--console-level")
parser.add_argument("-j", "--json", action="store_true")
parser.add_argument("-l", "--long-levels", action="store_true")
parser.add_argument("-t", "--tree", action="store_true")
args = parser.parse_args()
with global_logconf(shorten_levels=not args.long_levels,
colors=args.colors) as cfg:
if args.log_file:
cfg.log_to_file(args.log_file, args.file_level, json=args.json)
cfg.log_to_console_if_interactive(args.console_level)
log.debug("debug test msg with extra", extra=dict(foo="bar"))
log.debug("debug test msg")
log.info("info test msg")
log.warning("warning test msg")
log.error("error test msg")
log.critical("critical test msg")
try:
raise Exception("test exception")
except:
log.exception("exception test msg")
if args.tree:
import logging_tree
print()
logging_tree.printout()
def main():
q = Manager().Queue()
workers = []
for i in range(5):
wp = Process(target=worker_process,
name='worker %d' % (i + 1),
args=(q, ))
workers.append(wp)
wp.start()
logging.config.dictConfig(d)
# Messages to the logger need to come after you add their handlers
# to the root
foo = logging.getLogger('foo')
spam = logging.getLogger('spam')
spam.info('spam info in the main')
foo.info('foo info in the main')
lp = logging.handlers.QueueListener(q,
_SimpleHandler(),
respect_handler_level=False)
lp.start()
# At this point, the main process could do some useful work of its own
# Once it's done that, it can wait for the workers to terminate...
for wp in workers:
wp.join()
# And now tell the logging thread to finish up, too
# q.put(None)
# lp.join()
lp.stop()
printout()
def check_logging(self):
from snout.core.config import LOGCFG_FILE as loggingcfg
if os.path.exists(loggingcfg):
click.secho(f"- Logging configuration:{loggingcfg}", nl=False)
if self.verbose:
from logging_tree import printout
printout()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--colors', action='store_true')
parser.add_argument('-f', '--log-file')
parser.add_argument('--file-level')
parser.add_argument('--console-level')
parser.add_argument('-j', '--json', action='store_true')
parser.add_argument('-l', '--long-levels', action='store_true')
parser.add_argument('-t', '--tree', action='store_true')
args = parser.parse_args()
ls = LogSetup(
shorten_levels=not args.long_levels,
colors=args.colors,
)
if args.log_file:
ls.add_file(args.log_file, args.file_level, json=args.json)
ls.add_console(args.console_level)
ls.finish()
log.debug('debug test msg with extra', extra=dict(foo='bar'))
log.debug('debug test msg')
log.info('info test msg')
log.warning('warning test msg')
log.error('error test msg')
log.critical('critical test msg')
try:
raise Exception('test exception')
except:
log.exception('exception test msg')
if args.tree:
import logging_tree
print()
logging_tree.printout()
def show_setup():
if not LOGGING_TREE:
return
if not os.environ.get('CBL_LOGGING_TREE'):
return
logging_tree.printout()
def setup_logging(args):
handler = get_logging_handler(args)
LOG.addHandler(handler)
try:
import logging_tree
print('logging_tree:')
logging_tree.printout()
except ImportError:
pass
del_logging_flags(args)
def test_jsonlogging_with_partial_compiler(all_attr_fmt, benchmark, kwargs,
level, null_handler, null_logger):
# Given
null_handler.setFormatter(jsonlogging.Formatter(fmt=all_attr_fmt))
logging_tree.printout()
# Then
benchmark.pedantic(null_logger.log,
args=(level, 'message'),
kwargs=kwargs,
rounds=100,
iterations=100)
def setup(self, tree=None) -> None:
if tree is None:
tree = logging_tree.tree()
if self.level_by_logger.get("logging_tree",
"info").upper() == "DEBUG":
logging_tree.printout()
for n, l in self.level_by_logger.items():
if n == tree[0]:
print(
f"\033[33m setting logger \"{n}\" ({tree[1]}) at level {l}\033[0m"
)
tree[1].setLevel(l.upper())
for child in tree[2]:
self.setup(child)
def main():
m = Manager()
results_q = m.Queue()
exc_q = m.Queue()
lp = threading.Thread(target=logger_thread, args=(exc_q,))
lp.start()
with Pool(processes=4) as pool:
pool.apply_async(listener, args=(results_q, ))
jobs = [pool.apply_async(worker, args=(func, [x], results_q, exc_q)) for x in range(10)]
for job in jobs:
job.get()
results_q.put(None)
exc_q.put(None)
lp.join()
printout()
def test_stuff(formatter_class):
teardown_config()
logging_tree.printout()
# All of the default log record attributes
fmt_string = 'logger %(levelname)s %(asctime)s %(created)f %(msecs)d %(relativeCreated)d %(filename)s %(funcName)s ' + \
'%(levelno)s %(module)s %(pathname)s %(process)d %(processName)s' + \
'%(thread)d %(threadName)s %(name)s %(message)s'
logger, handler, formatter = setup_logger(color_groups=[
('name', ['name', 'levelname'])
],
fmt=fmt_string,
auto_color=True,
formatter_class=formatter_class)
sfmt = 'slogger %(levelname)s %(filename)s %(process)d' + \
'%(thread)d %(name)s %(message)s'
slogger, shandler, formatter = setup_logger(
color_groups=[('name', ['name', 'message']),
('thread', ['process', 'filename'])],
fmt=sfmt,
auto_color=False,
formatter_class=formatter_class)
logging_tree.printout()
logger.debug('D: %s', '_debug')
logger.info('I: %s', '_info')
logger.warning('W: %s', '_warn')
slogger.debug('foo1')
slogger.info('foo2')
slogger.error('foo4')
# for record in handler.record_buf:
# print(record.message)
for logged_item in handler.buf:
sys.stdout.write('%s\n' % logged_item)
for logged_item in shandler.buf:
sys.stdout.write('%s\n' % logged_item)
def main():
lfn = 'logging-cfg.json'
if os.path.exists(lfn):
logging.config.dictConfig(json.load(open(lfn)))
else:
logging.basicConfig()
logging.getLogger().setLevel(logging.NOTSET)
try:
import logging_tree
logging_tree.printout()
except ImportError:
pass
log.debug('DEBUG LOGGING ON')
log.warning('Available via env: JOB_TYPE={}, SLEEP_S={}'.format(
JOB_TYPE, SLEEP_S))
exitOnFailure=False
concurrent_jobs=2
#Workflow = pypeflow.controller.PypeThreadWorkflow
Workflow = PypeProcWatcherWorkflow
Workflow.setNumThreadAllowed(concurrent_jobs, concurrent_jobs)
wf = Workflow(job_type=JOB_TYPE)
par = dict(sleep_s=SLEEP_S)
DIR ='mytmp'
makedirs(DIR)
f0 = makePypeLocalFile('mytmp/f0')
f1 = makePypeLocalFile('mytmp/f1')
make_task = PypeTask(
#inputs = {'f': f},
outputs = {'f0': f0},
parameters = par,
TaskType = MyFakePypeThreadTaskBase)
task = make_task(taskrun0)
wf.addTasks([task])
make_task = PypeTask(
inputs = {'f0': f0},
outputs = {'f1': f1},
parameters = par,
TaskType = MyFakePypeThreadTaskBase)
task = make_task(taskrun1)
wf.addTasks([task])
wf.refreshTargets([task])
def setup_task_logfiles(**kwargs):
root_logfile_path = fps_app.conf.CELERY_LOG_PATHSTEM
task_logfile_path = fps_app.conf.CELERY_TASK_LOG_PATHSTEM
rootlogger = logging.getLogger()
rootlogger.handlers[0].addFilter(NoFpsFilter())
log_config.setup_email_errorhandler(rootlogger)
log_config.setup_logfile_handlers(rootlogger,
logfile_pathstem=root_logfile_path)
for pkg_name in ('voeventdb', 'fourpisky'):
pkglogger = logging.getLogger(pkg_name)
# # Don't intermix package output with celery output:
# pkglogger.propagate = False
pkglogger.setLevel(logging.DEBUG)
log_config.setup_logfile_handlers(pkglogger,
logfile_pathstem=task_logfile_path)
from logging_tree import printout
printout()
def main():
# use some reasonable defaults for setting up logging.
# - log to stderr
# - use a default format:
# """%(asctime)s,%(msecs)03d %(levelname)-0.1s %(name)s %(processName)s:%(process)d %(funcName)s:%(lineno)d - %(message)s"""
# main_log = alogging.default_setup(name='example.main')
main_log = alogging.app_setup('example')
main_log.debug('Log to logging "example.main"')
example_module.do_startup_stuff()
try:
example_module.do_work()
except Exception as exc:
# gruntle a bit and continue
log.exception(exc)
log.debug('end of main')
import logging_tree
logging_tree.printout()
def initialize(self):
self.stream_handler = StdlogStreamHandler(playbook_context=self.context)
stream_formatter = self._choose_stdout_formatter(self.stream_formatter)
self.stream_handler.setFormatter(stream_formatter)
self.file_handler = StdlogFileHandler(self.file_formatter_file,
playbook_context=self.context)
file_formatter = self._choose_file_formatter(self.file_formatter)
self.file_handler.setFormatter(file_formatter)
self.logger = logging.getLogger(self.logger_name)
if not any([isinstance(handler, (StdlogFileHandler, StdlogStreamHandler)) for handler in self.logger.handlers]):
self.logger.addHandler(self.stream_handler)
self.logger.addHandler(self.file_handler)
self.logger.setLevel(self.logger_level)
import logging_tree
logging_tree.printout()
def main():
lfn = 'logging-cfg.json'
if os.path.exists(lfn):
logging.config.dictConfig(json.load(open(lfn)))
else:
logging.basicConfig()
logging.getLogger().setLevel(logging.NOTSET)
try:
import logging_tree
logging_tree.printout()
except ImportError:
pass
log.debug('DEBUG LOGGING ON')
log.warning('Available via env: JOB_TYPE={}, SLEEP_S={}'.format(
JOB_TYPE, SLEEP_S))
exitOnFailure = False
concurrent_jobs = 2
#Workflow = pypeflow.controller.PypeThreadWorkflow
Workflow = PypeProcWatcherWorkflow
Workflow.setNumThreadAllowed(concurrent_jobs, concurrent_jobs)
wf = Workflow(job_type=JOB_TYPE)
par = dict(sleep_s=SLEEP_S)
DIR = 'mytmp'
makedirs(DIR)
f0 = makePypeLocalFile('mytmp/f0')
f1 = makePypeLocalFile('mytmp/f1')
make_task = PypeTask(
#inputs = {'f': f},
outputs={'f0': f0},
parameters=par,
TaskType=MyFakePypeThreadTaskBase)
task = make_task(taskrun0)
wf.addTasks([task])
make_task = PypeTask(inputs={'f0': f0},
outputs={'f1': f1},
parameters=par,
TaskType=MyFakePypeThreadTaskBase)
task = make_task(taskrun1)
wf.addTasks([task])
wf.refreshTargets([task])
def handle(self, **options):
try:
import logging_tree
except ImportError as ie:
raise django.core.management.base.CommandError(
'Need to pip install logging_tree first')
# logging_tree only prints the config for loggers that have been created
# some may not get created just by running a management command
logging.getLogger('debug')
logging.getLogger('django')
logging.getLogger('django.db')
logging.getLogger('django.db.backends.schema')
logging.getLogger('django.request')
logging.getLogger('django.security.csrf')
logging.getLogger('django.server')
logging.getLogger('django.template')
logging.getLogger('net')
logging.getLogger('py.warnings')
# only print config if these modules are present
try:
import rules
logging.getLogger('rules')
except ImportError:
pass
try:
import werkzeug
logging.getLogger('werkzeug')
except ImportError:
pass
# logging_tree always goes to stdout; need to capture it and redirect to django management stdout
orig_stdout = sys.stdout
try:
sys.stdout = self.stdout
logging_tree.printout()
finally:
sys.stdout = orig_stdout
def print_diagram():
import logging_tree
logging_tree.printout()
def debug():
"""Debug Logger by printing logging hierarchy."""
from logging_tree import printout
printout()
def logging():
'''Show loggers tree'''
printout()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c',
'--config',
help="Configuration",
default=None,
required=True)
parser.add_argument(
'-l',
'--log-level',
help='Log level for modules (by default for wca) in [module:]level form,'
'where level can be one of: CRITICAL,ERROR,WARNING,INFO,DEBUG,TRACE'
'Example -l debug -l example:debug. Defaults to wca:INFO.'
'Can be overridden at runtime with config.yaml "loggers" section.',
default=[],
action='append',
dest='levels',
)
parser.add_argument('-r',
'--register',
action='append',
dest='components',
help="Register additional components in config",
default=[])
parser.add_argument('-v',
'--version',
action='version',
version=platforms.get_wca_version(),
help="Show version")
parser.add_argument('-0',
'--root',
help="Allow WCA process to be run using root account",
dest='is_root_allowed',
action='store_true')
args = parser.parse_args()
# Do not allow to run WCA with root privileges unless user indicates that it is intended.
uid = os.geteuid()
if uid == 0 and not args.is_root_allowed:
log.fatal(
"Do not run WCA with root privileges. Consult documentation "
"to understand what capabilities are required. If root account "
"has to be used then set --root/-0 argument to override.")
exit(2)
# Initialize logging subsystem from command line options.
log_levels = logger.parse_loggers_from_list(args.levels)
log_levels_copy_with_default = dict(**log_levels)
log_levels_copy_with_default.setdefault(logger.DEFAULT_MODULE, 'info')
logger.configure_loggers_from_dict(log_levels_copy_with_default)
log.warning(
'This software is pre-production and should not be deployed to production servers.'
)
log.debug('started PID=%r', os.getpid())
log.info('Version wca: %s', platforms.get_wca_version())
# Register internal & external components.
components.register_components(extra_components=args.components)
valid_config_file(args.config)
# Initialize all necessary objects.
try:
configuration = config.load_config(args.config)
except config.ConfigLoadError as e:
log.error('Error: Cannot load config file! : %s', e)
if log.getEffectiveLevel() <= logging.DEBUG:
log.exception('Detailed exception:')
exit(1)
for key in configuration:
if key != 'loggers' and key != 'runner':
log.error('Error: Unknown fields in configuration '
'file! Possible are: \'loggers\', '
'\'runner\'')
exit(1)
assure_type(configuration, dict)
assert 'runner' in configuration, 'Improper config - missing runner instance!'
# Configure loggers using configuration file.
if 'loggers' in configuration:
log_levels_config = configuration['loggers']
if not isinstance(log_levels, dict):
log.error(
'Loggers configuration error: log levels are mapping from logger name to'
'log level!')
exit(1)
# Merge config from cmd line and config file.
# Overwrite config file values with values provided from command line.
log_levels = dict(log_levels_config, **log_levels)
logger.configure_loggers_from_dict(log_levels)
# Dump loggers configurations to debug issues with loggers.
if os.environ.get('WCA_DUMP_LOGGERS') == 'True':
print(
'------------------------------------ Logging tree ---------------------'
)
import logging_tree
logging_tree.printout()
print(
'------------------------------------ Logging tree END------------------'
)
# Extract main loop component.
runner = configuration['runner']
assure_type(runner, Runner)
# Prepare and run the "main loop".
exit_code = runner.run()
exit(exit_code)
def logging():
'''Show loggers tree'''
printout()
def logtree():
log.debug('logtree')
import logging_tree
logging_tree.printout()
"pbs=True given, but not running in a PBS job (pbs_jobid empty)."
)
return
logging.config.dictConfig(config_dict)
if __name__ == "__main__":
import logging
import cdsapi
from logging_tree import printout
c = cdsapi.Client()
logging.config.dictConfig(LOGGING)
logger0 = logging.getLogger("")
logger1 = logging.getLogger(__name__)
logger2 = logging.getLogger("testing")
printout()
for level, level_str in (
(logging.DEBUG, "DEBUG"),
(logging.INFO, "INFO"),
(logging.WARNING, "WARNING"),
):
for logger, logger_name in (
(logger0, "root logger"),
(logger1, "{} logger".format(__name__)),
(logger2, "testing logger"),
):
logger.log(level, "{} {}".format(level_str, logger_name + " test"))
def test_n_recentscw(cdciplatform,
timestamp=None,
osaversion="osa10.2",
n_scw=2,
e_offset=0,
source=None,
*a,
**aa):
import requests
import time
logging.getLogger('oda_api').setLevel(logging.DEBUG)
logging.getLogger('oda_api').addHandler(logging.StreamHandler())
from logging_tree import printout
printout()
if timestamp is None:
timestamp = time.time()
else:
timestamp = float(time.time())
if source is None:
ra = 0
dec = 0
radius = 180
t1 = timestamp - 24 * 3600 * 580
t2 = timestamp - 24 * 3600 * 530
elif source == "Crab":
ra = 83
dec = 22
radius = 5
t1 = timestamp - 24 * 3600 * 980
t2 = timestamp - 24 * 3600 * 530
elif source in ["Sco X-1", "Cyg X-1"]:
c = get_named_source_coord(source)
ra = c.ra.deg
dec = c.dec.deg
print(c)
radius = 5
t1 = timestamp - 24 * 3600 * 365 * 15
t2 = timestamp - 24 * 3600 * 365 * 1.5
else:
raise NotImplementedError
r = get_scw_list(ra_obj=ra,
dec_obj=dec,
radius=radius,
start_date=time.strftime("%Y-%m-%dT%H:00:00",
time.gmtime(t1)),
end_date=time.strftime("%Y-%m-%dT%H:00:00",
time.gmtime(t2)))
random.seed(0)
scwpick = random.sample(r, n_scw)
print("picked:", scwpick)
assert len(scwpick) > 0
print("running test image at ", cdciplatform)
from oda_api.api import DispatcherAPI
from oda_api.plot_tools import OdaImage, OdaLightCurve
from oda_api.data_products import BinaryData
import os
from astropy.io import fits
import numpy as np
from numpy import sqrt
endpoint = platform_endpoint(cdciplatform)
disp = DispatcherAPI(host=endpoint)
disp.set_custom_progress_formatter(custom_progress_formatter)
print(disp)
if '10.2' in osaversion:
osa_version = 'OSA10.2'
elif '11.0' in osaversion:
osa_version = 'OSA11.0'
else:
osa_version = 'OSA10.2' # default
data = disp.get_product(instrument='isgri',
product='isgri_image',
scw_list=[str(s) for s in scwpick],
E1_keV=25 + e_offset,
E2_keV=80 + e_offset,
osa_version=osa_version,
RA=0,
DEC=0,
detection_threshold=15,
product_type='Real')
printout()
print(data)
print(dir(data))
catalog_table = data.dispatcher_catalog_1.table
m = catalog_table['significance'] >= 0.0
print(catalog_table[m])
if source is not None:
print(f"\033[31m source check requested for {source}\033[0m")
print(f"found sources:", catalog_table['src_names'])
t = catalog_table[catalog_table['src_names'] == source]
print(t)
assert len(t) == 1
def handle(self, *args, **options):
import logging_tree
logging_tree.printout()
if __name__ == '__main__':
q = Queue()
root = logging.getLogger()
root.setLevel(logging.INFO)
root.info('main')
#h1 = logging.StreamHandler()
#h1.setLevel(logging.ERROR)
h2 = logging.FileHandler('process_logging.log')
h2.setLevel(logging.DEBUG)
#root.addHandler(h1)
#root.addHandler(h2)
workers = []
for i in range(5):
wp = Process(target=worker_process, name='worker %d' % (i + 1),
args=(func, i, q,))
workers.append(wp)
wp.start()
# logging.config.dictConfig(d)
lp = threading.Thread(target=logger_thread, args=(q,))
lp.start()
# At this point, the main process could do some useful work of its own
# Once it's done that, it can wait for the workers to terminate...
for wp in workers:
wp.join()
# And now tell the logging thread to finish up, too
q.put(None)
lp.join()
lt.printout()
from pprint import pprint
from config import Config
import logging, logging.config
from logging_tree import printout
import utils
cfg = Config(file("private_config_new.cfg"))
logging.config.dictConfig(utils.get_log_dict())
printout()
newhandler = mlp.handlers['sockethand']
mlp.loggers['watchedLog'].addHandler(newhandler)
for x in range(10):
lvl = choice(levels)
lvlname = logging.getLevelName(lvl)
mlp.loggers['watchedLog'].log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters')
# myls = mlog.Log_Server()
# myls.start_log_server(8080)
# myls.stop_log_server()
print("Current logging tree after tests")
logging_tree.printout(node=None)
print()
print('stopping que')
mlp.stopQueue()
print('Main Test Done!')
context = dict(vars(logging))
context['handlers'] = mlp.handlers
evaluator = eVal.Evaluator(context, True)
try:
while True:
line = input('Enter source to evaluate or enter to exit:').strip()
if not line:
break
def _logging():
printout()
import json;
import logging;
import logging.config
import logging_tree
logging.config.dictConfig(json.load(open("conf_dict.json")))
log = logging.getLogger("")
child_A = logging.getLogger("A")
child_B = logging.getLogger("B")
child_B_A = logging.getLogger("B.A")
log.info("Now this is %s logging!","good")
child_A.info("Now this is more logging!")
log.warning("Now this is %s logging!","worrisome")
logging_tree.printout()
def main(reloadp=False, debug=False, showlogconf=False):
if showlogconf:
logging_tree.printout()
LOG.warn('upload.main() running')
app.run(debug=debug, use_reloader=reloadp, host='0.0.0.0', port=5050)
from rq import Queue
import RedisConfigHelper
import json
import logging
import boto
import os
from utils import *
import logging_tree
from redis import StrictRedis
logging.config.fileConfig("%s/%s" % (os.getcwd(),"logging.cfg"))
logger = logging.getLogger()
logging_tree.printout()
parser = argparse.ArgumentParser(description='Subscribes to a Redis PubSub Channel and Manage the MDM on Request')
parser.add_argument('-c','--channel', help='Channel Name', required=True)
parser.add_argument('-q','--queue', help='RedisQueue queue name to dispatch to', required=True)
args = vars(parser.parse_args())
ready = PubSub(redis.Redis(), args['channel'])
mdmqueue = Queue("MDM",connection=redis.Redis())
config = RedisConfigHelper.RedisConfigHelper()
r = StrictRedis()
def debug():
logging_tree.printout()
