def do_some_logs_multiprocessing(again):
bxilog.cleanup()
bxilog.basicConfig(filename=FILENAME, filemode='a', level=bxilog.LOWEST)
while again.value:
bxilog.out("Doing a simple log: %s", again)
time.sleep(0.1)
bxilog.out("Termination requested. Exiting.")
def test_non_existing_dir(self):
"""Test logging into a non existing tmpdir - this should raise an error"""
tmpdir = tempfile.mkdtemp(".bxilog", "test_")
os.rmdir(tmpdir)
name = os.path.join(tmpdir, 'dummy.bxilog')
bxilog.basicConfig(filename=name)
# Raise an error because filename provided to basicConfig doesn't exist
self.assertRaises(bxierr.BXICError, bxilog.output,
"One log on non-existent (deleted) directory: %s",
name)
bxilog.cleanup()
self.assertFalse(os.path.exists(name))
def test_existing_file(self):
"""Test logging into an existing file"""
fd, name = tempfile.mkstemp(".bxilog", "test_")
print("Overriding file output to "
"%s for %s.%s()" %
(name, __name__, BXILogTest.test_existing_file.__name__))
self.assertEquals(os.stat(name).st_size, 0)
os.close(fd)
bxilog.basicConfig(filename=name)
self._check_log_produced(name, bxilog.output, "One log on file: %s",
name)
os.remove(name)
bxilog.cleanup()
def threads_in_process(again):
global __LOOP_AGAIN__
__LOOP_AGAIN__ = True
bxilog.cleanup()
bxilog.basicConfig(filename=FILENAME, filemode='a', level=bxilog.LOWEST)
threads = []
for i in xrange(multiprocessing.cpu_count()):
thread = threading.Thread(target=do_some_logs_threading)
bxilog.out("Starting new thread")
thread.start()
threads.append(thread)
while again.value:
time.sleep(0.05)
bxilog.out("Requesting termination of %s threads", len(threads))
__LOOP_AGAIN__ = False
for thread in threads:
try:
thread.join(5)
except Error as e:
bxilog.out("Exception: %s", e)
def main(file_out, url, bind, sync_nb, logs_nb):
config = {
'handlers': ['file', 'remote'],
'remote': {
'module': 'bxi.base.log.remote_handler',
'filters': ':all',
'url': url,
'bind': bind,
},
'file': {
'module': 'bxi.base.log.file_handler',
'filters': ':all',
'path': file_out,
'append': True,
}
}
bxilog.set_config(config)
nb = 0
nb += _do_log(0, logs_nb / 2)
bxilog.cleanup()
bxilog.set_config(config)
nb += _do_log(logs_nb / 2, logs_nb)
return nb
def tearDown(self):
"""Cleaning up for each test.
"""
bxilog.cleanup()
def _configure_log(parser):
"""
Configure the bxilog options for the given parser
@param[inout] parser the parser to add options to
"""
def _add_common(target_parser):
"""
Add common options to the given target_parser
@param[inout] target_parser the target_parser to add options to
"""
# Warning: do not introduce --log-STUFF unless STUFF is actually the name
# of a bxilog handler.
group = target_parser.add_argument_group('BXI Log options')
group.add_argument("--loglevels",
mustbeprinted=False,
action=LogLevelsAction,
help="Displays all log levels and exit.")
group.add_argument(
"--logoutput-default-config",
dest='output_default_logcfg',
action='store_true',
mustbeprinted=False,
help="Output the default logging configuration and exit.")
group.add_argument(
"--logcfgfile",
mustbeprinted=False,
metavar='logcfgfile',
default=None,
help="Logging configuration file. "
" Use '--logoutput-default-config' to customize one to your "
"own needs. Value: %(default)s.")
return group
def _add_others(target_parser, args, group, config):
"""
Add remaining options to the given target_parser
@param[inout] target_parser the target parser to add options to
@param[in] args the current set of parsed arguments
@param[in] group the group in which the arguments must be set
@param[in] config the configuration
"""
sections = find_logconfigs(bxilog_consolehandler.__name__, config)
if len(sections) > 1:
target_parser.error("Multiple instances of module %s is "
"currently unsupported. Configuration: %s " %
(bxilog_consolehandler.__name__, config))
if len(sections) == 0:
console_handler = None
else:
console_handler = sections[0]
console_section = config[console_handler]
if 'filters' not in console_section:
target_parser.error(
"Bad logging configuration: 'filters' is missing "
"in section '%s' of config %s" % (console_handler, config))
default = console_section['filters']
group.add_argument(
"-l",
"--log-%s-filters" % console_handler,
mustbeprinted=False,
metavar='log-%s-filters' % console_handler,
envvar='BXILOG_%s_FILTERS' % console_handler.upper(),
default=default,
help="Define the logging filters for the "
" %s handler " % console_handler + "Value: '%(default)s'. "
"Logging filters are defined by the following "
"format: logger_name_prefix:level[,prefix:level]*")
group.add_argument("--quiet",
action='store_true',
mustbeprinted=True,
help="Set log console filter to off.")
if 'colors' not in console_section:
default = DEFAULT_CONSOLE_COLORS
else:
default = console_section['colors']
group.add_argument(
"--log-%s-colors" % console_handler,
mustbeprinted=False,
metavar='log-%s-colors' % console_handler,
envvar='BXILOG_%s_COLORS' % console_handler.upper(),
default=default,
choices=list(bxilog_consolehandler.COLORS),
help="Define the logging colors for the %s handler " %
console_handler + "Value: '%(default)s'." +
" choices=%s. " % list(bxilog_consolehandler.COLORS))
sections = find_logconfigs(bxilog_filehandler.__name__, config)
for section in sections:
conf = config[section]
if 'filters' not in conf:
target_parser.error(
"Bad logging configuration: 'filters' is missing "
"in section '%s' of config %s" % (section, config))
default = conf['filters']
if console_handler is not None:
auto_help_msg = "If set to '%s', " % bxilog_filehandler.FILTERS_AUTO + \
"filters are automatically computed to " + \
"provide two levels more details than handler '%s' " % \
console_handler + \
"and at least error levels and above. "
else:
auto_help_msg = ""
group.add_argument(
"--log-%s-filters" % section,
metavar='log-%s-filters' % section,
mustbeprinted=False,
envvar='BXILOG_%s_FILTERS' % section.upper(),
default=default,
help="Define the logging filters for the "
"%s handler " % section +
"of the default logging configuration. " + auto_help_msg +
"The format is the one defined by "
"console_filters option. Value: '%(default)s'. ")
if 'path' not in conf:
target_parser.error(
"Bad logging configuration: 'path' is missing "
"in section '%s' of config %s" % (section, config))
default = conf['path']
group.add_argument(
"--log-%s-path" % section,
metavar='PATH',
envvar='BXILOGPATH',
mustbeprinted=False,
default=default,
help="Define the destination file for the %s handler " %
section + "Value: %(default)s")
if 'append' not in conf:
default = True
else:
default = conf['append']
group.add_argument(
"--log-%s-append" % section,
metavar='bool',
mustbeprinted=False,
default=default,
help="When true, append to destination path of handler "
"%s, instead of owerwriting. " % section +
"Value: %(default)s")
def _override_logconfig(config, known_args, parser):
"""
Override the given logging configuration with given known_args
@param[inout] config the logging configuration
@param[in] know_args known arguments
"""
def _override_kv(option, key, config, args):
"""
Override the config[key] value by the one given by args[option]
@param[in] option the option name (long format)
@param[in] key the key in the config file for the current handler
@param[inout] config the configuration
@param[in] known_args the currently known arguments
"""
# option has the following format: --log-handler-key=value
if option.endswith(key):
handler_name = option[len('log_'):option.index(key) - 1]
assert handler_name in config
# replace in config
# print("Overriding: %s -> %s[%s]=%s" %
# (option, handler_name, key, args[option]))
config[handler_name][key] = args[option]
args = vars(known_args)
for option in args:
_override_kv(option, 'filters', config, args)
_override_kv(option, 'colors', config, args)
_override_kv(option, 'path', config, args)
_override_kv(option, 'append', config, args)
# if --quiet option is provided, set output log level for console handlers
# to minimal settings so that nothing is printed on stdout (nothing change
# for stderr).
if option is "quiet":
if args[option] is True:
sections = find_logconfigs(bxilog_consolehandler.__name__,
config)
if len(sections) > 1:
parser.error(
"Multiple instances of module %s is "
"currently unsupported. Configuration: %s " %
(bxilog_consolehandler.__name__, config))
if len(sections) == 1:
option = "log_console_filters"
key = 'filters'
handler_name = option[len('log_'):option.index(key) -
1]
args[option] = ":%s" % config[handler_name][
"stderr_level"]
_override_kv(option, key, config, args)
parser.add_argument('--help-logs',
action=_LoggedHelpAction,
default=posless.SUPPRESS,
help=_('Show detailed logging options and exit'))
group = _add_common(parser)
known_args = parser.get_known_args()[0]
if known_args is None:
return
baseconf = {
'handlers': ['console', 'file'],
'setsighandler': True,
'console': {
'module': bxilog_consolehandler.__name__,
'filters': ':output',
'stderr_level': 'WARNING',
'colors': '216_dark',
},
'file': {
'module': bxilog_filehandler.__name__,
'filters': 'auto',
'path':
os.path.join(tempfile.gettempdir(), '%(prog)s') + '.bxilog',
'append': True,
}
}
if known_args.output_default_logcfg:
config = configobj.ConfigObj(infile=baseconf, interpolation=False)
for line in config.write():
print(line)
sys.exit(0)
if known_args.logcfgfile is None:
# The Environment and command line does not define a specific file
# Use the one from the configuration
default_logcfgfile = getdefaultvalue(parser, ['Defaults'],
BXILOG_DEFAULT_CONFIGFILE_KEY,
None)
parser.set_defaults(logcfgfile=default_logcfgfile)
infile = default_logcfgfile
if infile is None:
# Default case: no logging configuration given and no file found by default
infile = baseconf
logcfg_msg = "No logging configuration file given, using default."
else:
if not os.path.isabs(infile):
# Find the absolute path from config-file...
config_dir = os.path.dirname(parser.known_config_file)
infile = os.path.join(config_dir, infile)
if not os.path.exists(infile):
parser.error("Logging configuration file "
"not found: %s. " % infile +
"Check your configuration "
"from file: %s" % parser.known_config_file)
logcfg_msg = "Using logging configuration file '%s' specified by '%s'" %\
(infile, parser.known_config_file)
config = configobj.ConfigObj(infile=infile, interpolation=False)
elif not os.path.exists(known_args.logcfgfile):
parser.error("For option logcfgfile, provided file not found: %s" %
known_args.logcfgfile)
else:
config = configobj.ConfigObj(infile=known_args.logcfgfile,
interpolation=False)
logcfg_msg = "Using logging configuration file '%s' specified by command line" %\
known_args.logcfgfile
_add_others(parser, known_args, group, config)
known_args = parser.get_known_args()[0]
_override_logconfig(config, known_args, parser)
logging.captureWarnings(True)
if logging.is_configured():
logging.info("Reconfiguration of the logging system")
logging.cleanup()
logging.set_config(config)
if parser.known_config_file is not None:
_LOGGER.info("Configuration based on '%s': %s",
parser.known_config_file, parser.config)
else:
_LOGGER.info("No configuration file found, using default values: %s",
parser.config)
_LOGGER.debug(logcfg_msg)
def _add_config(parser, default_config_dirname, default_config_filename,
domain_name, cmd_config_file_suffix, configdir_envvar,
configfile_envvar):
"""
Add the configuration options to the given parser
@param[inout] parser the parser to add options to
@param[in] default_config_dirname the default configuration directory
@param[in] default_config_filename the default configuration file
@param[in] domain_name the configuration domain
@param[in] cmd_config_file_suffix the configuration file suffix
"""
if os.getuid() == 0:
config_dir_prefix = '/etc/'
else:
config_dir_prefix = os.path.join(os.path.expanduser('~'), '.config')
full_config_dir = os.path.join(config_dir_prefix, default_config_dirname)
full_config_dir = os.getenv(configdir_envvar, full_config_dir)
# First, we try to fetch a configuration for the command line
cmd_config = _get_configfile(parser, full_config_dir,
default_config_filename, domain_name,
cmd_config_file_suffix)
def _add_config_dir_arg(target_parser):
"""
Add the config_dir option to the given target_parser
@param[inout] target_parser the parser to add the option to
@param[in] known_args the current set of parsed arguments
"""
group = target_parser.add_argument_group('File Based Configuration'
' (domain: %s)' % domain_name)
group.add_argument("--config-dir",
help="The directory where the configuration file "
"must be looked for."
" Value: %(default)s. "
"Environment variable: %(envvar)s",
default=None,
envvar=configdir_envvar,
metavar="DIR")
group.add_argument(
"-C",
"--config-file",
mustbeprinted=False,
help="The configuration file to use. If None, "
"the file is taken from the configuration directory. "
"Value: %(default)s. "
"Environment variable: %(envvar)s",
default=None,
envvar=configfile_envvar,
metavar="FILE")
known_args = target_parser.get_known_args()[0]
if known_args is None:
target_parser.known_config_file = cmd_config
return
# Check if the --config-file option has been given
if known_args.config_file is not None:
default_config_dir = None
default_config_file = known_args.config_file
else:
if known_args.config_dir is None:
default_config_dir = full_config_dir
default_config_file = cmd_config
else:
default_config_dir = known_args.config_dir
candidate = _get_configfile(target_parser, default_config_dir,
default_config_filename,
domain_name,
cmd_config_file_suffix)
default_config_file = candidate
parser.set_defaults(config_file=default_config_file,
config_dir=default_config_dir)
target_parser.known_config_file = default_config_file
_add_config_dir_arg(parser)
if os.path.exists(parser.known_config_file):
try:
parser.config = _get_config_from_file(parser.known_config_file)
except IOError as err:
parser.error('Configuration file error: %s' % err)
logging.cleanup()
except ConfigObjError as err:
parser.error('Configuration file parsing error: %s' % err)
logging.cleanup()
else:
parser.config = ConfigObj()
def tearDown(self):
bxilog.cleanup()
if os.path.exists(self.bxiconfigdir):
rmtree(self.bxiconfigdir)
def test_def(self):
# Hack around posless/argparse problem with argv parsing.
SAVED_ARGV = sys.argv
sys.argv = [sys.argv[0], '--config-dir', self.bxiconfigdir]
parser = posless.ArgumentParser(
formatter_class=parserconf.FilteredHelpFormatter)
parserconf.addargs(parser, domain_name='bar')
self._populate_parser(parser)
args = parser.parse_args()
suffix = os.path.join(self.bxiconfigdir,
parserconf.DEFAULT_CONFIG_FILENAME)
self.assertTrue(args.config_file.endswith(suffix), args.config_file)
self.assertEquals(args.variable, 'Default_Value')
self.assertEquals(args.stuff, 'Default_stuff')
bxilog.cleanup()
fileconf = os.path.join(self.bxiconfigdir,
parserconf.DEFAULT_CONFIG_FILENAME)
with open(fileconf, 'w') as f:
f.write("""
[My Section]
variable = 'foo'
""")
parser = posless.ArgumentParser(
formatter_class=parserconf.FilteredHelpFormatter)
parserconf.addargs(parser, domain_name='bar')
self._populate_parser(parser)
args = parser.parse_args()
self.assertEquals(os.path.basename(args.config_file),
parserconf.DEFAULT_CONFIG_FILENAME)
self.assertEquals(args.variable, 'foo')
self.assertEquals(args.stuff, 'Default_stuff')
bxilog.cleanup()
fileconf = os.path.join(self.bxiconfigdir, 'bar.conf')
with open(fileconf, 'w') as f:
f.write("""
[My Section]
variable = 'bar'
""")
parser = posless.ArgumentParser(
formatter_class=parserconf.FilteredHelpFormatter)
parserconf.addargs(parser, domain_name='bar')
self._populate_parser(parser)
args = parser.parse_args()
self.assertEquals(args.config_file, fileconf)
self.assertEquals(args.variable, 'bar')
self.assertEquals(args.stuff, 'Default_stuff')
bxilog.cleanup()
fileconf = os.path.join(self.bxiconfigdir,
os.path.basename(sys.argv[0]) + '.conf')
with open(fileconf, 'w') as f:
f.write("""
[My Section]
variable = 'baz'
""")
parser = posless.ArgumentParser(
formatter_class=parserconf.FilteredHelpFormatter)
parserconf.addargs(parser, domain_name='bar')
self._populate_parser(parser)
args = parser.parse_args()
self.assertEquals(args.config_file, fileconf)
self.assertEquals(args.variable, 'baz')
self.assertEquals(args.stuff, 'Default_stuff')
bxilog.cleanup()
fileconf = os.path.join(self.bxiconfigdir, 'specific.conf')
with open(fileconf, 'w') as f:
f.write("""
[My Section]
variable = 'spec'
""")
sys.argv = [sys.argv[0], '--config-file', fileconf]
parser = posless.ArgumentParser(
formatter_class=parserconf.FilteredHelpFormatter)
parserconf.addargs(parser, domain_name='bar')
self._populate_parser(parser)
args = parser.parse_args()
self.assertEquals(args.config_file, fileconf)
self.assertEquals(args.variable, 'spec')
self.assertEquals(args.stuff, 'Default_stuff')
os.environ['VAR'] = 'env'
bxilog.cleanup()
parser = posless.ArgumentParser(
formatter_class=parserconf.FilteredHelpFormatter)
parserconf.addargs(parser, domain_name='bar')
self._populate_parser(parser)
args = parser.parse_args()
self.assertEquals(args.config_file, fileconf)
self.assertEquals(args.variable, 'env')
self.assertEquals(args.stuff, 'Default_stuff')
bxilog.cleanup()
parser = posless.ArgumentParser(
formatter_class=parserconf.FilteredHelpFormatter)
sys.argv = [
sys.argv[0], '--config-file', fileconf, '--variable',
'command_line'
]
parserconf.addargs(parser, domain_name='bar')
self._populate_parser(parser)
args = parser.parse_args()
self.assertEquals(args.config_file, fileconf)
self.assertEquals(args.variable, 'command_line')
self.assertEquals(args.stuff, 'Default_stuff')
sys.argv = SAVED_ARGV
