def do_some_logs_multiprocessing(again): bxilog.cleanup() bxilog.basicConfig(filename=FILENAME, filemode='a', level=bxilog.LOWEST) while again.value: bxilog.out("Doing a simple log: %s", again) time.sleep(0.1) bxilog.out("Termination requested. Exiting.")
def test_non_existing_dir(self): """Test logging into a non existing tmpdir - this should raise an error""" tmpdir = tempfile.mkdtemp(".bxilog", "test_") os.rmdir(tmpdir) name = os.path.join(tmpdir, 'dummy.bxilog') bxilog.basicConfig(filename=name) # Raise an error because filename provided to basicConfig doesn't exist self.assertRaises(bxierr.BXICError, bxilog.output, "One log on non-existent (deleted) directory: %s", name) bxilog.cleanup() self.assertFalse(os.path.exists(name))
def test_existing_file(self): """Test logging into an existing file""" fd, name = tempfile.mkstemp(".bxilog", "test_") print("Overriding file output to " "%s for %s.%s()" % (name, __name__, BXILogTest.test_existing_file.__name__)) self.assertEquals(os.stat(name).st_size, 0) os.close(fd) bxilog.basicConfig(filename=name) self._check_log_produced(name, bxilog.output, "One log on file: %s", name) os.remove(name) bxilog.cleanup()
def threads_in_process(again): global __LOOP_AGAIN__ __LOOP_AGAIN__ = True bxilog.cleanup() bxilog.basicConfig(filename=FILENAME, filemode='a', level=bxilog.LOWEST) threads = [] for i in xrange(multiprocessing.cpu_count()): thread = threading.Thread(target=do_some_logs_threading) bxilog.out("Starting new thread") thread.start() threads.append(thread) while again.value: time.sleep(0.05) bxilog.out("Requesting termination of %s threads", len(threads)) __LOOP_AGAIN__ = False for thread in threads: try: thread.join(5) except Error as e: bxilog.out("Exception: %s", e)
def main(file_out, url, bind, sync_nb, logs_nb): config = { 'handlers': ['file', 'remote'], 'remote': { 'module': 'bxi.base.log.remote_handler', 'filters': ':all', 'url': url, 'bind': bind, }, 'file': { 'module': 'bxi.base.log.file_handler', 'filters': ':all', 'path': file_out, 'append': True, } } bxilog.set_config(config) nb = 0 nb += _do_log(0, logs_nb / 2) bxilog.cleanup() bxilog.set_config(config) nb += _do_log(logs_nb / 2, logs_nb) return nb
def tearDown(self): """Cleaning up for each test. """ bxilog.cleanup()
def _configure_log(parser): """ Configure the bxilog options for the given parser @param[inout] parser the parser to add options to """ def _add_common(target_parser): """ Add common options to the given target_parser @param[inout] target_parser the target_parser to add options to """ # Warning: do not introduce --log-STUFF unless STUFF is actually the name # of a bxilog handler. group = target_parser.add_argument_group('BXI Log options') group.add_argument("--loglevels", mustbeprinted=False, action=LogLevelsAction, help="Displays all log levels and exit.") group.add_argument( "--logoutput-default-config", dest='output_default_logcfg', action='store_true', mustbeprinted=False, help="Output the default logging configuration and exit.") group.add_argument( "--logcfgfile", mustbeprinted=False, metavar='logcfgfile', default=None, help="Logging configuration file. " " Use '--logoutput-default-config' to customize one to your " "own needs. Value: %(default)s.") return group def _add_others(target_parser, args, group, config): """ Add remaining options to the given target_parser @param[inout] target_parser the target parser to add options to @param[in] args the current set of parsed arguments @param[in] group the group in which the arguments must be set @param[in] config the configuration """ sections = find_logconfigs(bxilog_consolehandler.__name__, config) if len(sections) > 1: target_parser.error("Multiple instances of module %s is " "currently unsupported. Configuration: %s " % (bxilog_consolehandler.__name__, config)) if len(sections) == 0: console_handler = None else: console_handler = sections[0] console_section = config[console_handler] if 'filters' not in console_section: target_parser.error( "Bad logging configuration: 'filters' is missing " "in section '%s' of config %s" % (console_handler, config)) default = console_section['filters'] group.add_argument( "-l", "--log-%s-filters" % console_handler, mustbeprinted=False, metavar='log-%s-filters' % console_handler, envvar='BXILOG_%s_FILTERS' % console_handler.upper(), default=default, help="Define the logging filters for the " " %s handler " % console_handler + "Value: '%(default)s'. " "Logging filters are defined by the following " "format: logger_name_prefix:level[,prefix:level]*") group.add_argument("--quiet", action='store_true', mustbeprinted=True, help="Set log console filter to off.") if 'colors' not in console_section: default = DEFAULT_CONSOLE_COLORS else: default = console_section['colors'] group.add_argument( "--log-%s-colors" % console_handler, mustbeprinted=False, metavar='log-%s-colors' % console_handler, envvar='BXILOG_%s_COLORS' % console_handler.upper(), default=default, choices=list(bxilog_consolehandler.COLORS), help="Define the logging colors for the %s handler " % console_handler + "Value: '%(default)s'." + " choices=%s. " % list(bxilog_consolehandler.COLORS)) sections = find_logconfigs(bxilog_filehandler.__name__, config) for section in sections: conf = config[section] if 'filters' not in conf: target_parser.error( "Bad logging configuration: 'filters' is missing " "in section '%s' of config %s" % (section, config)) default = conf['filters'] if console_handler is not None: auto_help_msg = "If set to '%s', " % bxilog_filehandler.FILTERS_AUTO + \ "filters are automatically computed to " + \ "provide two levels more details than handler '%s' " % \ console_handler + \ "and at least error levels and above. " else: auto_help_msg = "" group.add_argument( "--log-%s-filters" % section, metavar='log-%s-filters' % section, mustbeprinted=False, envvar='BXILOG_%s_FILTERS' % section.upper(), default=default, help="Define the logging filters for the " "%s handler " % section + "of the default logging configuration. " + auto_help_msg + "The format is the one defined by " "console_filters option. Value: '%(default)s'. ") if 'path' not in conf: target_parser.error( "Bad logging configuration: 'path' is missing " "in section '%s' of config %s" % (section, config)) default = conf['path'] group.add_argument( "--log-%s-path" % section, metavar='PATH', envvar='BXILOGPATH', mustbeprinted=False, default=default, help="Define the destination file for the %s handler " % section + "Value: %(default)s") if 'append' not in conf: default = True else: default = conf['append'] group.add_argument( "--log-%s-append" % section, metavar='bool', mustbeprinted=False, default=default, help="When true, append to destination path of handler " "%s, instead of owerwriting. " % section + "Value: %(default)s") def _override_logconfig(config, known_args, parser): """ Override the given logging configuration with given known_args @param[inout] config the logging configuration @param[in] know_args known arguments """ def _override_kv(option, key, config, args): """ Override the config[key] value by the one given by args[option] @param[in] option the option name (long format) @param[in] key the key in the config file for the current handler @param[inout] config the configuration @param[in] known_args the currently known arguments """ # option has the following format: --log-handler-key=value if option.endswith(key): handler_name = option[len('log_'):option.index(key) - 1] assert handler_name in config # replace in config # print("Overriding: %s -> %s[%s]=%s" % # (option, handler_name, key, args[option])) config[handler_name][key] = args[option] args = vars(known_args) for option in args: _override_kv(option, 'filters', config, args) _override_kv(option, 'colors', config, args) _override_kv(option, 'path', config, args) _override_kv(option, 'append', config, args) # if --quiet option is provided, set output log level for console handlers # to minimal settings so that nothing is printed on stdout (nothing change # for stderr). if option is "quiet": if args[option] is True: sections = find_logconfigs(bxilog_consolehandler.__name__, config) if len(sections) > 1: parser.error( "Multiple instances of module %s is " "currently unsupported. Configuration: %s " % (bxilog_consolehandler.__name__, config)) if len(sections) == 1: option = "log_console_filters" key = 'filters' handler_name = option[len('log_'):option.index(key) - 1] args[option] = ":%s" % config[handler_name][ "stderr_level"] _override_kv(option, key, config, args) parser.add_argument('--help-logs', action=_LoggedHelpAction, default=posless.SUPPRESS, help=_('Show detailed logging options and exit')) group = _add_common(parser) known_args = parser.get_known_args()[0] if known_args is None: return baseconf = { 'handlers': ['console', 'file'], 'setsighandler': True, 'console': { 'module': bxilog_consolehandler.__name__, 'filters': ':output', 'stderr_level': 'WARNING', 'colors': '216_dark', }, 'file': { 'module': bxilog_filehandler.__name__, 'filters': 'auto', 'path': os.path.join(tempfile.gettempdir(), '%(prog)s') + '.bxilog', 'append': True, } } if known_args.output_default_logcfg: config = configobj.ConfigObj(infile=baseconf, interpolation=False) for line in config.write(): print(line) sys.exit(0) if known_args.logcfgfile is None: # The Environment and command line does not define a specific file # Use the one from the configuration default_logcfgfile = getdefaultvalue(parser, ['Defaults'], BXILOG_DEFAULT_CONFIGFILE_KEY, None) parser.set_defaults(logcfgfile=default_logcfgfile) infile = default_logcfgfile if infile is None: # Default case: no logging configuration given and no file found by default infile = baseconf logcfg_msg = "No logging configuration file given, using default." else: if not os.path.isabs(infile): # Find the absolute path from config-file... config_dir = os.path.dirname(parser.known_config_file) infile = os.path.join(config_dir, infile) if not os.path.exists(infile): parser.error("Logging configuration file " "not found: %s. " % infile + "Check your configuration " "from file: %s" % parser.known_config_file) logcfg_msg = "Using logging configuration file '%s' specified by '%s'" %\ (infile, parser.known_config_file) config = configobj.ConfigObj(infile=infile, interpolation=False) elif not os.path.exists(known_args.logcfgfile): parser.error("For option logcfgfile, provided file not found: %s" % known_args.logcfgfile) else: config = configobj.ConfigObj(infile=known_args.logcfgfile, interpolation=False) logcfg_msg = "Using logging configuration file '%s' specified by command line" %\ known_args.logcfgfile _add_others(parser, known_args, group, config) known_args = parser.get_known_args()[0] _override_logconfig(config, known_args, parser) logging.captureWarnings(True) if logging.is_configured(): logging.info("Reconfiguration of the logging system") logging.cleanup() logging.set_config(config) if parser.known_config_file is not None: _LOGGER.info("Configuration based on '%s': %s", parser.known_config_file, parser.config) else: _LOGGER.info("No configuration file found, using default values: %s", parser.config) _LOGGER.debug(logcfg_msg)
def _add_config(parser, default_config_dirname, default_config_filename, domain_name, cmd_config_file_suffix, configdir_envvar, configfile_envvar): """ Add the configuration options to the given parser @param[inout] parser the parser to add options to @param[in] default_config_dirname the default configuration directory @param[in] default_config_filename the default configuration file @param[in] domain_name the configuration domain @param[in] cmd_config_file_suffix the configuration file suffix """ if os.getuid() == 0: config_dir_prefix = '/etc/' else: config_dir_prefix = os.path.join(os.path.expanduser('~'), '.config') full_config_dir = os.path.join(config_dir_prefix, default_config_dirname) full_config_dir = os.getenv(configdir_envvar, full_config_dir) # First, we try to fetch a configuration for the command line cmd_config = _get_configfile(parser, full_config_dir, default_config_filename, domain_name, cmd_config_file_suffix) def _add_config_dir_arg(target_parser): """ Add the config_dir option to the given target_parser @param[inout] target_parser the parser to add the option to @param[in] known_args the current set of parsed arguments """ group = target_parser.add_argument_group('File Based Configuration' ' (domain: %s)' % domain_name) group.add_argument("--config-dir", help="The directory where the configuration file " "must be looked for." " Value: %(default)s. " "Environment variable: %(envvar)s", default=None, envvar=configdir_envvar, metavar="DIR") group.add_argument( "-C", "--config-file", mustbeprinted=False, help="The configuration file to use. If None, " "the file is taken from the configuration directory. " "Value: %(default)s. " "Environment variable: %(envvar)s", default=None, envvar=configfile_envvar, metavar="FILE") known_args = target_parser.get_known_args()[0] if known_args is None: target_parser.known_config_file = cmd_config return # Check if the --config-file option has been given if known_args.config_file is not None: default_config_dir = None default_config_file = known_args.config_file else: if known_args.config_dir is None: default_config_dir = full_config_dir default_config_file = cmd_config else: default_config_dir = known_args.config_dir candidate = _get_configfile(target_parser, default_config_dir, default_config_filename, domain_name, cmd_config_file_suffix) default_config_file = candidate parser.set_defaults(config_file=default_config_file, config_dir=default_config_dir) target_parser.known_config_file = default_config_file _add_config_dir_arg(parser) if os.path.exists(parser.known_config_file): try: parser.config = _get_config_from_file(parser.known_config_file) except IOError as err: parser.error('Configuration file error: %s' % err) logging.cleanup() except ConfigObjError as err: parser.error('Configuration file parsing error: %s' % err) logging.cleanup() else: parser.config = ConfigObj()
def tearDown(self): bxilog.cleanup() if os.path.exists(self.bxiconfigdir): rmtree(self.bxiconfigdir)
def test_def(self): # Hack around posless/argparse problem with argv parsing. SAVED_ARGV = sys.argv sys.argv = [sys.argv[0], '--config-dir', self.bxiconfigdir] parser = posless.ArgumentParser( formatter_class=parserconf.FilteredHelpFormatter) parserconf.addargs(parser, domain_name='bar') self._populate_parser(parser) args = parser.parse_args() suffix = os.path.join(self.bxiconfigdir, parserconf.DEFAULT_CONFIG_FILENAME) self.assertTrue(args.config_file.endswith(suffix), args.config_file) self.assertEquals(args.variable, 'Default_Value') self.assertEquals(args.stuff, 'Default_stuff') bxilog.cleanup() fileconf = os.path.join(self.bxiconfigdir, parserconf.DEFAULT_CONFIG_FILENAME) with open(fileconf, 'w') as f: f.write(""" [My Section] variable = 'foo' """) parser = posless.ArgumentParser( formatter_class=parserconf.FilteredHelpFormatter) parserconf.addargs(parser, domain_name='bar') self._populate_parser(parser) args = parser.parse_args() self.assertEquals(os.path.basename(args.config_file), parserconf.DEFAULT_CONFIG_FILENAME) self.assertEquals(args.variable, 'foo') self.assertEquals(args.stuff, 'Default_stuff') bxilog.cleanup() fileconf = os.path.join(self.bxiconfigdir, 'bar.conf') with open(fileconf, 'w') as f: f.write(""" [My Section] variable = 'bar' """) parser = posless.ArgumentParser( formatter_class=parserconf.FilteredHelpFormatter) parserconf.addargs(parser, domain_name='bar') self._populate_parser(parser) args = parser.parse_args() self.assertEquals(args.config_file, fileconf) self.assertEquals(args.variable, 'bar') self.assertEquals(args.stuff, 'Default_stuff') bxilog.cleanup() fileconf = os.path.join(self.bxiconfigdir, os.path.basename(sys.argv[0]) + '.conf') with open(fileconf, 'w') as f: f.write(""" [My Section] variable = 'baz' """) parser = posless.ArgumentParser( formatter_class=parserconf.FilteredHelpFormatter) parserconf.addargs(parser, domain_name='bar') self._populate_parser(parser) args = parser.parse_args() self.assertEquals(args.config_file, fileconf) self.assertEquals(args.variable, 'baz') self.assertEquals(args.stuff, 'Default_stuff') bxilog.cleanup() fileconf = os.path.join(self.bxiconfigdir, 'specific.conf') with open(fileconf, 'w') as f: f.write(""" [My Section] variable = 'spec' """) sys.argv = [sys.argv[0], '--config-file', fileconf] parser = posless.ArgumentParser( formatter_class=parserconf.FilteredHelpFormatter) parserconf.addargs(parser, domain_name='bar') self._populate_parser(parser) args = parser.parse_args() self.assertEquals(args.config_file, fileconf) self.assertEquals(args.variable, 'spec') self.assertEquals(args.stuff, 'Default_stuff') os.environ['VAR'] = 'env' bxilog.cleanup() parser = posless.ArgumentParser( formatter_class=parserconf.FilteredHelpFormatter) parserconf.addargs(parser, domain_name='bar') self._populate_parser(parser) args = parser.parse_args() self.assertEquals(args.config_file, fileconf) self.assertEquals(args.variable, 'env') self.assertEquals(args.stuff, 'Default_stuff') bxilog.cleanup() parser = posless.ArgumentParser( formatter_class=parserconf.FilteredHelpFormatter) sys.argv = [ sys.argv[0], '--config-file', fileconf, '--variable', 'command_line' ] parserconf.addargs(parser, domain_name='bar') self._populate_parser(parser) args = parser.parse_args() self.assertEquals(args.config_file, fileconf) self.assertEquals(args.variable, 'command_line') self.assertEquals(args.stuff, 'Default_stuff') sys.argv = SAVED_ARGV