def _print_label_stats(self, stats, set_name): values = list(stats.values()) if sum(values) == 0: self.info("No %s labels specified", set_name) return mean = int(numpy.mean(values)) stddev = int(numpy.std(values)) lmin = numpy.min(values) amin = list(stats.keys())[numpy.argmin(values)] lmax = numpy.max(values) amax = list(stats.keys())[numpy.argmax(values)] if has_colors() and stddev > mean / 10: endstr = "\033[0m" if stddev > mean / 2: openstr = "\033[1;31m" # red else: openstr = "\033[1;33m" # yellow else: openstr = endstr = "" self.info( u"%s label cardinalities: min: %d (\"%s\"), max: %d (\"%s\"), avg:" u" %d, %sσ: %d (%d%%)%s", set_name, lmin, amin, lmax, amax, mean, openstr, stddev, stddev * 100 // mean, endstr) if not self.logger.isEnabledFor(logging.DEBUG): return total = sum(values) table = PrettyTable("Label", "Cardinality", "%", "Histogram") table.align["Cardinality"] = "r" table.align["%"] = "r" table.align["Histogram"] = "l" for k, v in stats.items(): table.add_row(k, v, "%.1f" % (v * 100 / total), "*" * (v * 25 // lmax)) self.debug("Detailed %s label stats:\n%s", set_name, table)
def setup_logging(level): if Logger.SET_UP: raise Logger.LoggerHasBeenAlreadySetUp() Logger.SET_UP = True Logger.ensure_utf8_streams() # Set basic log level logging.basicConfig(level=level, stream=sys.stdout) # Override the global log level and the output stream in case they have # been already changed previously root_logger = logging.getLogger() root_logger.level = level root_logger.handlers[0].stream = sys.stdout ProgressBar().logger.level = level # Turn on colors in case of an interactive out tty or IPython if has_colors(): root = logging.getLogger() handler = root.handlers[0] handler.setFormatter(Logger.ColorFormatter())
def redirect_all_logging_to_file(file_name, max_bytes=1024 * 1024, backups=1): handler = logging.handlers.RotatingFileHandler( filename=file_name, maxBytes=max_bytes, backupCount=backups, encoding="utf-8" ) formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(name)s: " "%(message)s", "%Y-%m-%d %H:%M:%S") handler.setFormatter(formatter) logging.getLogger("Logger").info("Will save the logs to %s", file_name) if not has_colors(): logging.getLogger().handlers[0] = handler sys.stderr.flush() stderr = open("%s.stderr%s" % os.path.splitext(file_name), 'a', encoding="utf-8") redirect_stream(sys.stderr, stderr) sys.stderr = stderr else: logging.getLogger().handlers.append(handler) logging.getLogger().addFilter(handler) logging.getLogger("Logger").info("Continuing to log in %s", file_name)
def redirect_all_logging_to_file(file_name, max_bytes=1024 * 1024, backups=1): handler = logging.handlers.RotatingFileHandler(filename=file_name, maxBytes=max_bytes, backupCount=backups, encoding="utf-8") formatter = logging.Formatter( "%(asctime)s:%(levelname)s:%(name)s: " "%(message)s", "%Y-%m-%d %H:%M:%S") handler.setFormatter(formatter) logging.getLogger("Logger").info("Will save the logs to %s", file_name) if not has_colors(): logging.getLogger().handlers[0] = handler sys.stderr.flush() stderr = open("%s.stderr%s" % os.path.splitext(file_name), 'a', encoding="utf-8") redirect_stream(sys.stderr, stderr) sys.stderr = stderr else: logging.getLogger().handlers.append(handler) logging.getLogger().addFilter(handler) logging.getLogger("Logger").info("Continuing to log in %s", file_name)
class CommandLineBase(object): """ Start point of any VELES engine executions. """ LOGO_PLAIN = veles.__logo__ LOGO_COLORED = "\033" r"[1;32m _ _ _____ _ _____ _____ " \ "\033[0m\n" \ "\033" r"[1;32m| | | | ___| | | ___/ ___| " \ "\033[0m" + \ (" Version \033[1;36m%s\033[0m" % veles.__version__) + \ (" %s\n" % formatdate(veles.__date__, True)) + \ "\033" r"[1;32m| | | | |__ | | | |__ \ `--. " \ "\033[0m\033[0;37m %s\033[0m\n" % veles.__logo_ext__[0] + \ "\033" r"[1;32m| | | | __|| | | __| `--. \ " "\033[0m" \ "\033[0;37m %s" % veles.__logo_ext__[1] + \ "\033[0m\n" \ "\033" r"[1;32m\ \_/ / |___| |___| |___/\__/ / " "\033[0m" \ "\033[0;37m %s" % veles.__logo_ext__[2] + \ "\033[0m\n" \ "\033" r"[1;32m \___/\____/\_____|____/\____/ " "\033[0m" \ "\033[0;37m %s\033[0m\n" % veles.__logo_ext__[3] LOGO = LOGO_COLORED if has_colors() else LOGO_PLAIN DRY_RUN_CHOICES = ["load", "init", "exec", "no"] LOG_LEVEL_MAP = { "debug": logging.DEBUG, "info": logging.INFO, "warning": logging.WARNING, "error": logging.ERROR } SPECIAL_OPTS = [ "--help", "--html-help", "--version", "--frontend", "--dump-config" ] _argv = tuple() class SortingRawDescriptionHelpFormatter(RawDescriptionHelpFormatter): def add_arguments(self, actions): actions = sorted(actions, key=lambda x: x.dest) super(CommandLineBase.SortingRawDescriptionHelpFormatter, self).add_arguments(actions) @staticmethod def init_parser(sphinx=False, ignore_conflicts=False): """ Creates the command line argument parser. """ parser = ArgumentParser( description=CommandLineBase.LOGO if not sphinx else "", formatter_class=CommandLineBase.SortingRawDescriptionHelpFormatter) for cls in CommandLineArgumentsRegistry.classes: try: parser = cls.init_parser(parser=parser) except ArgumentError as e: if not ignore_conflicts: raise from_none(e) parser.add_argument("--no-logo", default=False, help="Do not print VELES version, copyright and " "other information on startup.", action='store_true') parser.add_argument("--version", action="store_true", help="Print version number, date, commit hash and " "exit.") parser.add_argument("--html-help", action="store_true", help="Open VELES help in your web browser.") parser.add_argument( "--frontend", action="store_true", help="Open VELES command line frontend in the default web browser " "and run the composed line.") parser.add_argument("-v", "--verbosity", type=str, default="info", choices=CommandLineBase.LOG_LEVEL_MAP.keys(), help="Set the logging verbosity level.") parser.add_argument("--debug", type=str, default="", help="Set DEBUG logging level for these classes " "(separated by comma)") parser.add_argument("--debug-pickle", default=False, help="Turn on pickle diagnostics.", action='store_true') parser.add_argument("-r", "--random-seed", type=str, default="/dev/urandom:16", help="Set the random generator seed, e.g. " "veles/samples/seed:1024,:1024 or " "/dev/urandom:16:uint32 or " "hex string with even number of digits") parser.add_argument('-w', '--snapshot', default="", help='workflow snapshot') parser.add_argument("--dump-config", default=False, help="Print the initial global configuration", action='store_true') parser.add_argument("--dry-run", default="no", choices=CommandLineBase.DRY_RUN_CHOICES, help="no: normal work; load: stop before loading/" "creating the workflow; init: stop before " "workflow initialization; exec: stop before " "workflow execution.") parser.add_argument("--visualize", default=False, help="initialize, but do not run the loaded " "model, show workflow graph and plots", action='store_true') parser.add_argument( "--optimize", help="Perform optimization of the model's parameters using the " "genetic algorithm. Format: <size>[:<generations>], where " "<size> is the number of species in the population (if not " "sure, set it to 50) and <generations> is the optional limit " "of evaluated generations. If <generations> is not set, " "the optimization will continue until there is no fitness " "improvement.") parser.add_argument( "--ensemble-train", help="Parameters to assemble the ensemble of trained models. " "Format is <size>:<ratio>, where size is the number of models" " to train and ratio is the part of the training set to use " "during the training for each model (picked randomly). The " "models' evaluation results will be written to --result-file." " They include achieved metric values and outputs on the test" " dataset (loader must support test mode).") parser.add_argument( "--ensemble-test", help="Test the trained ensemble (see --ensemble-train). The value " "of this argument must be a path to --result-file with the " "ensemble_train definition. --test is ignored.") parser.add_argument("--workflow-graph", default="", help="Save workflow graph to file.") parser.add_argument("--dump-unit-attributes", default="no", help="Print unit __dict__-s after workflow " "initialization, excluding large numpy arrays" " if \"pretty\" is chosen.", choices=['no', 'pretty', 'all']) parser.add_argument('workflow', help='Path to Python script with the VELES model.' ).pretty_name = "workflow file" parser.add_argument( 'config', help="Path to the configuration file " "(pass \"-\" to make it <workflow>_config.py, " "pass empty to ignore).").pretty_name = "configuration file" arg = parser.add_argument( 'config_list', help="Configuration overrides separated by a whitespace, for " "example: \nroot.global_alpha=0.006\n " "root.snapshot_prefix='test_pr'", nargs='*', metavar="key=value") arg.pretty_name = "override configuration" arg.important = True parser.add_argument("-b", "--background", default=False, help="Run in background as a daemon.", action='store_true') try: class NoEscapeCompleter(argcomplete.CompletionFinder): def quote_completions(self, completions, *args, **kwargs): return completions NoEscapeCompleter()(parser) # pylint: disable=E1102 except: pass return parser @staticmethod def map_parser_keyword_arguments(): parser = CommandLineBase.init_parser() return {action.dest: action for action in parser._actions} @classproperty def argv(cls): if cls._argv is None: cls.setup_argv() return cls._argv @staticmethod def setup_argv(sys_argv=True, reset_argv=False, *args, **kwargs): argv = list(CommandLineBase._argv) args = list(args) if reset_argv: del argv[:] if len(argv) == 0: if sys_argv: argv.extend(sys.argv[1:]) if len(kwargs) > 0: available = CommandLineBase.map_parser_keyword_arguments() if not set(kwargs).issubset(set(available)): raise ValueError( "The following keyword arguments are not supported: " "%s" % ",".join(set(kwargs) - set(available))) for key, val in kwargs.items(): action = available[key] if len(action.option_strings) > 0: argv.append(action.option_strings[-1]) if not isinstance(action, _StoreConstAction): argv.append(str(val)) else: args.append(str(val)) argv.extend(args) if not sys_argv: assert len(args) >= 2 CommandLineBase._argv = tuple(argv)