def _setup_cli_logging(self): config = self._config terminal_reporter = config.pluginmanager.get_plugin("terminalreporter") if terminal_reporter is None: # terminal reporter is disabled e.g. by pytest-xdist. return capture_manager = config.pluginmanager.get_plugin("capturemanager") # if capturemanager plugin is disabled, live logging still works. log_cli_handler = _LiveLoggingStreamHandler(terminal_reporter, capture_manager) log_cli_format = get_option_ini(config, "log_cli_format", "log_format") log_cli_date_format = get_option_ini( config, "log_cli_date_format", "log_date_format" ) if ( config.option.color != "no" and ColoredLevelFormatter.LEVELNAME_FMT_REGEX.search(log_cli_format) ): log_cli_formatter = ColoredLevelFormatter( create_terminal_writer(config), log_cli_format, datefmt=log_cli_date_format, ) else: log_cli_formatter = logging.Formatter( log_cli_format, datefmt=log_cli_date_format ) log_cli_level = get_actual_log_level(config, "log_cli_level", "log_level") self.log_cli_handler = log_cli_handler self.live_logs_context = lambda: catching_logs( log_cli_handler, formatter=log_cli_formatter, level=log_cli_level )
def _create_formatter(self, log_format, log_date_format): # color option doesn't exist if terminal plugin is disabled color = getattr(self._config.option, "color", "no") if color != "no" and ColoredLevelFormatter.LEVELNAME_FMT_REGEX.search( log_format ): formatter = ColoredLevelFormatter( create_terminal_writer(self._config), log_format, log_date_format ) else: formatter = logging.Formatter(log_format, log_date_format) if not six.PY2: formatter._style = PercentStyleMultiline(formatter._style._fmt) return formatter
def _setup_cli_logging(self): """Sets up the handler and logger for the Live Logs feature, if enabled. This must be done right before starting the loop so we can access the terminal reporter plugin. """ terminal_reporter = self._config.pluginmanager.get_plugin('terminalreporter') if self._config.getini('log_cli') and terminal_reporter is not None: capture_manager = self._config.pluginmanager.get_plugin('capturemanager') log_cli_handler = _LiveLoggingStreamHandler(terminal_reporter, capture_manager) log_cli_format = get_option_ini(self._config, 'log_cli_format', 'log_format') log_cli_date_format = get_option_ini(self._config, 'log_cli_date_format', 'log_date_format') if self._config.option.color != 'no' and ColoredLevelFormatter.LEVELNAME_FMT_REGEX.search(log_cli_format): log_cli_formatter = ColoredLevelFormatter(create_terminal_writer(self._config), log_cli_format, datefmt=log_cli_date_format) else: log_cli_formatter = logging.Formatter(log_cli_format, datefmt=log_cli_date_format) log_cli_level = get_actual_log_level(self._config, 'log_cli_level', 'log_level') self.log_cli_handler = log_cli_handler self.live_logs_context = catching_logs(log_cli_handler, formatter=log_cli_formatter, level=log_cli_level) else: self.live_logs_context = _dummy_context_manager()
def pytest_terminal_summary_main(tr, id): """ Generate multiple reports at the end of test suite run - each report goes into a dedicated file in the current directory. The report files are prefixed with the test suite name. This function emulates --duration and -rA pytest arguments. This function is to be called from `conftest.py` via `pytest_terminal_summary` wrapper that has to be defined there. Args: - tr: `terminalreporter` passed from `conftest.py` - id: unique id like `tests` or `examples` that will be incorporated into the final reports filenames - this is needed as some jobs have multiple runs of pytest, so we can't have them overwrite each other. NB: this functions taps into a private _pytest API and while unlikely, it could break should pytest do internal changes - also it calls default internal methods of terminalreporter which can be hijacked by various `pytest-` plugins and interfere. """ from _pytest.config import create_terminal_writer if not len(id): id = "tests" config = tr.config orig_writer = config.get_terminal_writer() orig_tbstyle = config.option.tbstyle orig_reportchars = tr.reportchars dir = "reports" Path(dir).mkdir(parents=True, exist_ok=True) report_files = { k: f"{dir}/{id}_{k}.txt" for k in [ "durations", "errors", "failures_long", "failures_short", "failures_line", "passes", "stats", "summary_short", "warnings", ] } # custom durations report # note: there is no need to call pytest --durations=XX to get this separate report # adapted from https://github.com/pytest-dev/pytest/blob/897f151e/src/_pytest/runner.py#L66 dlist = [] for replist in tr.stats.values(): for rep in replist: if hasattr(rep, "duration"): dlist.append(rep) if dlist: dlist.sort(key=lambda x: x.duration, reverse=True) with open(report_files["durations"], "w") as f: durations_min = 0.05 # sec f.write("slowest durations\n") for i, rep in enumerate(dlist): if rep.duration < durations_min: f.write(f"{len(dlist)-i} durations < {durations_min} secs were omitted") break f.write(f"{rep.duration:02.2f}s {rep.when:<8} {rep.nodeid}\n") def summary_failures_short(tr): # expecting that the reports were --tb=long (default) so we chop them off here to the last frame reports = tr.getreports("failed") if not reports: return tr.write_sep("=", "FAILURES SHORT STACK") for rep in reports: msg = tr._getfailureheadline(rep) tr.write_sep("_", msg, red=True, bold=True) # chop off the optional leading extra frames, leaving only the last one longrepr = re.sub(r".*_ _ _ (_ ){10,}_ _ ", "", rep.longreprtext, 0, re.M | re.S) tr._tw.line(longrepr) # note: not printing out any rep.sections to keep the report short # use ready-made report funcs, we are just hijacking the filehandle to log to a dedicated file each # adapted from https://github.com/pytest-dev/pytest/blob/897f151e/src/_pytest/terminal.py#L814 # note: some pytest plugins may interfere by hijacking the default `terminalreporter` (e.g. # pytest-instafail does that) # report failures with line/short/long styles config.option.tbstyle = "auto" # full tb with open(report_files["failures_long"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_failures() # config.option.tbstyle = "short" # short tb with open(report_files["failures_short"], "w") as f: tr._tw = create_terminal_writer(config, f) summary_failures_short(tr) config.option.tbstyle = "line" # one line per error with open(report_files["failures_line"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_failures() with open(report_files["errors"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_errors() with open(report_files["warnings"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_warnings() # normal warnings tr.summary_warnings() # final warnings tr.reportchars = "wPpsxXEf" # emulate -rA (used in summary_passes() and short_test_summary()) with open(report_files["passes"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_passes() with open(report_files["summary_short"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.short_test_summary() with open(report_files["stats"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_stats() # restore: tr._tw = orig_writer tr.reportchars = orig_reportchars config.option.tbstyle = orig_tbstyle
def pytest_terminal_summary_main(tr, id): """ Generate multiple reports at the end of test suite run - each report goes into a dedicated file in the current directory. The report files are prefixed with the test suite name. This function emulates --duration and -rA pytest arguments. This function is to be called from `conftest.py` via `pytest_terminal_summary` wrapper that has to be defined there. Args: - tr: `terminalreporter` passed from `conftest.py` - id: unique id like `tests` or `examples` that will be incorporated into the final reports filenames - this is needed as some jobs have multiple runs of pytest, so we can't have them overwrite each other. NB: this functions taps into a private _pytest API and while unlikely, it could break should pytest do internal changes - also it calls default internal methods of terminalreporter which can be hijacked by various `pytest-` plugins and interfere. """ from _pytest.config import create_terminal_writer if not len(id): id = "tests" config = tr.config orig_writer = config.get_terminal_writer() orig_tbstyle = config.option.tbstyle orig_reportchars = tr.reportchars report_files = dict( durations="durations", short_summary="short_summary", summary_errors="errors", summary_failures="failures", summary_warnings="warnings", summary_passes="passes", summary_stats="stats", ) dir = "reports" Path(dir).mkdir(parents=True, exist_ok=True) report_files.update( (k, f"{dir}/report_{id}_{v}.txt") for k, v in report_files.items()) # custom durations report # note: there is no need to call pytest --durations=XX to get this separate report # adapted from https://github.com/pytest-dev/pytest/blob/897f151e/src/_pytest/runner.py#L66 dlist = [] for replist in tr.stats.values(): for rep in replist: if hasattr(rep, "duration"): dlist.append(rep) if dlist: dlist.sort(key=lambda x: x.duration, reverse=True) with open(report_files["durations"], "w") as f: durations_min = 0.05 # sec f.write("slowest durations\n") for i, rep in enumerate(dlist): if rep.duration < durations_min: f.write( f"{len(dlist)-i} durations < {durations_min} secs were omitted" ) break f.write(f"{rep.duration:02.2f}s {rep.when:<8} {rep.nodeid}\n") # use ready-made report funcs, we are just hijacking the filehandle to log to a dedicated file each # adapted from https://github.com/pytest-dev/pytest/blob/897f151e/src/_pytest/terminal.py#L814 # note: some pytest plugins may interfere by hijacking the default `terminalreporter` (e.g. # pytest-instafail does that) tr.reportchars = "wPpsxXEf" # emulate -rA (used in summary_passes() and short_test_summary()) config.option.tbstyle = "auto" with open(report_files["summary_failures"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_failures() with open(report_files["summary_errors"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_errors() with open(report_files["summary_warnings"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_warnings() # normal warnings tr.summary_warnings() # final warnings with open(report_files["summary_passes"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_passes() with open(report_files["short_summary"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.short_test_summary() with open(report_files["summary_stats"], "w") as f: tr._tw = create_terminal_writer(config, f) tr.summary_stats() # restore: tr._tw = orig_writer tr.reportchars = orig_reportchars config.option.tbstyle = orig_tbstyle
def __init__(self, config): from _pytest.config import create_terminal_writer self.filename = '.last-run' self.results = {} self.ran_any = False self._tw = create_terminal_writer(config, sys.stdout)
def _create_terminal_writer(config: Config, _file: Optional[TextIO] = None) -> TerminalWriter: file = output if output is not None else get_sink_io() return create_terminal_writer(config, file)
def pytest_collection_modifyitems(session, config, items): yield group_count = config.getoption('test-group-count') group_id = config.getoption('test-group') seed = config.getoption('random-seed') prescheduled_path = config.getoption('prescheduled') if not group_count or not group_id: return test_dict = {item.name: item for item in items} original_order = {item: index for index, item in enumerate(items)} # schema: prescheduled_data[node_id] = [*test_names] prescheduled_data = [[] for _ in range(group_count)] if prescheduled_path: try: with open(prescheduled_path, 'r') as f: prescheduled_data = json.load(f) if len(prescheduled_data) != group_count: print( 'WARNING: Prescheduled tests do not match up with the group count. ' 'Prescheduling will be skipped.') except Exception: print( 'WARNING: Unable to load prescheduled tests. Prescheduling will be skipped.' ) all_prescheduled_tests = [ test_dict[test_name] for sublist in prescheduled_data for test_name in sublist if test_name in test_dict ] prescheduled_tests = [ test_dict[test_name] for test_name in prescheduled_data[group_id - 1] if test_name in test_dict ] unscheduled_tests = [ item for item in items if item not in all_prescheduled_tests ] unscheduled_tests.sort(key=lambda x: x.name) if seed is not False: seeded = Random(seed) seeded.shuffle(unscheduled_tests) total_unscheduled_items = len(unscheduled_tests) group_size = get_group_size(total_unscheduled_items, group_count) tests_in_group = get_group(unscheduled_tests, group_size, group_id) items[:] = tests_in_group + prescheduled_tests items.sort(key=original_order.__getitem__) terminal_reporter = config.pluginmanager.get_plugin('terminalreporter') if terminal_reporter is not None: terminal_writer = create_terminal_writer(config) message = terminal_writer.markup( '\nRunning test group #{0} ({1} tests)\n'.format( group_id, len(items)), yellow=True) terminal_reporter.write(message) message = terminal_writer.markup( '\n'.join([item.name for item in items]) + '\n', yellow=True) terminal_reporter.write(message)