def setup_outputs(self, args_outfiles=None): if self.outputs: assert not args_outfiles, "ONLY-ONCE" return # -- NORMAL CASE: Setup only initially (once). if not args_outfiles: self.outputs.append(StreamOpener(stream=sys.stdout)) else: for outfile in args_outfiles: if outfile and outfile != "-": self.outputs.append(StreamOpener(outfile)) else: self.outputs.append(StreamOpener(stream=sys.stdout))
def __init__(self, config): super(SummaryReporter, self).__init__(config) stream = getattr(sys, self.output_stream_name, sys.stderr) self.stream = StreamOpener.ensure_stream_with_encoder(stream) self.feature_summary = { Status.passed.name: 0, Status.failed.name: 0, Status.skipped.name: 0, Status.untested.name: 0 } self.scenario_summary = { Status.passed.name: 0, Status.failed.name: 0, Status.skipped.name: 0, Status.untested.name: 0 } self.step_summary = { Status.passed.name: 0, Status.failed.name: 0, Status.skipped.name: 0, Status.untested.name: 0, Status.undefined.name: 0 } self.start = monotonic() self.failed_scenarios = []
def setUp(self): self.config = Mock() self.config.color = True self.config.outputs = [ StreamOpener(stream=sys.stdout) for i in self.formatters ] self.config.format = self.formatters
def run_tests(room, vendor, tags, override): app = create_app() with app.app_context(): test_run = TestRun() db.session.add(test_run) db.session.commit() def on_snapshot(snapshot, plan): test_run.save_snapshot(snapshot, plan) socketio.emit('snapshot', test_run.event, room=room) db.session.commit() try: output = io.StringIO() output_stream = StreamOpener(stream=output) config = Configuration( outputs=[output_stream], format=['json.chunked'], on_snapshot=on_snapshot, vendor=vendor, override=override, command_args=[], tags=[','.join(tags)], ) runner = Runner(config) runner.run() except Exception as err: # pylint: disable=broad-except socketio.emit('global_error', str(err), room=room) finally: socketio.emit('tests_complete', room=room)
def __init__(self, config): super(SummaryReporter, self).__init__(config) stream = getattr(sys, self.output_stream_name, sys.stderr) self.stream = StreamOpener.ensure_stream_with_encoder(stream) self.feature_summary = { 'passed': 0, 'failed': 0, 'skipped': 0, 'untested': 0 } self.scenario_summary = { 'passed': 0, 'failed': 0, 'skipped': 0, 'untested': 0 } self.step_summary = { 'passed': 0, 'failed': 0, 'skipped': 0, 'undefined': 0, 'untested': 0 } self.duration = 0.0 self.failed_scenarios = []
def worker(self, proc_number): while 1: try: joblist_index = self.joblist_index_queue.get_nowait() except (RuntimeError, TypeError, NameError): break current_job = self.joblist[joblist_index] writebuf = StringIO.StringIO() self.setfeature(current_job) self.config.outputs = [] self.config.outputs.append(StreamOpener(stream=writebuf)) stream_openers = self.config.outputs self.formatters = formatters.get_formatter(self.config, stream_openers) for formatter in self.formatters: formatter.uri(current_job.filename) start_time = time.strftime("%Y-%m-%d %H:%M:%S") current_job.run(self) end_time = time.strftime("%Y-%m-%d %H:%M:%S") sys.stderr.write(current_job.status[0] + " ") if current_job.type == 'feature': for reporter in self.config.reporters: reporter.feature(current_job) job_report_text = self.generatereport(proc_number, current_job, start_time, end_time, writebuf) if job_report_text: results = {} results['steps_passed'] = 0 results['steps_failed'] = 0 results['steps_skipped'] = 0 results['steps_undefined'] = 0 results['steps_untested'] = 0 results['jobtype'] = current_job.type results['reportinginfo'] = job_report_text results['status'] = current_job.status if current_job.type != 'feature': results['uniquekey'] = \ current_job.filename + current_job.feature.name else: results['scenarios_passed'] = 0 results['scenarios_failed'] = 0 results['scenarios_skipped'] = 0 self.countscenariostatus(current_job, results) self.countstepstatus(current_job, results) if current_job.type != 'feature' and \ getattr(self.config, 'junit'): results['junit_report'] = \ self.generate_junit_report(current_job, writebuf) self.resultsqueue.put(results)
def get_formatter(config, stream_openers): # -- BUILD: Formatter list default_stream_opener = StreamOpener(stream=sys.stdout) formatter_list = [] for i, name in enumerate(config.format): stream_opener = default_stream_opener if i < len(stream_openers): stream_opener = stream_openers[i] formatter_list.append(formatters[name](stream_opener, config)) return formatter_list
def setup_allure_logger(self, output_directory): """Inserts the Allure formatter as the first element of the outputs list, in order to make it match with the formats. Note that the default Behave formatter does not add a format)""" allure_report_directory = "allure" self.behave_config.outputs.insert( 0, StreamOpener(filename=os.path.join(output_directory, allure_report_directory))) self.behave_config.format.append( "allure_behave.formatter:AllureFormatter")
def __init__(self, config): super(SummaryReporter, self).__init__(config) stream = getattr(sys, self.output_stream_name, sys.stderr) self.stream = StreamOpener.ensure_stream_with_encoder(stream) self.feature_summary = {'passed': 0, 'failed': 0, 'skipped': 0, 'untested': 0} self.scenario_summary = {'passed': 0, 'failed': 0, 'skipped': 0, 'untested': 0} self.step_summary = {'passed': 0, 'failed': 0, 'skipped': 0, 'undefined': 0, 'untested': 0} self.duration = 0.0 self.failed_scenarios = []
def run(context, **kwargs): cmd_args = '-v -f allure_behave.formatter:AllureFormatter -f pretty' cmd = '{options} {cmd}'.format(cmd=cmd_args, options=kwargs.get('args', '')) config = Configuration(command_args=cmd) result_tmp_dir = mkdtemp(dir=os.environ.get('TEST_TMP', None)) stream_opener = StreamOpener(filename=result_tmp_dir) model_runner = ModelRunner(config, [context.feature_definition]) model_runner.formatters = make_formatters(config, [stream_opener]) model_runner.run() context.allure_report = AllureReport(result_tmp_dir)
def run_behave_with_allure(context, **kwargs): with test_context(): cmd_args = '-f allure_behave.formatter:AllureFormatter' cmd = '{options} {cmd}'.format(cmd=cmd_args, options=kwargs.get('args', '')) config = Configuration(command_args=cmd) result_tmp_dir = mkdtemp(dir=os.environ.get('TEST_TMP', None)) stream_opener = StreamOpener(filename=result_tmp_dir) model_runner = ModelRunner(config, context.feature_definition) model_runner.formatters = make_formatters(config, [stream_opener]) model_runner.hooks = getattr(context, 'globals', dict()) model_runner.run() context.allure_report = AllureReport(result_tmp_dir) os.environ.pop("ALLURE_TESTPLAN_PATH", None)
def setUp(self): self.config = Mock() self.config.reporters = [] self.config.logging_level = None self.config.logging_filter = None self.config.outputs = [Mock(), StreamOpener(stream=sys.stdout)] self.config.format = ["plain", "progress"] self.runner = runner.Runner(self.config) self.load_hooks = self.runner.load_hooks = Mock() self.load_step_definitions = self.runner.load_step_definitions = Mock() self.run_hook = self.runner.run_hook = Mock() self.run_step = self.runner.run_step = Mock() self.feature_locations = self.runner.feature_locations = Mock() self.calculate_summaries = self.runner.calculate_summaries = Mock() self.formatter_class = patch("behave.formatter.pretty.PrettyFormatter") formatter_class = self.formatter_class.start() formatter_class.return_value = self.formatter = Mock()
def __init__(self, config): super(SummaryReporter, self).__init__(config) stream = getattr(sys, self.output_stream_name, sys.stderr) self.stream = StreamOpener.ensure_stream_with_encoder(stream) summary_zero_data = { Status.passed.name: 0, Status.failed.name: 0, Status.skipped.name: 0, Status.untested.name: 0 } self.feature_summary = summary_zero_data.copy() self.rule_summary = summary_zero_data.copy() self.scenario_summary = summary_zero_data.copy() self.step_summary = {Status.undefined.name: 0} self.step_summary.update(summary_zero_data) self.duration = 0.0 self.run_starttime = 0 self.run_endtime = 0 self.failed_scenarios = [] self.show_rules = True
def make_formatters(config, stream_openers): """Build a list of formatter, used by a behave runner. :param config: Configuration object to use. :param stream_openers: List of stream openers to use (for formatters). :return: List of formatters. :raises: LookupError/KeyError if a formatter class is unknown. :raises: ImportError, if a formatter class cannot be loaded/resolved. """ # -- BUILD: Formatter list default_stream_opener = StreamOpener(stream=sys.stdout) formatter_list = [] for i, name in enumerate(config.format): stream_opener = default_stream_opener if i < len(stream_openers): stream_opener = stream_openers[i] formatter_class = select_formatter_class(name) formatter_object = formatter_class(stream_opener, config) formatter_list.append(formatter_object) return formatter_list
def __init__(self, config): super(SummaryReporter, self).__init__(config) stream = getattr(sys, self.output_stream_name, sys.stderr) self.stream = StreamOpener.ensure_stream_with_encoder(stream) summary_zero_data = { "all": 0, Status.passed.name: 0, Status.failed.name: 0, Status.skipped.name: 0, Status.untested.name: 0 } self.feature_summary = summary_zero_data.copy() self.rule_summary = summary_zero_data.copy() self.scenario_summary = summary_zero_data.copy() self.step_summary = {Status.undefined.name: 0} self.step_summary.update(summary_zero_data) self.duration = 0.0 self.run_starttime = 0 self.run_endtime = 0 self.failed_scenarios = [] self.show_rules = True
def test_parses_feature_files_and_appends_to_feature_list(self, abspath, parse_file): feature_locations = ["one", "two", "three"] feature = Mock() feature.tags = [] feature.__iter__ = Mock(return_value=iter([])) feature.run.return_value = False self.runner.feature_locations.return_value = feature_locations abspath.side_effect = lambda x: x.upper() self.config.lang = "fritz" self.config.format = ["plain"] self.config.outputs = [StreamOpener(stream=sys.stdout)] self.config.output.encoding = None self.config.exclude = lambda s: False self.config.junit = False self.config.summary = False parse_file.return_value = feature self.runner.run_with_paths() expected_parse_file_args = \ [((x.upper(),), {"language": "fritz"}) for x in feature_locations] eq_(parse_file.call_args_list, expected_parse_file_args) eq_(self.runner.features, [feature] * 3)
def __init__(self, command_args=None, load_config=True, verbose=None, **kwargs): """ Constructs a behave configuration object. * loads the configuration defaults (if needed). * process the command-line args * store the configuration results :param command_args: Provide command args (as sys.argv). If command_args is None, sys.argv[1:] is used. :type command_args: list<str>, str :param load_config: Indicate if configfile should be loaded (=true) :param verbose: Indicate if diagnostic output is enabled :param kwargs: Used to hand-over/overwrite default values. """ if command_args is None: command_args = sys.argv[1:] elif isinstance(command_args, six.string_types): if six.PY2 and isinstance(command_args, six.text_type): command_args = command_args.encode("utf-8") elif six.PY3 and isinstance(command_args, six.binary_type): command_args = command_args.decode("utf-8") command_args = shlex.split(command_args) if verbose is None: # -- AUTO-DISCOVER: Verbose mode from command-line args. verbose = ('-v' in command_args) or ('--verbose' in command_args) defaults = self.defaults.copy() for name, value in kwargs.items(): defaults[name] = value self.defaults = defaults self.formatters = [] self.reporters = [] self.name_re = None self.outputs = [] self.include_re = None self.exclude_re = None self.scenario_outline_annotation_schema = None self.steps_dir = "steps" self.environment_file = "environment.py" self.userdata_defines = None if load_config: load_configuration(self.defaults, verbose=verbose) parser = setup_parser() parser.set_defaults(**self.defaults) args = parser.parse_args(command_args) for key, value in args.__dict__.items(): if key.startswith('_') and key not in self.cmdline_only_options: continue setattr(self, key, value) self.paths = [os.path.normpath(path) for path in self.paths] if not args.outfiles: self.outputs.append(StreamOpener(stream=sys.stdout)) else: for outfile in args.outfiles: if outfile and outfile != '-': self.outputs.append(StreamOpener(outfile)) else: self.outputs.append(StreamOpener(stream=sys.stdout)) if self.steps_catalog: # -- SHOW STEP-CATALOG: As step summary. self.default_format = "steps.catalog" self.format = ["steps.catalog"] self.dry_run = True self.summary = False self.show_skipped = False self.quiet = True if self.wip: # Only run scenarios tagged with "wip". # Additionally: # * use the "plain" formatter (per default) # * do not capture stdout or logging output and # * stop at the first failure. self.default_format = "plain" self.tags = ["wip"] self.color = False self.stop = True self.log_capture = False self.stdout_capture = False self.tags = TagExpression(self.tags or []) if self.quiet: self.show_source = False self.show_snippets = False if self.exclude_re: self.exclude_re = re.compile(self.exclude_re) if self.include_re: self.include_re = re.compile(self.include_re) if self.name: # -- SELECT: Scenario-by-name, build regular expression. self.name_re = self.build_name_re(self.name) if self.junit: # Buffer the output (it will be put into Junit report) self.stdout_capture = True self.stderr_capture = True self.log_capture = True self.reporters.append(JUnitReporter(self)) if self.summary: self.reporters.append(SummaryReporter(self)) unknown_formats = self.collect_unknown_formats() if unknown_formats: parser.error("format=%s is unknown" % ", ".join(unknown_formats)) if self.stage is None: # -- USE ENVIRONMENT-VARIABLE, if stage is undefined. self.stage = os.environ.get("BEHAVE_STAGE", None) self.setup_stage(self.stage) self.setup_model() self.setup_userdata()
def _formatter(self, file_object, config): # pylint: disable=no-self-use stream_opener = StreamOpener(stream=file_object) f = make_formatters(config, [stream_opener])[0] f.uri("<string>") return f
def _formatters(self, file_object, config): # pylint: disable=no-self-use stream_opener = StreamOpener(stream=file_object) formatters = make_formatters(config, [stream_opener]) for f in formatters: f.uri('<string>') return formatters
def __init__(self, command_args=None, verbose=None): """ Constructs a behave configuration object. * loads the configuration defaults. * process the command-line args * store the configuration results :param command_args: Provide command args (as sys.argv). If command_args is None, sys.argv[1:] is used. :type command_args: list<str>, str :param verbose: Indicate if diagnostic output is enabled """ if command_args is None: command_args = sys.argv[1:] elif isinstance(command_args, basestring): command_args = shlex.split(command_args) if verbose is None: # -- AUTO-DISCOVER: Verbose mode from command-line args. verbose = ('-v' in command_args) or ('--verbose' in command_args) self.formatters = [] self.reporters = [] self.name_re = None self.outputs = [] self.include_re = None self.exclude_re = None load_configuration(self.defaults, verbose=verbose) parser.set_defaults(**self.defaults) args = parser.parse_args(command_args) for key, value in args.__dict__.items(): if key.startswith('_'): continue setattr(self, key, value) self.paths = [os.path.normpath(path) for path in self.paths] if not args.outfiles: self.outputs.append(StreamOpener(stream=sys.stdout)) else: for outfile in args.outfiles: if outfile and outfile != '-': self.outputs.append(StreamOpener(outfile)) else: self.outputs.append(StreamOpener(stream=sys.stdout)) if self.wip: # Only run scenarios tagged with "wip". Additionally: use the # "plain" formatter, do not capture stdout or logging output and # stop at the first failure. self.format = ['plain'] self.tags = ['wip'] self.stop = True self.log_capture = False self.stdout_capture = False self.tags = TagExpression(self.tags or []) if self.quiet: self.show_source = False self.show_snippets = False if self.exclude_re: self.exclude_re = re.compile(self.exclude_re) if self.include_re: self.include_re = re.compile(self.include_re) if self.name: # -- SELECT: Scenario-by-name, build regular expression. self.name_re = self.build_name_re(self.name) if self.junit: # Buffer the output (it will be put into Junit report) self.stdout_capture = True self.stderr_capture = True self.log_capture = True self.reporters.append(JUnitReporter(self)) if self.summary: self.reporters.append(SummaryReporter(self)) unknown_formats = self.collect_unknown_formats() if unknown_formats: parser.error("format=%s is unknown" % ", ".join(unknown_formats))
def setUp(self): self.config = Mock() self.config.color = True self.config.outputs = [StreamOpener(stream=sys.stdout)] self.config.format = [self.formatter_name]
def __init__(self): self.formatters = [] self.reporters = [] self.name_re = None self.outputs = [] load_configuration(self.defaults) parser.set_defaults(**self.defaults) args = parser.parse_args() for key, value in args.__dict__.items(): if key.startswith('_'): continue setattr(self, key, value) self.paths = [os.path.normpath(path) for path in self.paths] if not args.outfiles: self.outputs.append(StreamOpener(stream=sys.stdout)) else: for outfile in args.outfiles: if outfile and outfile != '-': self.outputs.append(StreamOpener(outfile)) else: self.outputs.append(StreamOpener(stream=sys.stdout)) if self.wip: # Only run scenarios tagged with "wip". Additionally: use the # "plain" formatter, do not capture stdout or logging output and # stop at the first failure. self.format = ['plain'] self.tags = ['wip'] self.stop = True self.log_capture = False self.stdout_capture = False self.tags = TagExpression(self.tags or []) if self.quiet: self.show_source = False self.show_snippets = False if self.exclude_re: self.exclude_re = re.compile(self.exclude_re) if self.include_re: self.include_re = re.compile(self.include_re) if self.name: # -- SELECT: Scenario-by-name, build regular expression. self.name_re = self.build_name_re(self.name) if self.junit: # Buffer the output (it will be put into Junit report) self.stdout_capture = True self.stderr_capture = True self.log_capture = True self.reporters.append(JUnitReporter(self)) if self.summary: self.reporters.append(SummaryReporter(self)) unknown_formats = self.collect_unknown_formats() if unknown_formats: parser.error("format=%s is unknown" % ", ".join(unknown_formats))
def worker(self, proc_number): self.context.procnum = proc_number while 1: try: joblist_index = self.joblist_index_queue.get_nowait() except Exception, e: break current_job = self.joblist[joblist_index] writebuf = StringIO.StringIO() self.setfeature(current_job) self.config.outputs = [] self.config.outputs.append(StreamOpener(stream=writebuf)) stream_openers = self.config.outputs self.formatters = make_formatters(self.config, stream_openers) for formatter in self.formatters: formatter.uri(current_job.filename) if self.step_registry is None: self.step_registry = the_step_registry start_time = time.strftime("%Y-%m-%d %H:%M:%S") # if current_job.type == 'scenario' and getattr(self.config, # 'parallel_scenario_feature_hooks'): # self.run_hook('before_feature', self.context, current_job) current_job.run(self) # if current_job.type == 'scenario' and getattr(self.config, # 'parallel_scenario_feature_hooks'): # self.run_hook('after_feature', self.context, current_job) end_time = time.strftime("%Y-%m-%d %H:%M:%S") sys.stderr.write("%s: %s \n" % (current_job.filename, current_job.status.name)) if current_job.type == 'feature': for reporter in self.config.reporters: reporter.feature(current_job) job_report_text = self.generatereport(proc_number, current_job, start_time, end_time, writebuf) if job_report_text: results = {} results['steps_passed'] = 0 results['steps_failed'] = 0 results['steps_skipped'] = 0 results['steps_undefined'] = 0 results['steps_untested'] = 0 results['jobtype'] = current_job.type results['reportinginfo'] = job_report_text results['status'] = str(current_job.status.name) if current_job.type != 'feature': results['uniquekey'] = \ current_job.filename + current_job.feature.name else: results['scenarios_passed'] = 0 results['scenarios_failed'] = 0 results['scenarios_skipped'] = 0 results['scenarios_untested'] = 0 # results['scenarios_undefined'] = 0 # results['scenarios_executing'] = 0 self.countscenariostatus(current_job, results) self.countstepstatus(current_job, results) if current_job.type != 'feature' and \ getattr(self.config, 'junit'): results['junit_report'] = \ self.generate_junit_report(current_job, writebuf) self.resultsqueue.put(results)
def _formatters(self, file, config): stream_opener = StreamOpener(stream=file) fs = formatters.get_formatter(config, [stream_opener]) for f in fs: f.uri('<string>') return fs