def setup_outputs(self, args_outfiles=None):
if self.outputs:
assert not args_outfiles, "ONLY-ONCE"
return
# -- NORMAL CASE: Setup only initially (once).
if not args_outfiles:
self.outputs.append(StreamOpener(stream=sys.stdout))
else:
for outfile in args_outfiles:
if outfile and outfile != "-":
self.outputs.append(StreamOpener(outfile))
else:
self.outputs.append(StreamOpener(stream=sys.stdout))
def __init__(self, config):
super(SummaryReporter, self).__init__(config)
stream = getattr(sys, self.output_stream_name, sys.stderr)
self.stream = StreamOpener.ensure_stream_with_encoder(stream)
self.feature_summary = {
Status.passed.name: 0,
Status.failed.name: 0,
Status.skipped.name: 0,
Status.untested.name: 0
}
self.scenario_summary = {
Status.passed.name: 0,
Status.failed.name: 0,
Status.skipped.name: 0,
Status.untested.name: 0
}
self.step_summary = {
Status.passed.name: 0,
Status.failed.name: 0,
Status.skipped.name: 0,
Status.untested.name: 0,
Status.undefined.name: 0
}
self.start = monotonic()
self.failed_scenarios = []
def setUp(self):
self.config = Mock()
self.config.color = True
self.config.outputs = [
StreamOpener(stream=sys.stdout) for i in self.formatters
]
self.config.format = self.formatters
def run_tests(room, vendor, tags, override):
app = create_app()
with app.app_context():
test_run = TestRun()
db.session.add(test_run)
db.session.commit()
def on_snapshot(snapshot, plan):
test_run.save_snapshot(snapshot, plan)
socketio.emit('snapshot', test_run.event, room=room)
db.session.commit()
try:
output = io.StringIO()
output_stream = StreamOpener(stream=output)
config = Configuration(
outputs=[output_stream],
format=['json.chunked'],
on_snapshot=on_snapshot,
vendor=vendor,
override=override,
command_args=[],
tags=[','.join(tags)],
)
runner = Runner(config)
runner.run()
except Exception as err: # pylint: disable=broad-except
socketio.emit('global_error', str(err), room=room)
finally:
socketio.emit('tests_complete', room=room)
def __init__(self, config):
super(SummaryReporter, self).__init__(config)
stream = getattr(sys, self.output_stream_name, sys.stderr)
self.stream = StreamOpener.ensure_stream_with_encoder(stream)
self.feature_summary = {
'passed': 0,
'failed': 0,
'skipped': 0,
'untested': 0
}
self.scenario_summary = {
'passed': 0,
'failed': 0,
'skipped': 0,
'untested': 0
}
self.step_summary = {
'passed': 0,
'failed': 0,
'skipped': 0,
'undefined': 0,
'untested': 0
}
self.duration = 0.0
self.failed_scenarios = []
def worker(self, proc_number):
while 1:
try:
joblist_index = self.joblist_index_queue.get_nowait()
except (RuntimeError, TypeError, NameError):
break
current_job = self.joblist[joblist_index]
writebuf = StringIO.StringIO()
self.setfeature(current_job)
self.config.outputs = []
self.config.outputs.append(StreamOpener(stream=writebuf))
stream_openers = self.config.outputs
self.formatters = formatters.get_formatter(self.config,
stream_openers)
for formatter in self.formatters:
formatter.uri(current_job.filename)
start_time = time.strftime("%Y-%m-%d %H:%M:%S")
current_job.run(self)
end_time = time.strftime("%Y-%m-%d %H:%M:%S")
sys.stderr.write(current_job.status[0] + " ")
if current_job.type == 'feature':
for reporter in self.config.reporters:
reporter.feature(current_job)
job_report_text = self.generatereport(proc_number, current_job,
start_time, end_time,
writebuf)
if job_report_text:
results = {}
results['steps_passed'] = 0
results['steps_failed'] = 0
results['steps_skipped'] = 0
results['steps_undefined'] = 0
results['steps_untested'] = 0
results['jobtype'] = current_job.type
results['reportinginfo'] = job_report_text
results['status'] = current_job.status
if current_job.type != 'feature':
results['uniquekey'] = \
current_job.filename + current_job.feature.name
else:
results['scenarios_passed'] = 0
results['scenarios_failed'] = 0
results['scenarios_skipped'] = 0
self.countscenariostatus(current_job, results)
self.countstepstatus(current_job, results)
if current_job.type != 'feature' and \
getattr(self.config, 'junit'):
results['junit_report'] = \
self.generate_junit_report(current_job, writebuf)
self.resultsqueue.put(results)
def get_formatter(config, stream_openers):
# -- BUILD: Formatter list
default_stream_opener = StreamOpener(stream=sys.stdout)
formatter_list = []
for i, name in enumerate(config.format):
stream_opener = default_stream_opener
if i < len(stream_openers):
stream_opener = stream_openers[i]
formatter_list.append(formatters[name](stream_opener, config))
return formatter_list
def setup_allure_logger(self, output_directory):
"""Inserts the Allure formatter as the first element of the outputs list, in order to make it match with the
formats. Note that the default Behave formatter does not add a format)"""
allure_report_directory = "allure"
self.behave_config.outputs.insert(
0,
StreamOpener(filename=os.path.join(output_directory,
allure_report_directory)))
self.behave_config.format.append(
"allure_behave.formatter:AllureFormatter")
def __init__(self, config):
super(SummaryReporter, self).__init__(config)
stream = getattr(sys, self.output_stream_name, sys.stderr)
self.stream = StreamOpener.ensure_stream_with_encoder(stream)
self.feature_summary = {'passed': 0, 'failed': 0, 'skipped': 0,
'untested': 0}
self.scenario_summary = {'passed': 0, 'failed': 0, 'skipped': 0,
'untested': 0}
self.step_summary = {'passed': 0, 'failed': 0, 'skipped': 0,
'undefined': 0, 'untested': 0}
self.duration = 0.0
self.failed_scenarios = []
def run(context, **kwargs):
cmd_args = '-v -f allure_behave.formatter:AllureFormatter -f pretty'
cmd = '{options} {cmd}'.format(cmd=cmd_args,
options=kwargs.get('args', ''))
config = Configuration(command_args=cmd)
result_tmp_dir = mkdtemp(dir=os.environ.get('TEST_TMP', None))
stream_opener = StreamOpener(filename=result_tmp_dir)
model_runner = ModelRunner(config, [context.feature_definition])
model_runner.formatters = make_formatters(config, [stream_opener])
model_runner.run()
context.allure_report = AllureReport(result_tmp_dir)
def run_behave_with_allure(context, **kwargs):
with test_context():
cmd_args = '-f allure_behave.formatter:AllureFormatter'
cmd = '{options} {cmd}'.format(cmd=cmd_args,
options=kwargs.get('args', ''))
config = Configuration(command_args=cmd)
result_tmp_dir = mkdtemp(dir=os.environ.get('TEST_TMP', None))
stream_opener = StreamOpener(filename=result_tmp_dir)
model_runner = ModelRunner(config, context.feature_definition)
model_runner.formatters = make_formatters(config, [stream_opener])
model_runner.hooks = getattr(context, 'globals', dict())
model_runner.run()
context.allure_report = AllureReport(result_tmp_dir)
os.environ.pop("ALLURE_TESTPLAN_PATH", None)
def setUp(self):
self.config = Mock()
self.config.reporters = []
self.config.logging_level = None
self.config.logging_filter = None
self.config.outputs = [Mock(), StreamOpener(stream=sys.stdout)]
self.config.format = ["plain", "progress"]
self.runner = runner.Runner(self.config)
self.load_hooks = self.runner.load_hooks = Mock()
self.load_step_definitions = self.runner.load_step_definitions = Mock()
self.run_hook = self.runner.run_hook = Mock()
self.run_step = self.runner.run_step = Mock()
self.feature_locations = self.runner.feature_locations = Mock()
self.calculate_summaries = self.runner.calculate_summaries = Mock()
self.formatter_class = patch("behave.formatter.pretty.PrettyFormatter")
formatter_class = self.formatter_class.start()
formatter_class.return_value = self.formatter = Mock()
def __init__(self, config):
super(SummaryReporter, self).__init__(config)
stream = getattr(sys, self.output_stream_name, sys.stderr)
self.stream = StreamOpener.ensure_stream_with_encoder(stream)
summary_zero_data = {
Status.passed.name: 0,
Status.failed.name: 0,
Status.skipped.name: 0,
Status.untested.name: 0
}
self.feature_summary = summary_zero_data.copy()
self.rule_summary = summary_zero_data.copy()
self.scenario_summary = summary_zero_data.copy()
self.step_summary = {Status.undefined.name: 0}
self.step_summary.update(summary_zero_data)
self.duration = 0.0
self.run_starttime = 0
self.run_endtime = 0
self.failed_scenarios = []
self.show_rules = True
def make_formatters(config, stream_openers):
"""Build a list of formatter, used by a behave runner.
:param config: Configuration object to use.
:param stream_openers: List of stream openers to use (for formatters).
:return: List of formatters.
:raises: LookupError/KeyError if a formatter class is unknown.
:raises: ImportError, if a formatter class cannot be loaded/resolved.
"""
# -- BUILD: Formatter list
default_stream_opener = StreamOpener(stream=sys.stdout)
formatter_list = []
for i, name in enumerate(config.format):
stream_opener = default_stream_opener
if i < len(stream_openers):
stream_opener = stream_openers[i]
formatter_class = select_formatter_class(name)
formatter_object = formatter_class(stream_opener, config)
formatter_list.append(formatter_object)
return formatter_list
def __init__(self, config):
super(SummaryReporter, self).__init__(config)
stream = getattr(sys, self.output_stream_name, sys.stderr)
self.stream = StreamOpener.ensure_stream_with_encoder(stream)
summary_zero_data = {
"all": 0,
Status.passed.name: 0,
Status.failed.name: 0,
Status.skipped.name: 0,
Status.untested.name: 0
}
self.feature_summary = summary_zero_data.copy()
self.rule_summary = summary_zero_data.copy()
self.scenario_summary = summary_zero_data.copy()
self.step_summary = {Status.undefined.name: 0}
self.step_summary.update(summary_zero_data)
self.duration = 0.0
self.run_starttime = 0
self.run_endtime = 0
self.failed_scenarios = []
self.show_rules = True
def test_parses_feature_files_and_appends_to_feature_list(self, abspath,
parse_file):
feature_locations = ["one", "two", "three"]
feature = Mock()
feature.tags = []
feature.__iter__ = Mock(return_value=iter([]))
feature.run.return_value = False
self.runner.feature_locations.return_value = feature_locations
abspath.side_effect = lambda x: x.upper()
self.config.lang = "fritz"
self.config.format = ["plain"]
self.config.outputs = [StreamOpener(stream=sys.stdout)]
self.config.output.encoding = None
self.config.exclude = lambda s: False
self.config.junit = False
self.config.summary = False
parse_file.return_value = feature
self.runner.run_with_paths()
expected_parse_file_args = \
[((x.upper(),), {"language": "fritz"}) for x in feature_locations]
eq_(parse_file.call_args_list, expected_parse_file_args)
eq_(self.runner.features, [feature] * 3)
def __init__(self,
command_args=None,
load_config=True,
verbose=None,
**kwargs):
"""
Constructs a behave configuration object.
* loads the configuration defaults (if needed).
* process the command-line args
* store the configuration results
:param command_args: Provide command args (as sys.argv).
If command_args is None, sys.argv[1:] is used.
:type command_args: list<str>, str
:param load_config: Indicate if configfile should be loaded (=true)
:param verbose: Indicate if diagnostic output is enabled
:param kwargs: Used to hand-over/overwrite default values.
"""
if command_args is None:
command_args = sys.argv[1:]
elif isinstance(command_args, six.string_types):
if six.PY2 and isinstance(command_args, six.text_type):
command_args = command_args.encode("utf-8")
elif six.PY3 and isinstance(command_args, six.binary_type):
command_args = command_args.decode("utf-8")
command_args = shlex.split(command_args)
if verbose is None:
# -- AUTO-DISCOVER: Verbose mode from command-line args.
verbose = ('-v' in command_args) or ('--verbose' in command_args)
defaults = self.defaults.copy()
for name, value in kwargs.items():
defaults[name] = value
self.defaults = defaults
self.formatters = []
self.reporters = []
self.name_re = None
self.outputs = []
self.include_re = None
self.exclude_re = None
self.scenario_outline_annotation_schema = None
self.steps_dir = "steps"
self.environment_file = "environment.py"
self.userdata_defines = None
if load_config:
load_configuration(self.defaults, verbose=verbose)
parser = setup_parser()
parser.set_defaults(**self.defaults)
args = parser.parse_args(command_args)
for key, value in args.__dict__.items():
if key.startswith('_') and key not in self.cmdline_only_options:
continue
setattr(self, key, value)
self.paths = [os.path.normpath(path) for path in self.paths]
if not args.outfiles:
self.outputs.append(StreamOpener(stream=sys.stdout))
else:
for outfile in args.outfiles:
if outfile and outfile != '-':
self.outputs.append(StreamOpener(outfile))
else:
self.outputs.append(StreamOpener(stream=sys.stdout))
if self.steps_catalog:
# -- SHOW STEP-CATALOG: As step summary.
self.default_format = "steps.catalog"
self.format = ["steps.catalog"]
self.dry_run = True
self.summary = False
self.show_skipped = False
self.quiet = True
if self.wip:
# Only run scenarios tagged with "wip".
# Additionally:
# * use the "plain" formatter (per default)
# * do not capture stdout or logging output and
# * stop at the first failure.
self.default_format = "plain"
self.tags = ["wip"]
self.color = False
self.stop = True
self.log_capture = False
self.stdout_capture = False
self.tags = TagExpression(self.tags or [])
if self.quiet:
self.show_source = False
self.show_snippets = False
if self.exclude_re:
self.exclude_re = re.compile(self.exclude_re)
if self.include_re:
self.include_re = re.compile(self.include_re)
if self.name:
# -- SELECT: Scenario-by-name, build regular expression.
self.name_re = self.build_name_re(self.name)
if self.junit:
# Buffer the output (it will be put into Junit report)
self.stdout_capture = True
self.stderr_capture = True
self.log_capture = True
self.reporters.append(JUnitReporter(self))
if self.summary:
self.reporters.append(SummaryReporter(self))
unknown_formats = self.collect_unknown_formats()
if unknown_formats:
parser.error("format=%s is unknown" % ", ".join(unknown_formats))
if self.stage is None:
# -- USE ENVIRONMENT-VARIABLE, if stage is undefined.
self.stage = os.environ.get("BEHAVE_STAGE", None)
self.setup_stage(self.stage)
self.setup_model()
self.setup_userdata()
def _formatter(self, file_object, config): # pylint: disable=no-self-use
stream_opener = StreamOpener(stream=file_object)
f = make_formatters(config, [stream_opener])[0]
f.uri("<string>")
return f
def _formatters(self, file_object, config): # pylint: disable=no-self-use
stream_opener = StreamOpener(stream=file_object)
formatters = make_formatters(config, [stream_opener])
for f in formatters:
f.uri('<string>')
return formatters
def __init__(self, command_args=None, verbose=None):
"""
Constructs a behave configuration object.
* loads the configuration defaults.
* process the command-line args
* store the configuration results
:param command_args: Provide command args (as sys.argv).
If command_args is None, sys.argv[1:] is used.
:type command_args: list<str>, str
:param verbose: Indicate if diagnostic output is enabled
"""
if command_args is None:
command_args = sys.argv[1:]
elif isinstance(command_args, basestring):
command_args = shlex.split(command_args)
if verbose is None:
# -- AUTO-DISCOVER: Verbose mode from command-line args.
verbose = ('-v' in command_args) or ('--verbose' in command_args)
self.formatters = []
self.reporters = []
self.name_re = None
self.outputs = []
self.include_re = None
self.exclude_re = None
load_configuration(self.defaults, verbose=verbose)
parser.set_defaults(**self.defaults)
args = parser.parse_args(command_args)
for key, value in args.__dict__.items():
if key.startswith('_'):
continue
setattr(self, key, value)
self.paths = [os.path.normpath(path) for path in self.paths]
if not args.outfiles:
self.outputs.append(StreamOpener(stream=sys.stdout))
else:
for outfile in args.outfiles:
if outfile and outfile != '-':
self.outputs.append(StreamOpener(outfile))
else:
self.outputs.append(StreamOpener(stream=sys.stdout))
if self.wip:
# Only run scenarios tagged with "wip". Additionally: use the
# "plain" formatter, do not capture stdout or logging output and
# stop at the first failure.
self.format = ['plain']
self.tags = ['wip']
self.stop = True
self.log_capture = False
self.stdout_capture = False
self.tags = TagExpression(self.tags or [])
if self.quiet:
self.show_source = False
self.show_snippets = False
if self.exclude_re:
self.exclude_re = re.compile(self.exclude_re)
if self.include_re:
self.include_re = re.compile(self.include_re)
if self.name:
# -- SELECT: Scenario-by-name, build regular expression.
self.name_re = self.build_name_re(self.name)
if self.junit:
# Buffer the output (it will be put into Junit report)
self.stdout_capture = True
self.stderr_capture = True
self.log_capture = True
self.reporters.append(JUnitReporter(self))
if self.summary:
self.reporters.append(SummaryReporter(self))
unknown_formats = self.collect_unknown_formats()
if unknown_formats:
parser.error("format=%s is unknown" % ", ".join(unknown_formats))
def setUp(self):
self.config = Mock()
self.config.color = True
self.config.outputs = [StreamOpener(stream=sys.stdout)]
self.config.format = [self.formatter_name]
def __init__(self):
self.formatters = []
self.reporters = []
self.name_re = None
self.outputs = []
load_configuration(self.defaults)
parser.set_defaults(**self.defaults)
args = parser.parse_args()
for key, value in args.__dict__.items():
if key.startswith('_'):
continue
setattr(self, key, value)
self.paths = [os.path.normpath(path) for path in self.paths]
if not args.outfiles:
self.outputs.append(StreamOpener(stream=sys.stdout))
else:
for outfile in args.outfiles:
if outfile and outfile != '-':
self.outputs.append(StreamOpener(outfile))
else:
self.outputs.append(StreamOpener(stream=sys.stdout))
if self.wip:
# Only run scenarios tagged with "wip". Additionally: use the
# "plain" formatter, do not capture stdout or logging output and
# stop at the first failure.
self.format = ['plain']
self.tags = ['wip']
self.stop = True
self.log_capture = False
self.stdout_capture = False
self.tags = TagExpression(self.tags or [])
if self.quiet:
self.show_source = False
self.show_snippets = False
if self.exclude_re:
self.exclude_re = re.compile(self.exclude_re)
if self.include_re:
self.include_re = re.compile(self.include_re)
if self.name:
# -- SELECT: Scenario-by-name, build regular expression.
self.name_re = self.build_name_re(self.name)
if self.junit:
# Buffer the output (it will be put into Junit report)
self.stdout_capture = True
self.stderr_capture = True
self.log_capture = True
self.reporters.append(JUnitReporter(self))
if self.summary:
self.reporters.append(SummaryReporter(self))
unknown_formats = self.collect_unknown_formats()
if unknown_formats:
parser.error("format=%s is unknown" % ", ".join(unknown_formats))
def worker(self, proc_number):
self.context.procnum = proc_number
while 1:
try:
joblist_index = self.joblist_index_queue.get_nowait()
except Exception, e:
break
current_job = self.joblist[joblist_index]
writebuf = StringIO.StringIO()
self.setfeature(current_job)
self.config.outputs = []
self.config.outputs.append(StreamOpener(stream=writebuf))
stream_openers = self.config.outputs
self.formatters = make_formatters(self.config, stream_openers)
for formatter in self.formatters:
formatter.uri(current_job.filename)
if self.step_registry is None:
self.step_registry = the_step_registry
start_time = time.strftime("%Y-%m-%d %H:%M:%S")
# if current_job.type == 'scenario' and getattr(self.config,
# 'parallel_scenario_feature_hooks'):
# self.run_hook('before_feature', self.context, current_job)
current_job.run(self)
# if current_job.type == 'scenario' and getattr(self.config,
# 'parallel_scenario_feature_hooks'):
# self.run_hook('after_feature', self.context, current_job)
end_time = time.strftime("%Y-%m-%d %H:%M:%S")
sys.stderr.write("%s: %s \n" %
(current_job.filename, current_job.status.name))
if current_job.type == 'feature':
for reporter in self.config.reporters:
reporter.feature(current_job)
job_report_text = self.generatereport(proc_number, current_job,
start_time, end_time,
writebuf)
if job_report_text:
results = {}
results['steps_passed'] = 0
results['steps_failed'] = 0
results['steps_skipped'] = 0
results['steps_undefined'] = 0
results['steps_untested'] = 0
results['jobtype'] = current_job.type
results['reportinginfo'] = job_report_text
results['status'] = str(current_job.status.name)
if current_job.type != 'feature':
results['uniquekey'] = \
current_job.filename + current_job.feature.name
else:
results['scenarios_passed'] = 0
results['scenarios_failed'] = 0
results['scenarios_skipped'] = 0
results['scenarios_untested'] = 0
# results['scenarios_undefined'] = 0
# results['scenarios_executing'] = 0
self.countscenariostatus(current_job, results)
self.countstepstatus(current_job, results)
if current_job.type != 'feature' and \
getattr(self.config, 'junit'):
results['junit_report'] = \
self.generate_junit_report(current_job, writebuf)
self.resultsqueue.put(results)
def _formatters(self, file, config):
stream_opener = StreamOpener(stream=file)
fs = formatters.get_formatter(config, [stream_opener])
for f in fs:
f.uri('<string>')
return fs
