def test_runner_should_continue_running_steps_when_step_is_skipped_or_pending(
hook_registry, default_config, mocker):
"""The Runner should continue running Steps after a Step is skipped"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_step = mocker.MagicMock()
runner.run_step.side_effect = [
State.PASSED,
State.SKIPPED,
State.PENDING,
State.PASSED,
]
scenario_mock = mocker.MagicMock(name="Scenario")
scenario_mock.background = None
first_step = mocker.MagicMock(name="First Step")
second_step = mocker.MagicMock(name="Second Step")
third_step = mocker.MagicMock(name="Third Step")
fourth_step = mocker.MagicMock(name="Fourth Step")
scenario_mock.steps = [first_step, second_step, third_step, fourth_step]
# when
runner.run_scenario(scenario_mock)
# then
runner.run_step.assert_has_calls([
call(first_step),
call(second_step),
call(third_step),
call(fourth_step)
])
def test_runner_should_shuffle_scenarios_in_a_scenario_container_if_shuffle_scenarios_flag_set(
hook_registry, default_config, mocker):
"""
The Runner should shuffle the Scenarios within a Scenario Container before
running them if the shuffle Scenarios flag is set
"""
# given
default_config.shuffle_scenarios = True
runner = Runner(default_config, None, hook_registry)
runner.run_scenario = mocker.MagicMock()
scenario_container_mock = mocker.MagicMock(name="Rule")
scenario = mocker.MagicMock(name="Scenario")
scenario_container_mock.examples = [scenario]
mocker.patch("random.sample")
# when
runner.run_scenario_container(scenario_container_mock)
# then
import random
random.sample.assert_called_once_with(
scenario_container_mock.examples,
len(scenario_container_mock.examples))
def test_running_all(self):
"""
Test running a all features
"""
data = threading.local()
data.step_was_called = False
def some_step(step):
data.step_was_called = True
feature = Feature(1, "Feature", "Some feature", "somefile.feature", 1)
scenario = Scenario(1, 1, "Scenario", "Some scenario", "somefile.feature", 2, feature)
feature.scenarios.append(scenario)
step = Step(1, "Some step", "somefile.feature", 3, scenario, True)
step.definition_func = some_step
step.arguments = tuple()
step.keyword_arguments = {}
scenario.steps.append(step)
hook_mock = Mock()
hook_mock.call.return_value = True
runner = Runner(hook_mock)
runner.start([feature], None)
step.state.should.be.equal(Step.State.PASSED)
data.step_was_called.should.be.true
def test_running_a_feature(self):
"""
Test running a feature
"""
data = threading.local()
data.step_was_called = False
def some_step(step):
data.step_was_called = True
feature = Feature(1, "Feature", "Some feature", "somefile.feature", 1)
scenario = Scenario(1, 1, "Scenario", "Some scenario",
"somefile.feature", 2, feature)
feature.scenarios.append(scenario)
step = Step(1, "Some step", "somefile.feature", 3, scenario, True)
step.definition_func = some_step
argument_match_mock = Mock()
argument_match_mock.evaluate.return_value = (tuple(), {})
step.argument_match = argument_match_mock
scenario.steps.append(step)
hook_mock = Mock()
hook_mock.call.return_value = True
runner = Runner(hook_mock)
runner.run_feature(feature)
step.state.should.be.equal(Step.State.PASSED)
data.step_was_called.should.be.true
def test_running_a_scenario(self):
"""
Test running a scenario
"""
data = threading.local()
data.step_was_called = False
def some_step(step):
data.step_was_called = True
step = Step(1, "Some step", "somefile.feature", 3, None, True)
step.definition_func = some_step
step.arguments = tuple()
step.keyword_arguments = {}
scenario = Scenario(1, 1, "Scenario", "Some scenario", "somefile.feature", 2, None)
scenario.steps.append(step)
hook_mock = Mock()
hook_mock.call.return_value = True
runner = Runner(hook_mock)
returncode = runner.run_scenario(scenario)
returncode.should.be.equal(0)
step.state.should.be.equal(Step.State.PASSED)
data.step_was_called.should.be.true
def test_runner_should_exit_for_failed_scenario_in_scenario_container_if_early_exit_flag_is_set(
hook_registry, default_config, mocker):
"""
The Runner should exit for a failed Scenario in
Scenario Container if the early exit flag is set
"""
# given
default_config.early_exit = True
runner = Runner(default_config, None, hook_registry)
runner.run_scenario = mocker.MagicMock()
runner.run_scenario.side_effect = [
State.PASSED, State.FAILED, State.PASSED
]
scenario_container_mock = mocker.MagicMock(name="Scenario Container")
first_scenario = mocker.MagicMock(name="First Scenario")
second_scenario = mocker.MagicMock(name="Second Scenario")
third_scenario = mocker.MagicMock(name="Third Scenario")
scenario_container_mock.examples = [
first_scenario, second_scenario, third_scenario
]
# when
runner.run_scenario_container(scenario_container_mock)
# then
runner.run_scenario.assert_has_calls(
[call(first_scenario), call(second_scenario)])
def test_running_a_feature(self):
"""
Test running a feature
"""
data = threading.local()
data.step_was_called = False
def some_step(step):
data.step_was_called = True
feature = Feature(1, "Feature", "Some feature", "somefile.feature", 1)
scenario = Scenario(1, 1, "Scenario", "Some scenario", "somefile.feature", 2, feature)
feature.scenarios.append(scenario)
step = Step(1, "Some step", "somefile.feature", 3, scenario, True)
step.definition_func = some_step
step.arguments = tuple()
step.keyword_arguments = {}
scenario.steps.append(step)
hook_mock = Mock()
hook_mock.call.return_value = True
runner = Runner(hook_mock)
runner.run_feature(feature)
step.state.should.be.equal(Step.State.PASSED)
data.step_was_called.should.be.true
def run_features(core):
"""
Run the parsed features
:param Core core: the radish core object
"""
# FIXME: load dynamically
import radish.extensions.argumentexpressions
import radish.extensions.time_recorder
import radish.extensions.syslog_writer
import radish.extensions.console_writer
import radish.extensions.endreport_writer
import radish.extensions.failure_inspector
import radish.extensions.failure_debugger
import radish.extensions.bdd_xml_writer
# set needed configuration
world.config.expand = True
# load user's custom python files
loader = Loader(world.config.basedir)
loader.load_all()
# match feature file steps with user's step definitions
Matcher.merge_steps(core.features, StepRegistry().steps)
# run parsed features
if world.config.marker == "time.time()":
world.config.marker = int(time())
# scenario choice
amount_of_scenarios = sum(len(f.scenarios) for f in core.features_to_run)
if world.config.scenarios:
world.config.scenarios = [int(s) for s in world.config.scenarios.split(",")]
for s in world.config.scenarios:
if s <= 0 or s > amount_of_scenarios:
raise ScenarioNotFoundError(s, amount_of_scenarios)
# tags
if world.config.feature_tags:
world.config.feature_tags = [t for t in world.config.feature_tags.split(",")]
for tag in world.config.feature_tags:
if not any(f for f in core.features if tag in [t.name for t in f.tags]):
raise FeatureTagNotFoundError(tag)
if world.config.scenario_tags:
world.config.scenario_tags = [t for t in world.config.scenario_tags.split(",")]
for tag in world.config.scenario_tags:
if not any(s for f in core.features for s in f.scenarios if tag in [t.name for t in s.tags]):
raise ScenarioTagNotFoundError(tag)
runner = Runner(HookRegistry(), early_exit=world.config.early_exit)
runner.start(core.features_to_run, marker=world.config.marker)
def test_start_run_without_features(hook_registry, default_config):
"""When starting the Runner without any Features it should pass and call the ``all`` hooks"""
# given
runner = Runner(default_config, None, hook_registry)
# when
status = runner.start([])
# then
assert status
hook_registry.call.assert_has_calls(
[call("all", "before", False, []),
call("all", "after", False, [])])
def test_skip_single_step(hookregistry, mocker):
"""
Test skipping a single Step
"""
# given
runner = Runner(hookregistry)
step = mocker.MagicMock()
step.skip.return_value = None
# when
runner.skip_step(step)
# then
assert step.skip.call_count == 1
def test_skip_single_step(hookregistry, mocker):
"""
Test skipping a single Step
"""
# given
runner = Runner(hookregistry)
step = mocker.MagicMock()
step.skip.return_value = None
# when
runner.skip_step(step)
# then
assert step.skip.call_count == 1
def show_features(core):
"""
Show the parsed features
"""
# FIXME: load dynamically
import radish.extensions.console_writer
# set needed configuration
world.config.write_steps_once = True
if not sys.stdout.isatty():
world.config.no_ansi = True
runner = Runner(HookRegistry(), dry_run=True)
runner.start(core.features_to_run, marker="show")
def test_run_single_step(run_state, expected_returncode, hookregistry, mocker):
"""
Test running a single Step
"""
# given
runner = Runner(hookregistry)
step = mocker.MagicMock()
step.run.return_value = run_state
# when
returncode = runner.run_step(step)
# then
assert returncode == expected_returncode
assert step.run.call_count == 1
def test_run_single_step(run_state, expected_returncode, hookregistry, mocker):
"""
Test running a single Step
"""
# given
runner = Runner(hookregistry)
step = mocker.MagicMock()
step.run.return_value = run_state
# when
returncode = runner.run_step(step)
# then
assert returncode == expected_returncode
assert step.run.call_count == 1
def test_runner_should_fail_step_when_it_cannot_be_matched(
hook_registry, default_config, mocker):
"""The Runner should fail a Step when it cannot be matched with any Step Implementation"""
runner = Runner(default_config, None, hook_registry)
step_mock = mocker.MagicMock(name="Step")
matcher_mock = mocker.patch("radish.runner.matcher")
match_exc = RadishError("buuh!")
matcher_mock.match_step.side_effect = match_exc
# when
runner.run_step(step_mock)
# then
step_mock.fail.assert_called_once_with(match_exc)
def test_runner_should_call_hooks_when_running_a_feature(
hook_registry, default_config, mocker):
"""The Runner should call the ``each_feature`` hooks when running a Feature"""
# given
runner = Runner(default_config, None, hook_registry)
feature_mock = mocker.MagicMock(name="Feature")
# when
runner.run_feature(feature_mock)
# then
hook_registry.call.assert_has_calls([
call("each_feature", "before", False, feature_mock),
call("each_feature", "after", False, feature_mock),
])
def test_runner_should_call_hooks_when_running_a_step(hook_registry,
default_config, mocker):
"""The Runner should call the ``each_step`` hooks when running a Step"""
# given
runner = Runner(default_config, None, hook_registry)
step_mock = mocker.MagicMock(name="Step")
# when
runner.run_step(step_mock)
# then
hook_registry.call.assert_has_calls([
call("each_step", "before", False, step_mock),
call("each_step", "after", False, step_mock),
])
def test_run_single_step_show_only(hookregistry, mocker):
"""
Test running single step when show only mode is on
"""
# given
runner = Runner(hookregistry, show_only=True)
step = mocker.MagicMock()
step.run.return_value = Step.State.FAILED
# when
returncode = runner.run_step(step)
# then
assert returncode == 0
assert step.run.call_count == 0
def test_run_single_step_show_only(hookregistry, mocker):
"""
Test running single step when show only mode is on
"""
# given
runner = Runner(hookregistry, show_only=True)
step = mocker.MagicMock()
step.run.return_value = Step.State.FAILED
# when
returncode = runner.run_step(step)
# then
assert returncode == 0
assert step.run.call_count == 0
def test_start_run_should_iterate_all_given_features(hook_registry,
default_config, mocker):
"""All Features given to run should be iterated"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_feature = mocker.MagicMock()
first_feature = mocker.MagicMock(name="First Feature")
second_feature = mocker.MagicMock(name="Second Feature")
# when
runner.start([first_feature, second_feature])
# given
runner.run_feature.assert_has_calls(
[call(first_feature), call(second_feature)])
def test_runner_should_run_each_rule_in_a_feature(hook_registry,
default_config, mocker):
"""The Runner should run each Rule from a Feature"""
# given
runner = Runner(default_config, None, hook_registry)
feature_mock = mocker.MagicMock(name="Feature")
first_rule = mocker.MagicMock(name="First Rule")
second_rule = mocker.MagicMock(name="Second Rule")
feature_mock.rules = [first_rule, second_rule]
runner.run_rule = mocker.MagicMock()
# when
runner.run_feature(feature_mock)
# then
runner.run_rule.assert_has_calls([call(first_rule), call(second_rule)])
def test_runner_should_run_scenario_loop_and_outline_as_scenario_container(
scenario_container_type, hook_registry, default_config, mocker):
"""The Runner should run a ScenarioLoop and ScenarioOutline as a Scenario Container"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_scenario_container = mocker.MagicMock()
rule_mock = mocker.MagicMock(name="Rule")
scenario = mocker.MagicMock(name=scenario_container_type.__name__,
spec=scenario_container_type)
rule_mock.scenarios = [scenario]
# when
runner.run_rule(rule_mock)
# then
runner.run_scenario_container.assert_has_calls([call(scenario)])
def test_runner_should_run_step_after_being_matched_in_normal_mode(
hook_registry, default_config, mocker):
"""
The Runner should run a Step after it's being
successfully matched with a Step Implementation
in the normal mode.
"""
# given
runner = Runner(default_config, None, hook_registry)
step_mock = mocker.MagicMock(name="Step")
mocker.patch("radish.runner.matcher")
# when
runner.run_step(step_mock)
# then
step_mock.run.assert_called_once_with(ANY)
def test_runner_should_iterate_all_scenarios_when_running_a_scenario_container(
hook_registry, default_config, mocker):
"""The Runner should iterate all Scenarios when running a Scenario Container"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_scenario = mocker.MagicMock()
scenario_container_mock = mocker.MagicMock(name="Scenario Container")
first_scenario = mocker.MagicMock(name="First Scenario")
second_scenario = mocker.MagicMock(name="Second Scenario")
scenario_container_mock.examples = [first_scenario, second_scenario]
# when
runner.run_scenario_container(scenario_container_mock)
# then
runner.run_scenario.assert_has_calls(
[call(first_scenario), call(second_scenario)])
def test_runner_should_iterate_all_steps_in_a_scenario(hook_registry,
default_config, mocker):
"""The Runner should iterate all Steps in a Scenario"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_step = mocker.MagicMock()
runner.run_step.return_value = State.PASSED
scenario_mock = mocker.MagicMock(name="Scenario")
scenario_mock.background = None
first_step = mocker.MagicMock(name="First Step")
second_step = mocker.MagicMock(name="Second Step")
scenario_mock.steps = [first_step, second_step]
# when
runner.run_scenario(scenario_mock)
# then
runner.run_step.assert_has_calls([call(first_step), call(second_step)])
def test_should_return_good_status_if_all_features_passed(
hook_registry, default_config, mocker):
"""
The Runner should return a good status when finished if all features passed in normal mode
"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_feature = mocker.MagicMock()
runner.run_feature.side_effect = [State.PASSED, State.PASSED]
first_feature = mocker.MagicMock(name="First Feature", state=State.PASSED)
second_feature = mocker.MagicMock(name="Second Feature",
state=State.PASSED)
# when
status = runner.start([first_feature, second_feature])
# given
assert status
def test_debug_single_step(debug_state, expected_returncode, world_config,
hookregistry, mocker):
"""
Test debugging a single Step
"""
# given
runner = Runner(hookregistry)
step = mocker.MagicMock()
step.debug.return_value = debug_state
# set debug mode
world_config.debug_steps = True
# when
returncode = runner.run_step(step)
# then
assert returncode == expected_returncode
assert step.debug.call_count == 1
def test_runner_should_stop_running_steps_after_first_failed(
hook_registry, default_config, mocker):
"""The Runner should stop running Steps after the first Step failed in normal mode"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_step = mocker.MagicMock()
runner.run_step.side_effect = [State.PASSED, State.FAILED]
scenario_mock = mocker.MagicMock(name="Scenario")
scenario_mock.background = None
first_step = mocker.MagicMock(name="First Step")
second_step = mocker.MagicMock(name="Second Step")
scenario_mock.steps = [first_step, second_step]
# when
runner.run_scenario(scenario_mock)
# then
runner.run_step.assert_has_calls([call(first_step)])
def test_returncode_of_runner(self):
"""
Test returncode of run functions in Runner
"""
def some_passed_step(step):
pass
def some_failed_step(step):
raise AttributeError(
"I expect this error to test the behavior of a failed step")
feature = Feature(1, "Feature", "Some feature", "somefile.feature", 1)
scenario = Scenario(1, 1, "Scenario", "Some scenario",
"somefile.feature", 2, feature)
feature.scenarios.append(scenario)
step1 = Step(1, "Some passed step", "somefile.feature", 3, scenario,
True)
step1.definition_func = some_passed_step
argument_match_mock = Mock()
argument_match_mock.evaluate.return_value = (tuple(), {})
step1.argument_match = argument_match_mock
scenario.steps.append(step1)
step2 = Step(2, "Some failed step", "somefile.feature", 4, scenario,
True)
step2.definition_func = some_failed_step
argument_match_mock = Mock()
argument_match_mock.evaluate.return_value = (tuple(), {})
step2.argument_match = argument_match_mock
scenario.steps.append(step2)
hook_mock = Mock()
hook_mock.call.return_value = True
runner = Runner(hook_mock)
returncode = runner.run_feature(feature)
returncode.should.be.equal(1)
step1.state.should.be.equal(Step.State.PASSED)
step2.state.should.be.equal(Step.State.FAILED)
scenario.state.should.be.equal(Step.State.FAILED)
feature.state.should.be.equal(Step.State.FAILED)
def test_debug_single_step(
debug_state, expected_returncode, world_config, hookregistry, mocker
):
"""
Test debugging a single Step
"""
# given
runner = Runner(hookregistry)
step = mocker.MagicMock()
step.debug.return_value = debug_state
# set debug mode
world_config.debug_steps = True
# when
returncode = runner.run_step(step)
# then
assert returncode == expected_returncode
assert step.debug.call_count == 1
def test_should_only_run_feature_which_have_to_run(hook_registry,
default_config, mocker):
"""The Runner should only run features which need to be run"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_feature = mocker.MagicMock()
first_feature = mocker.MagicMock(name="First Feature")
first_feature.has_to_run.return_value = True
second_feature = mocker.MagicMock(name="Second Feature")
second_feature.has_to_run.return_value = False
third_feature = mocker.MagicMock(name="Third Feature")
third_feature.has_to_run.return_value = True
# when
runner.start([first_feature, second_feature, third_feature])
# given
runner.run_feature.assert_has_calls(
[call(first_feature), call(third_feature)])
def test_runner_should_not_run_nor_debug_step_after_being_matched_in_dry_run_mode(
hook_registry, default_config, mocker):
"""
The Runner should not run nor debug a Step after it's being
successfully matched with a Step Implementation in the dry run mode.
"""
# given
default_config.dry_run_mode = True
runner = Runner(default_config, None, hook_registry)
step_mock = mocker.MagicMock(name="Step")
mocker.patch("radish.runner.matcher")
# when
runner.run_step(step_mock)
# then
step_mock.run.assert_not_called()
step_mock.debug.assert_not_called()
def test_should_exit_for_failed_feature_if_early_exit_set(
hook_registry, default_config, mocker):
"""The Runner should exit early if a Feature failes if the early exit flag is set"""
# given
default_config.early_exit = True
runner = Runner(default_config, None, hook_registry)
runner.run_feature = mocker.MagicMock()
runner.run_feature.side_effect = [State.PASSED, State.FAILED, State.PASSED]
first_feature = mocker.MagicMock(name="First Feature")
second_feature = mocker.MagicMock(name="Second Feature")
third_feature = mocker.MagicMock(name="Third Feature")
# when
runner.start([first_feature, second_feature, third_feature])
# given
runner.run_feature.assert_has_calls(
[call(first_feature), call(second_feature)])
def test_runner_should_only_run_rule_which_need_to_be_run(
hook_registry, default_config, mocker):
"""The Runner should run only the Rules which need to be run"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_rule = mocker.MagicMock()
feature_mock = mocker.MagicMock(name="Feature")
first_rule = mocker.MagicMock(name="First Rule")
first_rule.has_to_run.return_value = True
second_rule = mocker.MagicMock(name="Second Rule")
second_rule.has_to_run.return_value = False
third_rule = mocker.MagicMock(name="Third Rule")
third_rule.has_to_run.return_value = True
feature_mock.rules = [first_rule, second_rule, third_rule]
# when
runner.run_feature(feature_mock)
# then
runner.run_rule.assert_has_calls([call(first_rule), call(third_rule)])
def test_running_a_step(self):
"""
Test running a step
"""
data = threading.local()
data.step_was_called = False
def some_step(step):
data.step_was_called = True
step = Step(1, "Some step", "somefile.feature", 3, None, True)
step.definition_func = some_step
step.arguments = tuple()
step.keyword_arguments = {}
hook_mock = Mock()
hook_mock.call.return_value = True
runner = Runner(hook_mock)
runner.run_step(step)
step.state.should.be.equal(Step.State.PASSED)
data.step_was_called.should.be.true
def test_runner_should_exit_for_failed_rule_if_early_exit_flag_is_set(
hook_registry, default_config, mocker):
"""The Runner should exit for a failed Rule if the early exit flag is set"""
# given
default_config.early_exit = True
runner = Runner(default_config, None, hook_registry)
runner.run_rule = mocker.MagicMock()
runner.run_rule.side_effect = [State.PASSED, State.FAILED, State.PASSED]
feature_mock = mocker.MagicMock(name="Feature")
first_rule = mocker.MagicMock(name="First Rule")
second_rule = mocker.MagicMock(name="Second Rule")
third_rule = mocker.MagicMock(name="Third Rule")
feature_mock.rules = [first_rule, second_rule, third_rule]
# when
runner.run_feature(feature_mock)
# then
runner.run_rule.assert_has_calls([call(first_rule), call(second_rule)])
def test_runner_should_only_not_run_steps_in_a_scenario_if_background_not_passed(
hook_registry, default_config, mocker):
"""
The Runner should not run Steps in a Scenario
if the Background is available and did not pass.
"""
# given
runner = Runner(default_config, None, hook_registry)
runner.run_step = mocker.MagicMock()
runner.run_step.return_value = State.PASSED
scenario_mock = mocker.MagicMock(name="Scenario")
scenario_mock.background.state = State.FAILED
first_step = mocker.MagicMock(name="First Step")
scenario_mock.steps = [first_step]
# when
runner.run_scenario(scenario_mock)
# then
runner.run_step.assert_not_called()
def test_runner_should_run_all_steps_even_when_failed_in_dry_run_mode(
hook_registry, default_config, mocker):
"""The Runner should runn all Steps even when one failed in the dry run mode"""
# given
default_config.dry_run_mode = True
runner = Runner(default_config, None, hook_registry)
runner.run_step = mocker.MagicMock()
runner.run_step.side_effect = [State.FAILED, State.UNTESTED]
scenario_mock = mocker.MagicMock(name="Scenario")
scenario_mock.background = None
first_step = mocker.MagicMock(name="First Step")
second_step = mocker.MagicMock(name="Second Step")
scenario_mock.steps = [first_step, second_step]
# when
runner.run_scenario(scenario_mock)
# then
runner.run_step.assert_has_calls([call(first_step), call(second_step)])
def test_should_call_hooks_in_correct_order(hookregistry, mocker):
"""
Test that hooks are called in correct order.
Correct order meaning:
* before hooks: ascending order
* after hooks: descending order
"""
# given
runner = Runner(hookregistry)
# register hooks
data = []
def first_before_stub(*args):
data.append(1)
def first_after_stub(*args):
data.append(1)
def second_before_stub(*args):
data.append(2)
def second_after_stub(*args):
data.append(2)
hookregistry.register("before", "each_step", second_before_stub, order=2)
hookregistry.register("after", "each_step", second_after_stub, order=2)
hookregistry.register("before", "each_step", first_before_stub, order=1)
hookregistry.register("after", "each_step", first_after_stub, order=1)
# setup dummy step
step = mocker.MagicMock()
# when
runner.run_step(step)
# then
assert data == [1, 2, 2, 1]
def test_returncode_of_runner(self):
"""
Test returncode of run functions in Runner
"""
def some_passed_step(step):
pass
def some_failed_step(step):
raise AttributeError("I expect this error to test the behavior of a failed step")
feature = Feature(1, "Feature", "Some feature", "somefile.feature", 1)
scenario = Scenario(1, 1, "Scenario", "Some scenario", "somefile.feature", 2, feature)
feature.scenarios.append(scenario)
step1 = Step(1, "Some passed step", "somefile.feature", 3, scenario, True)
step1.definition_func = some_passed_step
step1.arguments = tuple()
step1.keyword_arguments = {}
scenario.steps.append(step1)
step2 = Step(2, "Some failed step", "somefile.feature", 4, scenario, True)
step2.definition_func = some_failed_step
step2.arguments = tuple()
step2.keyword_arguments = {}
scenario.steps.append(step2)
hook_mock = Mock()
hook_mock.call.return_value = True
runner = Runner(hook_mock)
returncode = runner.run_feature(feature)
returncode.should.be.equal(1)
step1.state.should.be.equal(Step.State.PASSED)
step2.state.should.be.equal(Step.State.FAILED)
scenario.state.should.be.equal(Step.State.FAILED)
feature.state.should.be.equal(Step.State.FAILED)