def test_flow_error(self):
"""Create test flow with error component & validate its behavior.
We test the flow result was error and all the component run.
"""
MockFlow.blocks = (SuccessBlock, ErrorBlock)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.assertEqual(self.result.testsRun, 1,
"Result didn't run the correct number of tests")
self.assertEqual(len(self.result.errors), 1,
"Result didn't had the correct number of errors")
self.validate_blocks(test_flow, successes=1, errors=1)
# === Validate data object ===
self.assertFalse(test_flow.data.success,
'Flow data result should have been False')
self.assertEqual(test_flow.data.exception_type, TestOutcome.ERROR,
'Flow data status should have been error')
def test_unexpected_success(self):
"""Create test flow with unexpected success block and check behavior.
We test the flow result was failure and all the component run.
"""
MockFlow.blocks = (SuccessBlock, UnexpectedSuccessBlock)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.assertEqual(self.result.testsRun, 1,
"Result didn't run the correct number of tests")
self.assertEqual(len(self.result.failures), 1,
"Result didn't fail the correct number of tests")
self.validate_blocks(test_flow, successes=1, unexpected_successes=1)
# === Validate data object ===
self.assertFalse(test_flow.data.success,
'Flow data result should have been False')
self.assertEqual(test_flow.data.exception_type, TestOutcome.FAILED,
'Flow data status should have been failure')
def test_flow_expect_failure(self):
"""Validate behavior of test flow with blocks that fail with 'expect'.
We test the flow treats 'expect' failures like normal failures.
"""
MockFlow.blocks = (StoreFailuresBlock.params(mode=MODE_CRITICAL),
SuccessBlock.params(mode=MODE_CRITICAL),
StoreFailuresBlock.params(mode=MODE_FINALLY))
test_flow = MockFlow()
self.run_test(test_flow)
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.assertEqual(self.result.testsRun, 1,
"Result didn't run the correct number of tests")
self.assertEqual(len(self.result.failures), 1,
"Result didn't had the correct number of failures")
self.validate_blocks(test_flow, skips=1, failures=2)
# === Validate data object ===
self.assertFalse(test_flow.data.success,
'Flow data result should have been False')
self.assertEqual(test_flow.data.exception_type, TestOutcome.FAILED,
'Flow data status should have been failure')
def test_optional_blocks(self):
"""Validate behavior of block in OPTIONAL mode.
We check that other blocks run after a optional's failure but don't
after an error in an optional block.
"""
MockFlow.blocks = (SuccessBlock.params(mode=MODE_CRITICAL),
FailureBlock.params(mode=MODE_OPTIONAL),
SuccessBlock.params(mode=MODE_CRITICAL),
ErrorBlock.params(mode=MODE_OPTIONAL),
SuccessBlock.params(mode=MODE_CRITICAL))
test_flow = MockFlow()
self.run_test(test_flow)
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.assertEqual(self.result.testsRun, 1,
"Result didn't run the correct number of tests")
self.assertEqual(len(self.result.errors), 1,
"Result didn't had the correct number of errors")
self.validate_blocks(test_flow, successes=2, failures=1, errors=1,
skips=1)
# === Validate data object ===
self.assertFalse(test_flow.data.success,
'Flow data result should have been False')
self.assertEqual(test_flow.data.exception_type, TestOutcome.ERROR,
'Flow data status should have been error')
def test_parametrize(self):
"""Validate parametrize behavior.
* Checks that passing values via parametrize passes the 'inputs' check.
* Checks that the values set via parametrize are correct.
* Checks that passing values via parametrize is local to the block.
"""
PARAMETER_VALUE = 'some_value'
PARAMETER_NAME = 'some_parameter'
parameters = {PARAMETER_NAME: PARAMETER_VALUE}
# Block to check that input validates the parameter
InputsValidationBlock.inputs = ('res1', PARAMETER_NAME)
# Block to check that the correct value is injected into the block
ReadFromCommonBlock.READ_NAME = PARAMETER_NAME
ReadFromCommonBlock.READ_VALUE = PARAMETER_VALUE
MockFlow.blocks = (InputsValidationBlock.params(**parameters),
ReadFromCommonBlock.params(**parameters),
ReadFromCommonBlock)
test_flow = MockFlow()
self.run_test(test_flow)
# The third block should get an error since it wasn't injected with
# the parameters and it tries to read them.
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.validate_blocks(test_flow, successes=2, errors=1)
def test_finally_failure_blocks(self):
"""Validate behavior of blocks after FINALLY block failure.
We check that other blocks don't run after a finally's failure.
"""
MockFlow.blocks = (SuccessBlock.params(mode=MODE_CRITICAL),
FailureBlock.params(mode=MODE_FINALLY),
SuccessBlock.params(mode=MODE_CRITICAL),
SuccessBlock.params(mode=MODE_OPTIONAL))
test_flow = MockFlow()
self.run_test(test_flow)
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.assertEqual(self.result.testsRun, 1,
"Result didn't run the correct number of tests")
self.assertEqual(len(self.result.failures), 1,
"Result didn't fail the correct number of tests")
self.validate_blocks(test_flow, successes=1, failures=1, skips=2)
# === Validate data object ===
self.assertFalse(test_flow.data.success,
'Flow data result should have been False')
self.assertEqual(test_flow.data.exception_type, TestOutcome.FAILED,
'Flow data status should have been failure')
def test_inputs_static_check(self):
"""Test static check of inputs validation of blocks.
Run a flow with a block that expects an input it doesn't get,
then expect it to have an error.
"""
MockFlow.blocks = (create_reader_block(inject_name="missing_input",
inject_value=5),)
with self.assertRaises(AttributeError):
MockFlow()
def test_input_default_value(self):
"""Test that blocks' inputs' default value is injected by default."""
MockFlow.blocks = (create_reader_block(inject_value='default_value',
default='default_value'),)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.validate_blocks(test_flow, successes=1)
def test_inputs_static_check(self):
"""Test static check of inputs validation of blocks.
Run a flow with a block that expects an input it doesn't get,
then expect it to have an error.
"""
missing_input_name = 'noinput'
InputsValidationBlock.inputs = (missing_input_name, )
MockFlow.blocks = (InputsValidationBlock, )
with self.assertRaises(AttributeError):
MockFlow()
def test_shared_data_priority(self):
"""Test that shared data has higher priority than default values."""
MockFlow.blocks = (create_writer_block(inject_value='some_value'),
create_reader_block(inject_value='some_value',
default='default_value'))
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.validate_blocks(test_flow, successes=2)
def test_inputs_static_check_with_pipe(self):
"""Test static check of inputs validation of blocks when using pipes.
Run a flow with a block that expects an piped input it doesn't get,
then expect it to have an error.
"""
missing_input_name = 'noinput'
InputsValidationBlock.inputs = (missing_input_name, )
MockFlow.blocks = (InputsValidationBlock.params(
noinput=PipeTo(WriteToCommonBlock.INJECT_NAME)), )
with self.assertRaises(AttributeError):
MockFlow()
def test_inputs_static_check_with_pipe(self):
"""Test static check of inputs validation of blocks when using pipes.
Run a flow with a block that expects an piped input it doesn't get,
then expect it to have an error.
"""
class BlockWithInputs(SuccessBlock):
noinput = BlockInput()
MockFlow.blocks = (BlockWithInputs.params(
noinput=PipeTo('pipe_target')), )
with self.assertRaises(AttributeError):
MockFlow()
def test_pipes_happy_flow(self):
"""Validate parametrize behavior when using pipes."""
MockFlow.blocks = (
create_writer_block(inject_name='some_name'),
create_reader_block(inject_name='pipe_target').params(
pipe_target=PipeTo('some_name')))
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.validate_blocks(test_flow, successes=2)
def test_inputs_dynamic_check(self):
"""Test runtime validation of inputs of blocks.
Run a flow with a block that pretends to share data and a block that
needs this data as an input.
"""
pass_value = 'not_exist_value'
PretendToShareDataBlock.outputs = (pass_value, )
InputsValidationBlock.inputs = (pass_value, )
MockFlow.blocks = (PretendToShareDataBlock, InputsValidationBlock)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertEqual(len(self.result.errors), 1,
"Result didn't had the correct number of errors")
def test_optional_inputs_static_check(self):
"""Test static check of optional inputs validation of blocks.
Run a flow with a block that has an optional input,
then expect it to succeed.
"""
MockFlow.blocks = (create_reader_block(inject_name="missing_input",
inject_value=4,
default=5),)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.validate_blocks(test_flow, failures=1)
def test_local_dynamic_resources_locking(self):
"""Test that cases can dynamically lock resources.
* Runs a test that dynamically requests local resources.
* Validates that the other blocks don't have the resources.
* Validates that the resources were initialized and finalized.
"""
global_resource_name = 'test_res1'
dynamic_request_name = 'dynamic_resource'
dynamic_resource_name = 'test_res2'
class SharedDynamicResourceLockingBlock(DynamicResourceLockingBlock):
is_global = False
outputs = (dynamic_request_name,)
dynamic_resources = (request(dynamic_request_name,
DemoResource,
name=dynamic_resource_name),)
AttributeCheckingBlock.ATTRIBUTE_NAME = dynamic_request_name
MockFlow.blocks = (SharedDynamicResourceLockingBlock,
AttributeCheckingBlock)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.assertEqual(self.result.testsRun, 1,
"Result didn't run the correct number of tests")
self.assertEqual(len(self.result.failures), 1,
"Result didn't had the correct number of errors")
self.validate_blocks(test_flow, successes=1, failures=1)
# === Validate data object ===
self.assertFalse(test_flow.data.success,
'Flow data result should have been False')
test_resource = DemoResourceData.objects.get(name=global_resource_name)
self.validate_resource(test_resource)
test_resource = DemoResourceData.objects.get(
name=dynamic_resource_name)
self.validate_resource(test_resource)
def test_pipes_happy_flow(self):
"""Validate parametrize behavior when using pipes."""
# Block to check that the correct value is wrote through the pipe
ReadFromCommonBlock.READ_NAME = 'pipe_parameter'
ReadFromCommonBlock.READ_VALUE = WriteToCommonBlock.INJECT_VALUE
MockFlow.blocks = (
WriteToCommonBlock,
ReadFromCommonBlock.params(
pipe_parameter=PipeTo(WriteToCommonBlock.INJECT_NAME)))
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.validate_blocks(test_flow, successes=2)
def test_inputs_happy_flow(self):
"""Test behavior of inputs validation of blocks in positive case.
* The flow locks a resource.
* It's first block shares a value.
* The second block validates it has both the result and the shared
value using the 'inputs' field.
"""
InputsValidationBlock.inputs = ('res1', WriteToCommonBlock.INJECT_NAME)
MockFlow.blocks = (WriteToCommonBlock, InputsValidationBlock)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.validate_blocks(test_flow, successes=2)
def test_inputs_happy_flow(self):
"""Test behavior of inputs validation of blocks in positive case.
* The flow locks a resource.
* It's first block shares a value.
* The second block validates it has both the result and the shared
value using the 'inputs' field.
"""
MockFlow.blocks = (create_writer_block(inject_value='some_value'),
create_reader_block(inject_value='some_value'))
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.validate_blocks(test_flow, successes=2)
def test_inputs_check(self):
"""Test runtime validation of inputs of blocks.
Run a flow with a block that pretends to share data and a block that
needs this data as an input.
"""
class PretendToShareDataBlock(SuccessBlock):
pretend_output = BlockOutput()
MockFlow.blocks = (PretendToShareDataBlock,
create_reader_block(inject_name='pretend_output').
params(mode=MODE_FINALLY))
test_flow = MockFlow()
self.run_test(test_flow)
self.assertFalse(self.result.wasSuccessful(),
'Flow succeeded when it should have failed')
self.validate_blocks(test_flow, successes=1, failures=1)
def test_skip_and_success(self):
"""Validate test flow with success and skipped blocks."""
MockFlow.blocks = (SkipBlock, SuccessBlock)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.assertEqual(self.result.testsRun, 1,
"Flow didn't run the correct number of blocks")
self.validate_blocks(test_flow, successes=1, skips=1)
# === Validate data object ===
self.assertTrue(test_flow.data.success,
'Flow data result should have been True')
self.assertEqual(test_flow.data.exception_type, TestOutcome.SUCCESS,
'Flow data status should have been success')
def test_parametrize(self):
"""Validate parametrize behavior.
* Checks that passing values via parametrize passes the 'inputs' check.
* Checks that the values set via parametrize are correct.
* Checks that passing values via parametrize is local to the block.
"""
parameter_value = 'some_value'
parameter_name = 'some_parameter'
parameters = {parameter_name: parameter_value}
MockFlow.blocks = (create_reader_block(
inject_name=parameter_name,
inject_value=parameter_value).params(**parameters),)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.validate_blocks(test_flow, successes=1)
def test_failing_flow(self):
"""Test run delta with failing flow.
* Runs a flow with success failure & skipped blocks.
* Validates all the block were run.
* Runs the same flow with delta flag.
* Validates that all the blocks ran again.
"""
MockFlow.blocks = (SuccessBlock, SkipBlock, FailureBlock)
main_test = MockFlow()
result = self.create_result(main_test)
main_test.run(result)
self.validate_result(result, False, fails=1)
block_results1 = [block.data.exception_type for block in main_test]
main_test = MockFlow(run_data=self.run_data)
delta_result = self.create_result(main_test)
main_test.run(delta_result)
self.validate_result(delta_result, False, fails=1)
block_results2 = [block.data.exception_type for block in main_test]
self.assertEqual(block_results1, block_results2)
def test_shared_data_priority(self):
"""Test that shared data has higher priority than class fields."""
class BlockWithOptionalField(ReadFromCommonBlock):
pass
# Set class field
STATIC_VALUE = 'static_value'
setattr(BlockWithOptionalField, WriteToCommonBlock.INJECT_NAME,
STATIC_VALUE)
# Block to check that the correct value is injected into the block
BlockWithOptionalField.READ_NAME = WriteToCommonBlock.INJECT_NAME
BlockWithOptionalField.READ_VALUE = WriteToCommonBlock.INJECT_VALUE
MockFlow.blocks = (WriteToCommonBlock, BlockWithOptionalField)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.validate_blocks(test_flow, successes=2)
def test_happy_flow(self):
"""Create test flow with success components & validate run success.
We test the flow result was success and that all the components run.
"""
MockFlow.blocks = (SuccessBlock, SuccessBlock)
test_flow = MockFlow()
self.run_test(test_flow)
self.assertTrue(self.result.wasSuccessful(),
'Flow failed when it should have succeeded')
self.assertEqual(self.result.testsRun, 1,
"Flow didn't run the correct number of blocks")
self.validate_blocks(test_flow, successes=2)
# === Validate data object ===
self.assertTrue(test_flow.data.success,
'Flow data result should have been True')
self.assertEqual(test_flow.data.exception_type, TestOutcome.SUCCESS,
'Flow data status should have been success')
def test_flow_success(self):
"""Test run delta with successful flow.
* Runs a flow with success blocks.
* Validates all the tests were run.
* Runs the same suite with delta flag.
* Validates that the flow didn't run again.
* Runs the same suite with delta flag.
* Validates that the flow didn't run again.
"""
MockFlow.blocks = (SuccessBlock, SkipBlock, SuccessBlock)
main_test = MockFlow()
result = self.create_result(main_test)
main_test.run(result)
self.validate_result(result, True, successes=1)
main_test = MockFlow(run_data=self.run_data)
delta_result = self.create_result(main_test)
main_test.run(delta_result)
self.validate_result(delta_result, None, skips=1)
main_test = MockFlow(run_data=self.run_data)
delta_result = self.create_result(main_test)
main_test.run(delta_result)
self.validate_result(delta_result, None, skips=1)
