def test_flow_success(self): """Test run delta with successful flow. * Runs a flow with success blocks. * Validates all the tests were run. * Runs the same suite with delta flag. * Validates that the flow didn't run again. * Runs the same suite with delta flag. * Validates that the flow didn't run again. """ MockFlow.blocks = (SuccessBlock, SkipBlock, SuccessBlock) main_test = MockFlow() result = self.create_result(main_test) main_test.run(result) self.validate_result(result, True, successes=1) main_test = MockFlow(run_data=self.run_data) delta_result = self.create_result(main_test) main_test.run(delta_result) self.validate_result(delta_result, None, skips=1) main_test = MockFlow(run_data=self.run_data) delta_result = self.create_result(main_test) main_test.run(delta_result) self.validate_result(delta_result, None, skips=1)
def test_unexpected_success(self): """Create test flow with unexpected success block and check behavior. We test the flow result was failure and all the component run. """ MockFlow.blocks = (SuccessBlock, UnexpectedSuccessBlock) test_flow = MockFlow() self.run_test(test_flow) self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.assertEqual(self.result.testsRun, 1, "Result didn't run the correct number of tests") self.assertEqual(len(self.result.failures), 1, "Result didn't fail the correct number of tests") self.validate_blocks(test_flow, successes=1, unexpected_successes=1) # === Validate data object === self.assertFalse(test_flow.data.success, 'Flow data result should have been False') self.assertEqual(test_flow.data.exception_type, TestOutcome.FAILED, 'Flow data status should have been failure')
def test_flow_error(self): """Create test flow with error component & validate its behavior. We test the flow result was error and all the component run. """ MockFlow.blocks = (SuccessBlock, ErrorBlock) test_flow = MockFlow() self.run_test(test_flow) self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.assertEqual(self.result.testsRun, 1, "Result didn't run the correct number of tests") self.assertEqual(len(self.result.errors), 1, "Result didn't had the correct number of errors") self.validate_blocks(test_flow, successes=1, errors=1) # === Validate data object === self.assertFalse(test_flow.data.success, 'Flow data result should have been False') self.assertEqual(test_flow.data.exception_type, TestOutcome.ERROR, 'Flow data status should have been error')
def test_flow_expect_failure(self): """Validate behavior of test flow with blocks that fail with 'expect'. We test the flow treats 'expect' failures like normal failures. """ MockFlow.blocks = (StoreFailuresBlock.params(mode=MODE_CRITICAL), SuccessBlock.params(mode=MODE_CRITICAL), StoreFailuresBlock.params(mode=MODE_FINALLY)) test_flow = MockFlow() self.run_test(test_flow) self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.assertEqual(self.result.testsRun, 1, "Result didn't run the correct number of tests") self.assertEqual(len(self.result.failures), 1, "Result didn't had the correct number of failures") self.validate_blocks(test_flow, skips=1, failures=2) # === Validate data object === self.assertFalse(test_flow.data.success, 'Flow data result should have been False') self.assertEqual(test_flow.data.exception_type, TestOutcome.FAILED, 'Flow data status should have been failure')
def test_optional_blocks(self): """Validate behavior of block in OPTIONAL mode. We check that other blocks run after a optional's failure but don't after an error in an optional block. """ MockFlow.blocks = (SuccessBlock.params(mode=MODE_CRITICAL), FailureBlock.params(mode=MODE_OPTIONAL), SuccessBlock.params(mode=MODE_CRITICAL), ErrorBlock.params(mode=MODE_OPTIONAL), SuccessBlock.params(mode=MODE_CRITICAL)) test_flow = MockFlow() self.run_test(test_flow) self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.assertEqual(self.result.testsRun, 1, "Result didn't run the correct number of tests") self.assertEqual(len(self.result.errors), 1, "Result didn't had the correct number of errors") self.validate_blocks(test_flow, successes=2, failures=1, errors=1, skips=1) # === Validate data object === self.assertFalse(test_flow.data.success, 'Flow data result should have been False') self.assertEqual(test_flow.data.exception_type, TestOutcome.ERROR, 'Flow data status should have been error')
def test_parametrize(self): """Validate parametrize behavior. * Checks that passing values via parametrize passes the 'inputs' check. * Checks that the values set via parametrize are correct. * Checks that passing values via parametrize is local to the block. """ PARAMETER_VALUE = 'some_value' PARAMETER_NAME = 'some_parameter' parameters = {PARAMETER_NAME: PARAMETER_VALUE} # Block to check that input validates the parameter InputsValidationBlock.inputs = ('res1', PARAMETER_NAME) # Block to check that the correct value is injected into the block ReadFromCommonBlock.READ_NAME = PARAMETER_NAME ReadFromCommonBlock.READ_VALUE = PARAMETER_VALUE MockFlow.blocks = (InputsValidationBlock.params(**parameters), ReadFromCommonBlock.params(**parameters), ReadFromCommonBlock) test_flow = MockFlow() self.run_test(test_flow) # The third block should get an error since it wasn't injected with # the parameters and it tries to read them. self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.validate_blocks(test_flow, successes=2, errors=1)
def test_finally_failure_blocks(self): """Validate behavior of blocks after FINALLY block failure. We check that other blocks don't run after a finally's failure. """ MockFlow.blocks = (SuccessBlock.params(mode=MODE_CRITICAL), FailureBlock.params(mode=MODE_FINALLY), SuccessBlock.params(mode=MODE_CRITICAL), SuccessBlock.params(mode=MODE_OPTIONAL)) test_flow = MockFlow() self.run_test(test_flow) self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.assertEqual(self.result.testsRun, 1, "Result didn't run the correct number of tests") self.assertEqual(len(self.result.failures), 1, "Result didn't fail the correct number of tests") self.validate_blocks(test_flow, successes=1, failures=1, skips=2) # === Validate data object === self.assertFalse(test_flow.data.success, 'Flow data result should have been False') self.assertEqual(test_flow.data.exception_type, TestOutcome.FAILED, 'Flow data status should have been failure')
def test_inputs_static_check(self): """Test static check of inputs validation of blocks. Run a flow with a block that expects an input it doesn't get, then expect it to have an error. """ MockFlow.blocks = (create_reader_block(inject_name="missing_input", inject_value=5),) with self.assertRaises(AttributeError): MockFlow()
def test_input_default_value(self): """Test that blocks' inputs' default value is injected by default.""" MockFlow.blocks = (create_reader_block(inject_value='default_value', default='default_value'),) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.validate_blocks(test_flow, successes=1)
def test_inputs_static_check(self): """Test static check of inputs validation of blocks. Run a flow with a block that expects an input it doesn't get, then expect it to have an error. """ missing_input_name = 'noinput' InputsValidationBlock.inputs = (missing_input_name, ) MockFlow.blocks = (InputsValidationBlock, ) with self.assertRaises(AttributeError): MockFlow()
def test_shared_data_priority(self): """Test that shared data has higher priority than default values.""" MockFlow.blocks = (create_writer_block(inject_value='some_value'), create_reader_block(inject_value='some_value', default='default_value')) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.validate_blocks(test_flow, successes=2)
def test_inputs_static_check_with_pipe(self): """Test static check of inputs validation of blocks when using pipes. Run a flow with a block that expects an piped input it doesn't get, then expect it to have an error. """ missing_input_name = 'noinput' InputsValidationBlock.inputs = (missing_input_name, ) MockFlow.blocks = (InputsValidationBlock.params( noinput=PipeTo(WriteToCommonBlock.INJECT_NAME)), ) with self.assertRaises(AttributeError): MockFlow()
def test_inputs_static_check_with_pipe(self): """Test static check of inputs validation of blocks when using pipes. Run a flow with a block that expects an piped input it doesn't get, then expect it to have an error. """ class BlockWithInputs(SuccessBlock): noinput = BlockInput() MockFlow.blocks = (BlockWithInputs.params( noinput=PipeTo('pipe_target')), ) with self.assertRaises(AttributeError): MockFlow()
def test_pipes_happy_flow(self): """Validate parametrize behavior when using pipes.""" MockFlow.blocks = ( create_writer_block(inject_name='some_name'), create_reader_block(inject_name='pipe_target').params( pipe_target=PipeTo('some_name'))) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.validate_blocks(test_flow, successes=2)
def test_failing_flow(self): """Test run delta with failing flow. * Runs a flow with success failure & skipped blocks. * Validates all the block were run. * Runs the same flow with delta flag. * Validates that all the blocks ran again. """ MockFlow.blocks = (SuccessBlock, SkipBlock, FailureBlock) main_test = MockFlow() result = self.create_result(main_test) main_test.run(result) self.validate_result(result, False, fails=1) block_results1 = [block.data.exception_type for block in main_test] main_test = MockFlow(run_data=self.run_data) delta_result = self.create_result(main_test) main_test.run(delta_result) self.validate_result(delta_result, False, fails=1) block_results2 = [block.data.exception_type for block in main_test] self.assertEqual(block_results1, block_results2)
def test_inputs_dynamic_check(self): """Test runtime validation of inputs of blocks. Run a flow with a block that pretends to share data and a block that needs this data as an input. """ pass_value = 'not_exist_value' PretendToShareDataBlock.outputs = (pass_value, ) InputsValidationBlock.inputs = (pass_value, ) MockFlow.blocks = (PretendToShareDataBlock, InputsValidationBlock) test_flow = MockFlow() self.run_test(test_flow) self.assertEqual(len(self.result.errors), 1, "Result didn't had the correct number of errors")
def test_optional_inputs_static_check(self): """Test static check of optional inputs validation of blocks. Run a flow with a block that has an optional input, then expect it to succeed. """ MockFlow.blocks = (create_reader_block(inject_name="missing_input", inject_value=4, default=5),) test_flow = MockFlow() self.run_test(test_flow) self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.validate_blocks(test_flow, failures=1)
def test_local_dynamic_resources_locking(self): """Test that cases can dynamically lock resources. * Runs a test that dynamically requests local resources. * Validates that the other blocks don't have the resources. * Validates that the resources were initialized and finalized. """ global_resource_name = 'test_res1' dynamic_request_name = 'dynamic_resource' dynamic_resource_name = 'test_res2' class SharedDynamicResourceLockingBlock(DynamicResourceLockingBlock): is_global = False outputs = (dynamic_request_name,) dynamic_resources = (request(dynamic_request_name, DemoResource, name=dynamic_resource_name),) AttributeCheckingBlock.ATTRIBUTE_NAME = dynamic_request_name MockFlow.blocks = (SharedDynamicResourceLockingBlock, AttributeCheckingBlock) test_flow = MockFlow() self.run_test(test_flow) self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.assertEqual(self.result.testsRun, 1, "Result didn't run the correct number of tests") self.assertEqual(len(self.result.failures), 1, "Result didn't had the correct number of errors") self.validate_blocks(test_flow, successes=1, failures=1) # === Validate data object === self.assertFalse(test_flow.data.success, 'Flow data result should have been False') test_resource = DemoResourceData.objects.get(name=global_resource_name) self.validate_resource(test_resource) test_resource = DemoResourceData.objects.get( name=dynamic_resource_name) self.validate_resource(test_resource)
def test_pipes_happy_flow(self): """Validate parametrize behavior when using pipes.""" # Block to check that the correct value is wrote through the pipe ReadFromCommonBlock.READ_NAME = 'pipe_parameter' ReadFromCommonBlock.READ_VALUE = WriteToCommonBlock.INJECT_VALUE MockFlow.blocks = ( WriteToCommonBlock, ReadFromCommonBlock.params( pipe_parameter=PipeTo(WriteToCommonBlock.INJECT_NAME))) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.validate_blocks(test_flow, successes=2)
def test_inputs_happy_flow(self): """Test behavior of inputs validation of blocks in positive case. * The flow locks a resource. * It's first block shares a value. * The second block validates it has both the result and the shared value using the 'inputs' field. """ MockFlow.blocks = (create_writer_block(inject_value='some_value'), create_reader_block(inject_value='some_value')) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.validate_blocks(test_flow, successes=2)
def test_inputs_happy_flow(self): """Test behavior of inputs validation of blocks in positive case. * The flow locks a resource. * It's first block shares a value. * The second block validates it has both the result and the shared value using the 'inputs' field. """ InputsValidationBlock.inputs = ('res1', WriteToCommonBlock.INJECT_NAME) MockFlow.blocks = (WriteToCommonBlock, InputsValidationBlock) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.validate_blocks(test_flow, successes=2)
def test_inputs_check(self): """Test runtime validation of inputs of blocks. Run a flow with a block that pretends to share data and a block that needs this data as an input. """ class PretendToShareDataBlock(SuccessBlock): pretend_output = BlockOutput() MockFlow.blocks = (PretendToShareDataBlock, create_reader_block(inject_name='pretend_output'). params(mode=MODE_FINALLY)) test_flow = MockFlow() self.run_test(test_flow) self.assertFalse(self.result.wasSuccessful(), 'Flow succeeded when it should have failed') self.validate_blocks(test_flow, successes=1, failures=1)
def test_skip_and_success(self): """Validate test flow with success and skipped blocks.""" MockFlow.blocks = (SkipBlock, SuccessBlock) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.assertEqual(self.result.testsRun, 1, "Flow didn't run the correct number of blocks") self.validate_blocks(test_flow, successes=1, skips=1) # === Validate data object === self.assertTrue(test_flow.data.success, 'Flow data result should have been True') self.assertEqual(test_flow.data.exception_type, TestOutcome.SUCCESS, 'Flow data status should have been success')
def test_parametrize(self): """Validate parametrize behavior. * Checks that passing values via parametrize passes the 'inputs' check. * Checks that the values set via parametrize are correct. * Checks that passing values via parametrize is local to the block. """ parameter_value = 'some_value' parameter_name = 'some_parameter' parameters = {parameter_name: parameter_value} MockFlow.blocks = (create_reader_block( inject_name=parameter_name, inject_value=parameter_value).params(**parameters),) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.validate_blocks(test_flow, successes=1)
def test_shared_data_priority(self): """Test that shared data has higher priority than class fields.""" class BlockWithOptionalField(ReadFromCommonBlock): pass # Set class field STATIC_VALUE = 'static_value' setattr(BlockWithOptionalField, WriteToCommonBlock.INJECT_NAME, STATIC_VALUE) # Block to check that the correct value is injected into the block BlockWithOptionalField.READ_NAME = WriteToCommonBlock.INJECT_NAME BlockWithOptionalField.READ_VALUE = WriteToCommonBlock.INJECT_VALUE MockFlow.blocks = (WriteToCommonBlock, BlockWithOptionalField) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.validate_blocks(test_flow, successes=2)
def test_happy_flow(self): """Create test flow with success components & validate run success. We test the flow result was success and that all the components run. """ MockFlow.blocks = (SuccessBlock, SuccessBlock) test_flow = MockFlow() self.run_test(test_flow) self.assertTrue(self.result.wasSuccessful(), 'Flow failed when it should have succeeded') self.assertEqual(self.result.testsRun, 1, "Flow didn't run the correct number of blocks") self.validate_blocks(test_flow, successes=2) # === Validate data object === self.assertTrue(test_flow.data.success, 'Flow data result should have been True') self.assertEqual(test_flow.data.exception_type, TestOutcome.SUCCESS, 'Flow data status should have been success')