def test_pytest_parallel_threads(mock_client_init):
"""Verify "pytest_parallel" plugin run tests in two threads.
:param mock_client_init: Pytest fixture
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = item_id_gen
result = utils.run_pytest_tests(tests=['examples/hierarchy'],
args=['--tests-per-worker', '2'])
assert int(result) == 0, 'Exit code should be 0 (no errors)'
mock_client = mock_client_init.return_value
expect(mock_client.start_launch.call_count == 1,
'"start_launch" method was not called')
expect(mock_client.finish_launch.call_count == 1,
'"finish_launch" method was not called')
assert_expectations()
finish_args = mock_client.finish_launch.call_args_list
expect(finish_args[0][1]['status'] in ('PASSED', None), 'Launch failed')
launch_end_time = finish_args[0][1]['end_time']
expect(launch_end_time is not None and int(launch_end_time) > 0,
'Launch end time is empty')
assert_expectations()
def test_issue_report(mock_client_init):
"""Verify agent reports issue ids and defect type.
:param mock_client_init: Pytest fixture
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = utils.item_id_gen
mock_client.get_project_settings.side_effect = utils.project_settings
variables = dict()
variables['rp_issue_system_url'] = ISSUE_URL_PATTERN
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(tests=['examples/test_issue_id.py'],
variables=variables)
assert int(result) == 1, 'Exit code should be 1 (test failed)'
call_args = mock_client.finish_test_item.call_args_list
finish_test_step = call_args[0][1]
issue = finish_test_step['issue']
assert isinstance(issue, Issue)
expect(issue.issue_type == 'pb001')
expect(issue.comment is not None)
assert_expectations()
comments = issue.comment.split('\n')
assert len(comments) == 1
comment = comments[0]
assert comment == "* {}: [{}]({})" \
.format(test_issue_id.REASON, test_issue_id.ID,
ISSUE_URL_PATTERN.replace(ISSUE_PLACEHOLDER, test_issue_id.ID))
def test_external_issue(mock_client_init):
"""Verify skipped test with issue decorator handling.
:param mock_client_init: mocked Report Portal client Pytest fixture
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = utils.item_id_gen
mock_client.get_project_settings.side_effect = utils.project_settings
variables = dict()
variables['rp_bts_project'] = BTS_PROJECT
variables['rp_bts_url'] = BTS_URL
variables['rp_issue_system_url'] = ISSUE_URL_PATTERN
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(tests=['examples/test_issue_id.py'],
variables=variables)
assert int(result) == 1, 'Exit code should be 1 (test failed)'
call_args = mock_client.finish_test_item.call_args_list
finish_test_step = call_args[0][1]
actual_issue = finish_test_step.get('issue', None)
assert isinstance(actual_issue, Issue)
expect(actual_issue.issue_type == 'pb001')
expect(actual_issue.comment is not None)
external_issues = actual_issue._external_issues
expect(len(external_issues) == 1)
assert_expectations()
external_issue = external_issues[0]
expect(external_issue['btsUrl'] == BTS_URL)
expect(external_issue['btsProject'] == BTS_PROJECT)
expect(external_issue['ticketId'] == test_issue_id.ID)
expect(external_issue['url'] == ISSUE_URL_PATTERN.replace(
ISSUE_PLACEHOLDER, test_issue_id.ID))
assert_expectations()
def test_issue_id_attribute(mock_client_init, issue_id_mark):
"""Verify agent reports issue attribute if configured.
:param mock_client_init: Pytest fixture
:param issue_id_mark: Attribute report configuration
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = utils.item_id_gen
mock_client.get_project_settings.side_effect = utils.project_settings
variables = dict()
variables['rp_issue_id_marks'] = issue_id_mark
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(tests=['examples/test_issue_id.py'],
variables=variables)
assert int(result) == 1, 'Exit code should be 1 (test failed)'
call_args = mock_client.start_test_item.call_args_list
finish_test_step = call_args[-1][1]
attributes = finish_test_step['attributes']
if issue_id_mark:
assert len(attributes) == 1
issue_attribute = attributes[0]
expect(issue_attribute['key'] == 'issue')
expect(issue_attribute['value'] == test_issue_id.ID)
assert_expectations()
else:
assert len(attributes) == 0
def test_rp_parent_item_id_and_rp_launch_id(mock_client_init):
"""Verify RP handles both conf props 'rp_parent_item_id' & 'rp_launch_id'.
:param mock_client_init: Pytest fixture
"""
parent_id = "parent_id"
variables = dict()
variables['rp_parent_item_id'] = parent_id
variables['rp_launch_id'] = "test_launch_id"
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(tests=['examples/test_simple.py'],
variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no errors)'
mock_client = mock_client_init.return_value
expect(mock_client.start_launch.call_count == 0,
'"start_launch" method was called')
expect(mock_client.finish_launch.call_count == 0,
'"finish_launch" method was called')
start_call_args = mock_client.start_test_item.call_args_list
finish_call_args = mock_client.finish_test_item.call_args_list
expect(len(start_call_args) == len(finish_call_args))
expect(start_call_args[0][1]["parent_item_id"] == parent_id)
assert_expectations()
def test_skipped_custom_issue(mock_client_init):
"""Verify skipped test with issue decorator handling.
:param mock_client_init: mocked Report Portal client Pytest fixture
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = utils.item_id_gen
mock_client.get_project_settings.side_effect = utils.project_settings
variables = dict()
variables['rp_is_skipped_an_issue'] = True
variables['rp_issue_system_url'] = ISSUE_URL_PATTERN
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(tests=['examples/skip/test_skip_issue.py'],
variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no failures)'
call_args = mock_client.finish_test_item.call_args_list
finish_test_step = call_args[0][1]
actual_issue = finish_test_step.get('issue', None)
assert isinstance(actual_issue, Issue)
expect(actual_issue.issue_type == 'pb001')
expect(actual_issue.comment is not None)
assert_expectations()
def test_rp_launch_id(mock_client_init):
"""Verify that RP plugin does not start/stop launch if 'rp_launch_id' set.
:param mock_client_init: Pytest fixture
"""
variables = dict()
variables['rp_launch_id'] = TEST_LAUNCH_ID
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(tests=['examples/test_simple.py'],
variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no errors)'
expect(
mock_client_init.call_args_list[0][1]['launch_id'] == TEST_LAUNCH_ID)
mock_client = mock_client_init.return_value
expect(mock_client.start_launch.call_count == 0,
'"start_launch" method was called')
expect(mock_client.finish_launch.call_count == 0,
'"finish_launch" method was called')
start_call_args = mock_client.start_test_item.call_args_list
finish_call_args = mock_client.finish_test_item.call_args_list
expect(len(start_call_args) == len(finish_call_args))
assert_expectations()
def test_rp_hierarchy_parameters(mock_client_init, test, variables,
expected_items):
"""Verify suite hierarchy with `rp_hierarchy_dirs=True`.
:param mock_client_init: Pytest fixture
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = utils.item_id_gen
result = utils.run_pytest_tests(tests=test, variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no errors)'
verify_start_item_parameters(mock_client, expected_items)
def test_rp_log_batch_payload_size(mock_client_init):
log_size = 123456
variables = {'rp_log_batch_payload_size': log_size}
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(['examples/test_rp_logging.py'],
variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no errors)'
expect(mock_client_init.call_count == 1)
constructor_args = mock_client_init.call_args_list[0][1]
expect(constructor_args['log_batch_payload_size'] == log_size)
assert_expectations()
def test_parameters(mock_client_init, test, expected_params):
"""Verify different tests have correct parameters.
:param mock_client_init: Pytest fixture
:param test: a test to run
:param expected_params: an expected parameter dictionary
"""
variables = utils.DEFAULT_VARIABLES
result = utils.run_pytest_tests(tests=[test], variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no errors)'
mock_client = mock_client_init.return_value
assert mock_client.start_test_item.call_count > 0, \
'"start_test_item" called incorrect number of times'
call_args = mock_client.start_test_item.call_args_list
step_call_args = call_args[-1][1]
assert step_call_args['parameters'] == expected_params
def test_passed_no_issue_report(mock_client_init):
"""Verify agent do not report issue if test passed.
:param mock_client_init: Pytest fixture
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = utils.item_id_gen
mock_client.get_project_settings.side_effect = utils.project_settings
variables = dict()
variables['rp_issue_system_url'] = ISSUE_URL_PATTERN
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(tests=['examples/test_issue_id_pass.py'],
variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no failures)'
call_args = mock_client.finish_test_item.call_args_list
finish_test_step = call_args[0][1]
assert 'issue' not in finish_test_step or finish_test_step['issue'] is None
def test_simple_tests(mock_client_init, test, expected_run_status,
expected_item_status):
"""Verify a simple test creates correct structure and finishes all items.
Report 'None' for suites and launch due to possible parallel execution.
Leave status calculation on Server.
:param mock_client_init: mocked Report Portal client Pytest fixture
:param test: a test to run as use case
:param expected_run_status: expected pytest run status
:param expected_item_status: expected result test item status
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = utils.item_id_gen
result = utils.run_pytest_tests(tests=[test])
assert int(result) == expected_run_status, 'Exit code should be ' + str(
expected_run_status)
start_call_args = mock_client.start_test_item.call_args_list
finish_call_args = mock_client.finish_test_item.call_args_list
assert len(start_call_args) == len(finish_call_args), \
'Number of started items should be equal to finished items'
for i in range(len(start_call_args)):
start_test_step = start_call_args[-1 - i][1]
finish_test_step = finish_call_args[i][1]
expect(finish_test_step['item_id'].startswith(start_test_step['name']))
if i == 0:
actual_status = finish_test_step['status']
expect(
actual_status == expected_item_status,
'Invalid item status, actual "{}", expected: "{}"'.format(
actual_status, expected_item_status))
finish_launch_call_args = mock_client.finish_launch.call_args_list
expect(len(finish_launch_call_args) == 1)
expect('end_time' in finish_launch_call_args[0][1])
expect(finish_launch_call_args[0][1]['end_time'] is not None)
expect('status' not in finish_launch_call_args[0][1])
assert_expectations()
def test_skipped_not_issue(mock_client_init, flag_value, expected_issue):
"""Verify 'rp_is_skipped_an_issue' option handling.
:param mock_client_init: mocked Report Portal client Pytest fixture
:param flag_value: option value to set during the test
:param expected_issue: result issue value to verify
"""
mock_client = mock_client_init.return_value
mock_client.start_test_item.side_effect = utils.item_id_gen
variables = dict()
if flag_value is not None:
variables['rp_is_skipped_an_issue'] = flag_value
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(
tests=['examples/skip/test_simple_skip.py'], variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no failures)'
call_args = mock_client.finish_test_item.call_args_list
finish_test_step = call_args[0][1]
actual_issue = finish_test_step.get('issue', None)
assert actual_issue == expected_issue
def test_launch_mode(mock_client_init, mode, expected_mode):
"""Verify different launch modes are passed to `start_launch` method.
:param mock_client_init: Pytest fixture
:param mode: a variable to be passed to pytest
:param expected_mode: a value which should be passed to
ReportPortalService
"""
variables = dict()
if mode is not None:
variables['rp_mode'] = mode
variables.update(utils.DEFAULT_VARIABLES.items())
result = utils.run_pytest_tests(tests=['examples/test_simple.py'],
variables=variables)
assert int(result) == 0, 'Exit code should be 0 (no errors)'
mock_client = mock_client_init.return_value
assert mock_client.start_launch.call_count == 1, \
'"start_launch" method was not called'
call_args = mock_client.start_launch.call_args_list
start_launch_kwargs = call_args[0][1]
assert start_launch_kwargs['mode'] == expected_mode
def test_empty_run(mock_client_init):
"""Verify that RP plugin does not fail if there is not tests in run.
:param mock_client_init: Pytest fixture
"""
result = utils.run_pytest_tests(tests=['examples/epmty/'])
assert int(result) == 4, 'Exit code should be 4 (no tests)'
mock_client = mock_client_init.return_value
expect(mock_client.start_launch.call_count == 1,
'"start_launch" method was not called')
expect(mock_client.finish_launch.call_count == 1,
'"finish_launch" method was not called')
assert_expectations()
finish_args = mock_client.finish_launch.call_args_list
expect('status' not in finish_args[0][1],
'Launch status should not be defined')
launch_end_time = finish_args[0][1]['end_time']
expect(launch_end_time is not None and int(launch_end_time) > 0,
'Launch end time is empty')
assert_expectations()
