def test_log_report_on_fixture_one_document(self, mock_db_client,
report_when, report_passed,
report_failed, report_skipped,
expected_test_status,
expected_test_name):
report.when = report_when
report.passed = report_passed
report.failed = report_failed
report.skipped = report_skipped
with mock.patch("modules.mongo_reporter.base_reporter.DBClient",
mock_db_client):
reporter = MongoReporter()
run_document_before = copy.deepcopy(reporter._mongo_run_document)
reporter.log_report(report=report, item=item)
run_document_after = reporter._mongo_run_document
assert run_document_after["test_count"] == run_document_before[
"test_count"]
if expected_test_status == MongoReporter._RESULT_FAIL:
assert run_document_after[
"status"] == MongoReporter._RESULT_FAIL
else:
assert run_document_after["status"] == run_document_before[
"status"]
test_documents = self._get_test_result_documents(reporter)
assert len(test_documents) == 1
assert test_documents[0]["name"] == expected_test_name
if report_skipped:
assert test_documents[0][
"status"] == MongoReporter._RESULT_SKIPPED
else:
assert test_documents[0][
"status"] == MongoReporter._RESULT_FAIL
def test_save_test_run_no_run_id(self):
reporter = MongoReporter()
reporter._run_id = None
with mock.patch.object(reporter._db_client, "insert",
mock.Mock()) as mock_insert:
reporter._save_test_run()
mock_insert.assert_called_once_with(
collection_name=MongoReporter._TEST_RUN_COLLECTION_NAME,
document=reporter._mongo_run_document)
def test_report_components(self):
TEST_COMPONENTS = ["a", "b", "c"]
reporter = MongoReporter()
document_before = copy.deepcopy(reporter._mongo_run_document)
reporter.report_components(TEST_COMPONENTS)
document_after = reporter._mongo_run_document
_assert_all_keys_equal_except(document_before, document_after,
"components")
assert sorted(document_after["components"]) == sorted(TEST_COMPONENTS)
def test_save_test_run_run_id(self):
reporter = MongoReporter(run_id=ObjectId())
with mock.patch.object(reporter._db_client, "replace",
mock.Mock()) as mock_replace:
reporter._save_test_run()
mock_replace.assert_called_once_with(
collection_name=MongoReporter._TEST_RUN_COLLECTION_NAME,
document_id=reporter._run_id,
new_document=reporter._mongo_run_document)
def test_on_fixture_skipped(self):
reporter = MongoReporter(run_id=TEST_RUN_ID)
test_document, status = reporter._on_fixture_skipped(report=report,
item=item,
log=TEST_LOG)
assert status == MongoReporter._RESULT_SKIPPED
assert set(TEST_COMMON_REPORT.items()).issubset(
set(test_document.items()))
assert test_document["name"] == "{}: skipped".format(report.nodeid)
assert test_document["status"] == status
def test_on_fixture_error(self):
reporter = MongoReporter(run_id=TEST_RUN_ID)
test_document, status = reporter._on_fixture_error(report=report,
item=item,
log=TEST_LOG)
assert status == MongoReporter._RESULT_FAIL
assert set(TEST_COMMON_REPORT.items()).issubset(
set(test_document.items()))
assert test_document["status"] == MongoReporter._RESULT_FAIL
assert test_document["name"] == "{}: {} error".format(
report.nodeid, report.when)
def test_report_unavailable_environment(self):
reporter = MongoReporter()
document_before = copy.deepcopy(reporter._mongo_run_document)
reporter.report_unavailable_environment()
document_after = reporter._mongo_run_document
_assert_all_keys_equal_except(document_before, document_after,
"environment_availability", "end_date",
"finished")
assert document_after["environment_availability"] is False
assert_date_close_enough_to_now(document_after["end_date"])
assert document_after["finished"] is True
def test_report_test_type(self):
RUN_TYPE_GET_VALUE = "test-type"
reporter = MongoReporter()
document_before = copy.deepcopy(reporter._mongo_run_document)
with mock.patch("modules.mongo_reporter.reporter.RunType.get",
lambda *args, **kwargs: RUN_TYPE_GET_VALUE):
reporter.report_test_type("xxx")
document_after = reporter._mongo_run_document
_assert_all_keys_equal_except(document_before, document_after,
"test_type")
assert reporter._mongo_run_document["test_type"] == RUN_TYPE_GET_VALUE
def test_on_run_end(self):
TEST_COUNT = 123
reporter = MongoReporter()
document_before = copy.deepcopy(reporter._mongo_run_document)
reporter._total_test_counter = TEST_COUNT
reporter.on_run_end()
document_after = reporter._mongo_run_document
_assert_all_keys_equal_except(document_before, document_after,
"end_date", "total_test_count",
"finished")
assert_date_close_enough_to_now(document_after["start_date"])
assert document_after["total_test_count"] == TEST_COUNT
assert document_after["finished"] is True
def test_update_run_status(self, run_status, test_status,
expected_run_status):
reporter = MongoReporter()
reporter._mongo_run_document["status"] = run_status
run_document_before = copy.deepcopy(reporter._mongo_run_document)
reporter._update_run_status(result_document={},
test_status=test_status)
run_document_after = reporter._mongo_run_document
_assert_all_keys_equal_except(run_document_before, run_document_after,
"status", "result", "test_count")
assert run_document_after["status"] == expected_run_status
assert run_document_after["result"][
test_status] == run_document_before["result"][test_status] + 1
def log_test_run_in_database(request):
if config.database_url is not None:
mongo_reporter = MongoReporter(mongo_uri=config.database_url,
run_id=config.test_run_id)
mongo_reporter.on_run_start(
environment=config.tap_domain,
environment_version=config.appstack_version,
platform_components=[],
tests_to_run_count=len(request.session.items))
def finalizer():
mongo_reporter.on_run_end()
request.addfinalizer(finalizer)
def pytest_runtest_makereport(item, call):
# report for setup, call, teardown
outcome = yield
report = outcome.get_result()
if config.database_url is not None:
mongo_reporter = MongoReporter(mongo_uri=config.database_url,
run_id=config.test_run_id)
mongo_reporter.log_report(report, item)
# support for incremental tests
if INCREMENTAL_KEYWORD in item.keywords:
if call.excinfo is not None and call.excinfo.typename != "Skipped":
parent = item.parent
parent._previous_fail = item
def test_log_report_no_update(self, mock_db_client, report_when):
report.when = report_when
report.passed = True
report.failed = False
report.skipped = False
with mock.patch("modules.mongo_reporter.base_reporter.DBClient",
mock_db_client):
reporter = MongoReporter()
run_document_before = copy.deepcopy(reporter._mongo_run_document)
reporter.log_report(report=report, item=item)
run_document_after = reporter._mongo_run_document
assert run_document_after == run_document_before
test_documents = self._get_test_result_documents(reporter)
assert len(test_documents) == 0
def test_marker_args_from_item_no_args(self, dummy_class):
class Item:
def get_marker(self, *args, **kwargs):
return dummy_class
args = MongoReporter._marker_args_from_item(Item(), "name")
assert args == tuple()
def test_update_run_status_increment_test_count(self,
increment_test_count):
reporter = MongoReporter()
run_document_before = copy.deepcopy(reporter._mongo_run_document)
reporter._update_run_status(result_document={},
test_status=MongoReporter._RESULT_PASS,
increment_test_count=increment_test_count)
run_document_after = reporter._mongo_run_document
_assert_all_keys_equal_except(run_document_before, run_document_after,
"result", "test_count")
if increment_test_count:
assert run_document_after[
"test_count"] == run_document_before["test_count"] + 1
else:
assert run_document_after["test_count"] == run_document_before[
"test_count"]
def test_test_status_from_report(self, dummy_class, report_attr_true,
expected_test_status):
for attr_name in ("passed", "failed", "skipped", "other"):
setattr(dummy_class, attr_name, False)
setattr(dummy_class, report_attr_true, True)
test_status = MongoReporter._test_status_from_report(dummy_class)
assert test_status == expected_test_status
def test_type_from_report_priority(self, priority, expected_test_type,
dummy_class):
with mock.patch(
"modules.mongo_reporter.reporter.MongoReporter._priority_from_report",
lambda *args, **kwargs: priority):
test_type = MongoReporter._get_test_type_from_report(dummy_class)
assert test_type == expected_test_type
def test_test_type_from_report_directory_other(self, dummy_class):
report = dummy_class
report.fspath = "xx"
with mock.patch(
"modules.mongo_reporter.reporter.MongoReporter._priority_from_report",
lambda *args, **kwargs: "kitten"):
test_type = MongoReporter._get_test_type_from_report(report)
assert test_type == MongoReporter._TEST_TYPE_OTHER
def test_stacktrace_from_report(self, dummy_class):
TEST_STACKTRACE = "test_stacktrace"
class Report:
dummy_class.reprtraceback = TEST_STACKTRACE
longrepr = dummy_class
stacktrace = MongoReporter._stacktrace_from_report(Report())
assert stacktrace == TEST_STACKTRACE
def test_marker_args_from_item(self, dummy_class):
TEST_ARGS = ("a", "b", "c")
class Item:
def get_marker(self, *args, **kwargs):
dummy_class.args = TEST_ARGS
return dummy_class
args = MongoReporter._marker_args_from_item(Item(), "name")
assert args == TEST_ARGS
def test_log_report_on_call(self, mock_db_client, report_passed,
report_failed, report_skipped):
report.when = "call"
report.passed = report_passed
report.failed = report_failed
report.skipped = report_skipped
with mock.patch("modules.mongo_reporter.base_reporter.DBClient",
mock_db_client):
reporter = MongoReporter()
run_document_before = copy.deepcopy(reporter._mongo_run_document)
reporter.log_report(report=report, item=item)
run_document_after = reporter._mongo_run_document
assert run_document_after[
"test_count"] == run_document_before["test_count"] + 1
test_documents = self._get_test_result_documents(reporter)
assert len(test_documents) == 1
assert test_documents[0]["name"] == report.nodeid
assert test_documents[0]["status"] == TEST_STATUS
def test_log_report_two_documents(self, mock_db_client):
report.when = "setup"
report.passed = False
report.failed = True
report.skipped = False
with mock.patch("modules.mongo_reporter.base_reporter.DBClient",
mock_db_client):
reporter = MongoReporter()
run_document_before = copy.deepcopy(reporter._mongo_run_document)
reporter.log_report(report=report, item=item)
run_document_after = reporter._mongo_run_document
assert run_document_after["test_count"] == run_document_before[
"test_count"]
test_documents = self._get_test_result_documents(reporter)
assert len(test_documents) == 1
fixture_document = next(
(d for d in test_documents
if d["status"] == MongoReporter._RESULT_FAIL), None)
assert fixture_document is not None
assert fixture_document["name"] == "{}: setup error".format(
report.nodeid)
def test_on_test_not_failed_by_setup(self, failed_by_setup):
reporter = MongoReporter(run_id=TEST_RUN_ID)
reporter._mongo_run_document["test_count"] = TEST_TEST_COUNT
test_document, status = reporter._on_test(
report=report,
item=item,
log=TEST_LOG,
failed_by_setup=failed_by_setup)
assert status == TEST_STATUS
assert set(TEST_COMMON_REPORT.items()).issubset(
set(test_document.items()))
assert test_document["status"] == TEST_STATUS
if not failed_by_setup:
assert test_document["duration"] == report.duration
assert test_document["name"] == report.nodeid
assert test_document["order"] == TEST_TEST_COUNT
else:
assert test_document["duration"] == 0.0
assert test_document["name"] == "{}: failed on setup".format(
report.nodeid)
assert "order" not in test_document
def test_init_run_document(self):
TEST_VERSION = "test.version"
with mock.patch(
"modules.mongo_reporter.reporter.MongoReporter._get_test_version",
lambda *args, **kwargs: TEST_VERSION):
run_document = MongoReporter()._mongo_run_document
# values from config
assert run_document["appstack_version"] == MockConfig.appstack_version
assert run_document["environment"] == MockConfig.tap_domain
assert run_document["environment_version"] == MockConfig.tap_version
assert run_document[
"infrastructure_type"] == MockConfig.tap_infrastructure_type
assert run_document["kerberos"] == MockConfig.kerberos
assert run_document["stress_run_id"] == MockConfig.stress_run_id
# values from TeamCityConfiguration
assert run_document["parameters"][
"configuration_parameters"] == MockTeamcityConfiguration.GET_ALL_VALUE
assert run_document[
"teamcity_build_id"] == MockTeamcityConfiguration.GETINT_VALUE
assert run_document[
"teamcity_server_url"] == MockTeamcityConfiguration.GET_VALUE
# other dynamic values
assert_date_close_enough_to_now(run_document["start_date"])
assert run_document["started_by"] == socket.gethostname()
assert run_document["test_version"] == TEST_VERSION
assert run_document["parameters"][
"environment_variables"] == os.environ
assert run_document["tap_build_number"] is None
# non-dynamic values
assert_date_close_enough_to_now(run_document["start_date"])
assert run_document["end_date"] is None
assert run_document["finished"] is False
assert run_document["log"] == ""
assert run_document["platform_components"] == []
assert run_document["result"] == {
MongoReporter._RESULT_PASS: 0,
MongoReporter._RESULT_FAIL: 0,
MongoReporter._RESULT_SKIPPED: 0,
MongoReporter._RESULT_UNKNOWN: 0
}
assert run_document["status"] == MongoReporter._RESULT_PASS
assert run_document["test_count"] == 0
assert run_document["total_test_count"] == 0
assert run_document["components"] == []
assert run_document["environment_availability"] is True
assert run_document["test_type"] is None
from modules.tap_object_model import ServiceOffering
pytest_plugins = ["tests.fixtures.context",
"tests.fixtures.fixtures",
"tests.fixtures.fixtures_core_components",
"tests.fixtures.remote_logging",
"tests.fixtures.fixtures_ng",
"modules.mongo_reporter.request_reporter"]
logger = get_logger(__name__)
INCREMENTAL_KEYWORD = "incremental"
mongo_reporter = MongoReporter(run_id=config.test_run_id)
def _log_test_configuration():
logger.info("============== configuration variables ==============")
pt_env_vars = []
for key, value in os.environ.items():
if key.startswith("PT_"):
if "password" in key.lower():
value = "*******"
pt_env_vars.append("{}={}".format(key, value))
pt_env_vars.sort()
for env_var in pt_env_vars:
logger.info(env_var)
def test_get_test_version_bad_file(self):
with pytest.raises(AssertionError) as e:
MongoReporter._get_test_version()
assert "Version not found" in e.value.msg
def test_priority_from_report(self, dummy_class, keywords,
expected_priority):
dummy_class.keywords = keywords
priority = MongoReporter._priority_from_report(dummy_class)
assert priority == expected_priority
def test_stacktrace_from_report_no_stacktrace(self, dummy_class):
class Report:
longrepr = dummy_class
stacktrace = MongoReporter._stacktrace_from_report(Report())
assert stacktrace is None
def test_test_type_from_report_directory_smoke(self, dummy_class):
report = dummy_class
report.fspath = Path.test_directories["test_smoke"]
test_type = MongoReporter._get_test_type_from_report(report)
assert test_type == MongoReporter._TEST_TYPE_SMOKE
def test_collection_name_is_not_none(self):
reporter = MongoReporter()
assert reporter._TEST_RUN_COLLECTION_NAME is not None