def test_finish(self, mock_verification_update):
v = objects.Verification(self.db_obj)
totals = {
"tests_count": 2,
"tests_duration": 0.54,
"success": 2,
"skip": 0,
"expected_failures": 0,
"unexpected_success": 0,
"failures": 0
}
tests = {
"foo_test[gate,negative]": {
"name": "foo_test",
"duration": 0.25,
"status": "success",
"tags": ["gate", "negative"]
},
"bar_test[gate,negative]": {
"name": "bar_test",
"duration": 0.29,
"status": "success",
"tags": ["gate", "negative"]
}
}
v.finish(totals, tests)
mock_verification_update.assert_called_once_with(
self.db_obj["uuid"],
status=consts.VerificationStatus.FINISHED,
tests=tests,
**totals)
v = objects.Verification(self.db_obj)
totals.update(failures=1)
mock_verification_update.reset_mock()
v.finish(totals, tests)
mock_verification_update.assert_called_once_with(
self.db_obj["uuid"],
status=consts.VerificationStatus.FAILED,
tests=tests,
**totals)
v = objects.Verification(self.db_obj)
totals.update(failures=0, unexpected_success=1)
mock_verification_update.reset_mock()
v.finish(totals, tests)
mock_verification_update.assert_called_once_with(
self.db_obj["uuid"],
status=consts.VerificationStatus.FAILED,
tests=tests,
**totals)
def test_show(self, mock_objects_verification,
mock_verification_result_get, mock_verification_get,
mock_print_list):
class Test_dummy():
data = {"test_cases": {"test_a": {"name": "test_a", "time": 20,
"status": "PASS"},
"test_b": {"name": "test_b", "time": 20,
"status": "SKIP"},
"test_c": {"name": "test_c", "time": 20,
"status": "FAIL"}}}
verification_id = "39121186-b9a4-421d-b094-6c6b270cf9e9"
total_fields = ["UUID", "Deployment UUID", "Set name", "Tests",
"Failures", "Created at", "Status"]
fields = ["name", "time", "status"]
verification = mock.MagicMock()
tests = Test_dummy()
mock_verification_result_get.return_value = tests
mock_verification_get.return_value = verification
mock_objects_verification.return_value = 1
values = [objects.Verification(t)
for t in six.itervalues(tests.data["test_cases"])]
self.verify.show(verification_id)
self.assertEqual([mock.call([verification], fields=total_fields),
mock.call(values, fields, sortby_index=0)],
mock_print_list.call_args_list)
mock_verification_get.assert_called_once_with(verification_id)
mock_verification_result_get.assert_called_once_with(verification_id)
def verify(cls,
deployment,
set_name,
regex,
tempest_config,
system_wide_install=False):
"""Start verifying.
:param deployment: UUID or name of a deployment.
:param set_name: Valid name of tempest test set.
:param regex: Regular expression of test
:param tempest_config: User specified Tempest config file
:param system_wide_install: Use virtualenv else run tests in local
environment
:returns: Verification object
"""
deployment_uuid = objects.Deployment.get(deployment)["uuid"]
verification = objects.Verification(deployment_uuid=deployment_uuid)
verifier = cls._create_verifier(deployment_uuid, verification,
tempest_config, system_wide_install)
LOG.info("Starting verification of deployment: %s" % deployment_uuid)
verification.set_running()
verifier.verify(set_name=set_name, regex=regex)
return verification
def verify(cls, deployment, set_name, regex, tempest_config):
"""Start verifying.
:param deployment: UUID or name of a deployment.
:param set_name: Valid name of tempest test set.
:param regex: Regular expression of test
:param tempest_config: User specified Tempest config file
:returns: Verification object
"""
deployment_uuid = objects.Deployment.get(deployment)["uuid"]
verification = objects.Verification(deployment_uuid=deployment_uuid)
verifier = tempest.Tempest(deployment_uuid,
verification=verification,
tempest_config=tempest_config)
if not verifier.is_installed():
print("Tempest is not installed for specified deployment.")
print("Installing Tempest for deployment %s" % deployment_uuid)
verifier.install()
LOG.info("Starting verification of deployment: %s" % deployment_uuid)
verification.set_running()
verifier.verify(set_name=set_name, regex=regex)
return verification
def test_to_dict(self):
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
data = {
"created_at": dt.date(2017, 2, 3),
"updated_at": dt.date(2017, 3, 3),
"id": "v_id",
"deployment_uuid": "d_uuid",
"uuid": "v_uuid",
"verifier_uuid": "v_uuid",
"unexpected_success": "2",
"status": "False",
"tests": {
"test1": "tdata1",
"test2": "tdata2"
},
"skipped": 2,
"tests_duration": "",
"tags": None,
"run_args": "args",
"success": 0,
"expected_failures": 2,
"tests_count": 3,
"failures": 2
}
verification = objects.Verification("verification_id")
verification._db_entry = data
result = objects.Verification.to_dict(verification)
data["created_at"] = data["created_at"].strftime(TIME_FORMAT)
data["updated_at"] = data["updated_at"].strftime(TIME_FORMAT)
self.assertEqual(data, result)
def test_init_without_create(self, mock_verification_create):
verification = objects.Verification(db_object=self.db_obj)
self.assertEqual(0, mock_verification_create.call_count)
self.assertEqual(self.db_obj["failures"], verification.failures)
self.assertEqual(self.db_obj["tests"], verification.tests)
self.assertEqual(self.db_obj["errors"], verification.errors)
self.assertEqual(self.db_obj["time"], verification.time)
def verify(cls,
deployment,
set_name="",
regex=None,
tests_file=None,
tests_file_to_skip=None,
tempest_config=None,
expected_failures=None,
system_wide=False,
concur=0,
failing=False):
"""Start verification.
:param deployment: UUID or name of a deployment
:param set_name: Name of a Tempest test set
:param regex: Regular expression of test
:param tests_file: Path to a file with a list of Tempest tests
to run them
:param tests_file_to_skip: Path to a file with a list of Tempest tests
to skip them
:param tempest_config: User specified Tempest config file location
:param expected_failures: Dictionary with Tempest tests that are
expected to fail. Keys are test names;
values are reasons of test failures
:param system_wide: Whether or not to create a virtual env when
installing Tempest; whether or not to use
the local env instead of the Tempest virtual
env when running the tests
:param concur: How many processes to use to run Tempest tests.
The default value (0) auto-detects CPU count
:param failing: Re-run tests that failed during the last execution
:returns: Verification object
"""
deployment_uuid = objects.Deployment.get(deployment)["uuid"]
verification = objects.Verification(deployment_uuid=deployment_uuid)
verifier = tempest.Tempest(deployment_uuid,
verification=verification,
tempest_config=tempest_config,
system_wide=system_wide)
cls._check_tempest_tree_existence(verifier)
LOG.info("Starting verification of deployment: %s" % deployment_uuid)
verification.set_running()
verifier.verify(set_name=set_name,
regex=regex,
tests_file=tests_file,
tests_file_to_skip=tests_file_to_skip,
expected_failures=expected_failures,
concur=concur,
failing=failing)
return verification
def verify(cls,
deployment,
set_name="",
regex=None,
tests_file=None,
tempest_config=None,
expected_failures=None,
system_wide=False,
concur=0):
"""Start verification.
:param deployment: UUID or name of a deployment
:param set_name: Name of a Tempest test set
:param regex: Regular expression of test
:param tests_file: Path to a file with a list of Tempest tests
:param tempest_config: User specified Tempest config file location
:param expected_failures: Dictionary with Tempest tests that are
expected to fail. Keys are test names;
values are reasons of test failures
:param system_wide: Whether or not to create a virtual env when
installing Tempest; whether or not to use
the local env instead of the Tempest virtual
env when running the tests
:param concur: How many processes to use to run Tempest tests.
The default value (0) auto-detects CPU count
:returns: Verification object
"""
deployment_uuid = objects.Deployment.get(deployment)["uuid"]
verification = objects.Verification(deployment_uuid=deployment_uuid)
verifier = tempest.Tempest(deployment_uuid,
verification=verification,
tempest_config=tempest_config,
system_wide=system_wide)
if not verifier.is_installed():
LOG.info(
_("Tempest is not installed "
"for the specified deployment."))
LOG.info(
_("Installing Tempest "
"for deployment: %s") % deployment_uuid)
verifier.install()
LOG.info("Starting verification of deployment: %s" % deployment_uuid)
verification.set_running()
verifier.verify(set_name=set_name,
regex=regex,
tests_file=tests_file,
expected_failures=expected_failures,
concur=concur)
return verification
def show(self, verification_uuid=None, sort_by="name", detailed=False):
"""Display results table of the verification."""
try:
sortby_index = ("name", "duration").index(sort_by)
except ValueError:
print("Sorry, but verification results can't be sorted "
"by '%s'." % sort_by)
return 1
try:
verification = db.verification_get(verification_uuid)
tests = db.verification_result_get(verification_uuid)
except exceptions.NotFoundException as e:
print(six.text_type(e))
return 1
print("Total results of verification:\n")
total_fields = [
"UUID", "Deployment UUID", "Set name", "Tests", "Failures",
"Created at", "Status"
]
cliutils.print_list([verification], fields=total_fields)
print("\nTests:\n")
fields = ["name", "time", "status"]
values = [
objects.Verification(test)
for test in six.itervalues(tests.data["test_cases"])
]
cliutils.print_list(values, fields, sortby_index=sortby_index)
if detailed:
for test in six.itervalues(tests.data["test_cases"]):
if test["status"] == "FAIL":
header = cliutils.make_header(
"FAIL: %(name)s\n"
"Time: %(time)s\n"
"Type: %(type)s" % {
"name": test["name"],
"time": test["time"],
"type": test["failure"]["type"]
})
formatted_test = "%(header)s%(log)s\n" % {
"header": header,
"log": test["failure"]["log"]
}
print(formatted_test)
def test_finish_verification(self, mock_verification_update,
mock_verification_result_create):
verification = objects.Verification(db_object=self.db_obj)
fake_results = fakes.get_fake_test_case()
verification.finish_verification(fake_results["total"],
fake_results["test_cases"])
expected_values = {"status": "finished"}
expected_values.update(fake_results["total"])
mock_verification_update.assert_called_with(self.db_obj["uuid"],
expected_values)
expected_data = fake_results["total"].copy()
expected_data["test_cases"] = fake_results["test_cases"]
mock_verification_result_create.assert_called_once_with(
verification.uuid, expected_data)
def import_results(cls, deployment, set_name="", log_file=None):
"""Import Tempest tests results into the Rally database.
:param deployment: UUID or name of a deployment
:param log_file: User specified Tempest log file in subunit format
:returns: Deployment and verification objects
"""
# TODO(aplanas): Create an external deployment if this is
# missing, as required in the blueprint [1].
# [1] https://blueprints.launchpad.net/rally/+spec/verification-import
deployment_uuid = objects.Deployment.get(deployment)["uuid"]
verification = objects.Verification(deployment_uuid=deployment_uuid)
verifier = tempest.Tempest(deployment_uuid, verification=verification)
LOG.info("Importing verification of deployment: %s" % deployment_uuid)
verification.set_running()
verifier.import_results(set_name=set_name, log_file=log_file)
return deployment, verification
def test_show(self, mock_verification, mock_print_list):
verification = mock_verification.get.return_value
tests = {
"test_cases": {
"test_a": {
"name": "test_a",
"time": 20,
"status": "success"
},
"test_b": {
"name": "test_b",
"time": 20,
"status": "skip"
},
"test_c": {
"name": "test_c",
"time": 20,
"status": "fail"
}
}
}
verification_id = "39121186-b9a4-421d-b094-6c6b270cf9e9"
total_fields = [
"UUID", "Deployment UUID", "Set name", "Tests", "Failures",
"Created at", "Status"
]
fields = ["name", "time", "status"]
verification.get_results.return_value = tests
values = [
objects.Verification(t)
for t in six.itervalues(tests["test_cases"])
]
self.verify.show(verification_id)
self.assertEqual([
mock.call([verification], fields=total_fields),
mock.call(values, fields, sortby_index=0)
], mock_print_list.call_args_list)
mock_verification.get.assert_called_once_with(verification_id)
verification.get_results.assert_called_once_with()
def test_create_and_delete(self, mock_verification_create,
mock_verification_delete):
verification = objects.Verification(db_object=self.db_obj)
verification.delete()
mock_verification_delete.assert_called_once_with(self.db_obj["uuid"])
def test_update_status(self, mock_verification_update):
v = objects.Verification(self.db_obj)
v.update_status(status="some-status")
mock_verification_update.assert_called_once_with(self.db_obj["uuid"],
status="some-status")
def test_delete(self, mock_verification_delete):
objects.Verification(self.db_obj).delete()
mock_verification_delete.assert_called_once_with(self.db_obj["uuid"])
def test_init(self, mock_verification_create):
v = objects.Verification(self.db_obj)
self.assertEqual(0, mock_verification_create.call_count)
self.assertEqual(self.db_obj["uuid"], v.uuid)
self.assertEqual(self.db_obj["uuid"], v["uuid"])
def test_set_error(self, mock_verification_update):
v = objects.Verification(self.db_obj)
v.set_error("Some error")
mock_verification_update.assert_called_once_with(
self.db_obj["uuid"], status=consts.VerificationStatus.CRASHED)
def test_set_failed(self, mock_verification_update):
mock_verification_update.return_value = self.db_obj
verification = objects.Verification(db_object=self.db_obj)
verification.set_failed()
mock_verification_update.assert_called_once_with(
self.db_obj["uuid"], {"status": "failed"})
def test_init_with_create(self, mock_verification_create):
objects.Verification(deployment_uuid="some_deployment_uuid")
mock_verification_create.assert_called_once_with(
"some_deployment_uuid")