def __init__(self, iut, etos):
"""Initialize.
:param iut: IUT to execute tests on.
:type iut: :obj:`etr.lib.iut.Iut`
:param etos: ETOS library
:type etos: :obj:`etos_lib.etos.ETOS`
"""
self.etos = etos
self.iut = iut
self.config = self.etos.config.get("test_config")
self.log_area = LogArea(self.etos)
self.iut_monitoring = IutMonitoring(self.iut)
self.issuer = {"name": "ETOS Test Runner"}
def test_start_monitoring_single_script(self):
"""Verify that the start monitoring method can execute a single script.
Approval criteria:
- Start monitoring shall execute a script to completion.
Test steps::
1. Initialize IUT monitoring.
2. Load a script to the config.
3. Start monitoring.
4. Verify that the script executed.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info("STEP: Load script to the config.")
self.config.set("scripts", [{
"name": str(self.script),
"parameters": ["world"]
}])
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
self.logger.info("STEP: Verify that the script executed.")
self._wait_for_file(Path.cwd().joinpath("output"))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
hello_world = output.read().strip()
self.logger.info(hello_world)
self.assertEqual(hello_world, "Hello world")
def test_start_monitoring_multiple_scripts(self):
"""Verify that the start monitoring method can execute multiple scripts.
Approval criteria:
- Start monitoring shall execute multiple scripts to completion.
Test steps::
1. Initialize IUT monitoring.
2. Load the scripts to the config.
3. Start monitoring.
4. Verify that the scripts executed.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info("STEP: Load the scripts to the config.")
script = ["#!/bin/bash", "echo Goodbye $1 > $(pwd)/output2"]
second_script = Path.cwd().joinpath("second.sh")
self.files.append(second_script)
self.files.append(Path.cwd().joinpath("output2"))
with open(second_script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config.set(
"scripts",
[
{
"name": str(self.script),
"parameters": ["world"]
},
{
"name": str(second_script),
"parameters": ["world"]
},
],
)
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
self.logger.info("STEP: Verify that the scripts executed.")
self._wait_for_file(Path.cwd().joinpath("output"))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
hello_world = output.read().strip()
self.logger.info(hello_world)
self._wait_for_file(Path.cwd().joinpath("output2"))
with open(Path.cwd().joinpath("output2"), encoding="utf-8") as output:
goodbye_world = output.read().strip()
self.logger.info(goodbye_world)
self.assertEqual(hello_world, "Hello world")
self.assertEqual(goodbye_world, "Goodbye world")
def __init__(self, iut, etos):
"""Initialize.
:param iut: IUT to execute tests on.
:type iut: :obj:`etr.lib.iut.Iut`
:param etos: ETOS library
:type etos: :obj:`etos_lib.etos.ETOS`
"""
self.etos = etos
self.iut = iut
self.config = self.etos.config.get("test_config")
self.iut_monitoring = IutMonitoring(self.iut)
self.issuer = {"name": "ETOS Test Runner"}
self.env = os.environ
if not os.path.isdir(os.getenv("TEST_ARTIFACT_PATH")):
os.makedirs(os.getenv("TEST_ARTIFACT_PATH"))
if not os.path.isdir(os.getenv("TEST_LOCAL_PATH")):
os.makedirs(os.getenv("TEST_LOCAL_PATH"))
if not os.path.isdir(os.getenv("GLOBAL_ARTIFACT_PATH")):
os.makedirs(os.getenv("GLOBAL_ARTIFACT_PATH"))
class TestRunner:
"""Test runner for ETOS."""
logger = logging.getLogger("ETR")
def __init__(self, iut, etos):
"""Initialize.
:param iut: IUT to execute tests on.
:type iut: :obj:`etr.lib.iut.Iut`
:param etos: ETOS library
:type etos: :obj:`etos_lib.etos.ETOS`
"""
self.etos = etos
self.iut = iut
self.config = self.etos.config.get("test_config")
self.log_area = LogArea(self.etos)
self.iut_monitoring = IutMonitoring(self.iut)
self.issuer = {"name": "ETOS Test Runner"}
def test_suite_started(self):
"""Publish a test suite started event.
:return: Reference to test suite started.
:rtype: :obj:`eiffel.events.base_event.BaseEvent`
"""
suite_name = self.config.get("name")
categories = ["Regression test_suite", "Sub suite"]
categories.append(self.iut.identity.name)
livelogs = self.config.get("log_area", {}).get("livelogs")
return self.etos.events.send_test_suite_started(
suite_name,
links={"CONTEXT": self.etos.config.get("context")},
categories=categories,
types=["FUNCTIONAL"],
liveLogs=[{"name": "console", "uri": livelogs}],
)
def main_suite_id(self, activity_id):
"""Get the eiffel event id of the mainsuite linked form the activity triggered event.
:param activity_id: Id of the activity linked to the main suite
:type activity_id: str
:return: Id of the main suite linked from the activity triggered event
:rtype: str
"""
for test_suite_started in request_test_suite_started(self.etos, activity_id):
if not "Sub suite" in test_suite_started["data"]["testSuiteCategories"]:
return test_suite_started["meta"]["id"]
raise Exception("Missing main suite events, exiting!")
def environment(self, context):
"""Send out which environment we're executing within.
:param context: Context where this environment is used.
:type context: str
"""
# TODO: Get this from prepare
if os.getenv("HOSTNAME") is not None:
self.etos.events.send_environment_defined(
"ETR Hostname",
links={"CONTEXT": context},
host={"name": os.getenv("HOSTNAME"), "user": "******"},
)
if os.getenv("EXECUTION_SPACE_URL") is not None:
self.etos.events.send_environment_defined(
"Execution Space URL",
links={"CONTEXT": context},
host={"name": os.getenv("EXECUTION_SPACE_URL"), "user": "******"},
)
def run_tests(self, workspace):
"""Execute test recipes within a test executor.
:param workspace: Which workspace to execute test suite within.
:type workspace: :obj:`etr.lib.workspace.Workspace`
:return: Result of test execution.
:rtype: bool
"""
recipes = self.config.get("recipes")
result = True
for num, test in enumerate(recipes):
self.logger.info("Executing test %s/%s", num + 1, len(recipes))
with Executor(test, self.iut, self.etos) as executor:
self.logger.info("Starting test '%s'", executor.test_name)
executor.execute(workspace)
if not executor.result:
result = executor.result
self.logger.info("Test finished. Result: %s.", executor.result)
return result
def outcome(self, result, executed, description):
"""Get outcome from test execution.
:param result: Result of execution.
:type result: bool
:param executed: Whether or not tests have successfully executed.
:type executed: bool
:param description: Optional description.
:type description: str
:return: Outcome of test execution.
:rtype: dict
"""
if executed:
conclusion = "SUCCESSFUL"
verdict = "PASSED" if result else "FAILED"
self.logger.info(
"Tests executed successfully. Verdict set to '%s' due to result being '%s'",
verdict,
result,
)
else:
conclusion = "FAILED"
verdict = "INCONCLUSIVE"
self.logger.info(
"Tests did not execute successfully. Setting verdict to '%s'",
verdict,
)
suite_name = self.config.get("name")
if not description and not result:
self.logger.info("No description but result is a failure. At least some tests failed.")
description = f"At least some {suite_name} tests failed."
elif not description and result:
self.logger.info(
"No description and result is a success. " "All tests executed successfully."
)
description = f"All {suite_name} tests completed successfully."
else:
self.logger.info("Description was set. Probably due to an exception.")
return {
"verdict": verdict,
"description": description,
"conclusion": conclusion,
}
def execute(self): # pylint:disable=too-many-branches,disable=too-many-statements
"""Execute all tests in test suite.
:return: Result of execution. Linux exit code.
:rtype: int
"""
self.logger.info("Send test suite started event.")
test_suite_started = self.test_suite_started()
sub_suite_id = test_suite_started.meta.event_id
main_suite_id = self.main_suite_id(self.etos.config.get("context"))
self.logger.info("Send test environment events.")
self.environment(sub_suite_id)
self.etos.config.set("main_suite_id", main_suite_id)
self.etos.config.set("sub_suite_id", sub_suite_id)
result = True
description = None
try:
with Workspace(self.log_area) as workspace:
self.logger.info("Start IUT monitoring.")
self.iut_monitoring.start_monitoring()
self.logger.info("Starting test executor.")
result = self.run_tests(workspace)
executed = True
self.logger.info("Stop IUT monitoring.")
self.iut_monitoring.stop_monitoring()
except Exception as exception: # pylint:disable=broad-except
result = False
executed = False
description = str(exception)
raise
finally:
if self.iut_monitoring.monitoring:
self.logger.info("Stop IUT monitoring.")
self.iut_monitoring.stop_monitoring()
self.logger.info("Figure out test outcome.")
outcome = self.outcome(result, executed, description)
pprint(outcome)
self.logger.info("Send test suite finished event.")
self.etos.events.send_test_suite_finished(
test_suite_started,
links={"CONTEXT": self.etos.config.get("context")},
outcome=outcome,
persistentLogs=self.log_area.persistent_logs,
)
timeout = time.time() + 600 # 10 minutes
self.logger.info("Waiting for eiffel publisher to deliver events (600s).")
previous = 0
# pylint:disable=protected-access
current = len(self.etos.publisher._deliveries)
while current:
current = len(self.etos.publisher._deliveries)
self.logger.info("Remaining events to send : %d", current)
self.logger.info("Events sent since last iteration: %d", previous - current)
if time.time() > timeout:
if current < previous:
self.logger.info(
"Timeout reached, but events are still being sent. Increase timeout by 10s."
)
timeout = time.time() + 10
else:
raise Exception("Eiffel publisher did not deliver all eiffel events.")
previous = current
time.sleep(1)
self.logger.info("Tests finished executing.")
return 0 if result else outcome
def test_stop_monitoring_kill(self):
"""Verify that stop monitoring will attempt to kill script that does not terminate.
Approval criteria:
- stop monitoring shall send SIGKILL to process if SIGTERM fails.
Test steps::
1. Initialize IUT monitoring.
2. Create and add a script catching SIGINT and SIGTERM.
3. Start monitoring.
4. Stop monitoring.
5. Verify that the script was killed.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info(
"STEP: Create and add a script catching SIGINT and SIGTERM.")
script = [
"#!/bin/bash",
"int_handler() {",
" echo interrupted! > $(pwd)/output",
"}",
"term_handler() {",
" echo terminated! >> $(pwd)/output",
"}",
"trap int_handler INT",
"trap term_handler TERM",
"while :",
"do",
" sleep 1",
"done",
]
interrupt_script = Path.cwd().joinpath("kill.sh")
self.files.append(interrupt_script)
self.files.append(Path.cwd().joinpath("output"))
with open(interrupt_script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config.set(
"scripts",
[
{
"name": str(interrupt_script)
},
],
)
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
time.sleep(1)
self.logger.info("STEP: Stop monitoring.")
iut_monitoring.stop_monitoring()
self.logger.info("STEP: Verify that the script was killed.")
self.assertTrue(self._wait_for_file(Path.cwd().joinpath("output")))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
lines = output.readlines()
interrupt = lines.pop(0).strip()
self.assertEqual(interrupt, "interrupted!")
terminate = lines.pop(0).strip()
self.assertEqual(terminate, "terminated!")
def test_stop_monitoring_multiple_scripts(self):
"""Verify that stop monitoring multiple scripts interrupts them.
Approval criteria:
- stop monitoring shall send SIGINT to all processes.
Test steps::
1. Initialize IUT monitoring.
2. Create and add multiple scripts with an infinite loop.
3. Start monitoring.
4. Stop monitoring.
5. Verify that the scripts were interrupted with SIGINT.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info(
"STEP: Create and add multiple scripts with an infinite loop.")
script = [
"#!/bin/bash",
"int_handler() {",
" echo interrupted! > $(pwd)/output",
" exit 0",
"}",
"trap int_handler INT",
"while :",
"do",
" sleep 1",
"done",
]
first_interrupt_script = Path.cwd().joinpath("first_interrupt.sh")
self.files.append(first_interrupt_script)
self.files.append(Path.cwd().joinpath("output"))
with open(first_interrupt_script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
script = [
"#!/bin/bash",
"int_handler() {",
" echo interrupted! > $(pwd)/output2",
" exit 0",
"}",
"trap int_handler INT",
"while :",
"do",
" sleep 1",
"done",
]
second_interrupt_script = Path.cwd().joinpath("second_interrupt.sh")
self.files.append(second_interrupt_script)
self.files.append(Path.cwd().joinpath("output2"))
with open(second_interrupt_script, "w",
encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config.set(
"scripts",
[
{
"name": str(first_interrupt_script)
},
{
"name": str(second_interrupt_script)
},
],
)
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
time.sleep(1)
self.logger.info("STEP: Stop monitoring.")
iut_monitoring.stop_monitoring()
self.logger.info(
"STEP: Verify that the scripts were interrupted with SIGINT.")
self.assertTrue(self._wait_for_file(Path.cwd().joinpath("output")))
self.assertTrue(self._wait_for_file(Path.cwd().joinpath("output2")))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
text = output.read().strip()
self.assertEqual(text, "interrupted!")
with open(Path.cwd().joinpath("output2"), encoding="utf-8") as output:
text = output.read().strip()
self.assertEqual(text, "interrupted!")
class TestRunner:
"""Test runner for ETOS."""
logger = logging.getLogger("ETR")
def __init__(self, iut, etos):
"""Initialize.
:param iut: IUT to execute tests on.
:type iut: :obj:`etr.lib.iut.Iut`
:param etos: ETOS library
:type etos: :obj:`etos_lib.etos.ETOS`
"""
self.etos = etos
self.iut = iut
self.config = self.etos.config.get("test_config")
self.iut_monitoring = IutMonitoring(self.iut)
self.issuer = {"name": "ETOS Test Runner"}
self.env = os.environ
if not os.path.isdir(os.getenv("TEST_ARTIFACT_PATH")):
os.makedirs(os.getenv("TEST_ARTIFACT_PATH"))
if not os.path.isdir(os.getenv("TEST_LOCAL_PATH")):
os.makedirs(os.getenv("TEST_LOCAL_PATH"))
if not os.path.isdir(os.getenv("GLOBAL_ARTIFACT_PATH")):
os.makedirs(os.getenv("GLOBAL_ARTIFACT_PATH"))
def test_suite_started(self):
"""Publish a test suite started event.
:return: Reference to test suite started.
:rtype: :obj:`eiffel.events.base_event.BaseEvent`
"""
suite_name = self.config.get("name")
categories = ["Regression test_suite", "Sub suite"]
categories.append(self.iut.identity.name)
livelogs = self.config.get("log_area", {}).get("livelogs")
return self.etos.events.send_test_suite_started(
suite_name,
links={"CONTEXT": self.etos.config.get("context")},
categories=categories,
types=["FUNCTIONAL"],
liveLogs=[{"name": "console", "uri": livelogs}],
)
def main_suite_id(self, activity_id):
"""Get the eiffel event id of the mainsuite linked form the activity triggered event.
:param activity_id: Id of the activity linked to the main suite
:type activity_id: str
:return: Id of the main suite linked from the activity triggered event
:rtype: str
"""
for test_suite_started in request_test_suite_started(self.etos, activity_id):
if not "Sub suite" in test_suite_started["data"]["testSuiteCategories"]:
return test_suite_started["meta"]["id"]
raise Exception("Missing main suite events, exiting!")
def confidence_level(self, test_results, test_suite_started):
"""Publish confidence level modified event.
:param test_results: Results of the test suite.
:type test_results: bool
:param test_suite_started: Test suite started event to use as Cause for this confidence.
:type test_suite_started: :obj:`eiffellib.events.EiffelTestSuiteStartedEvent`
"""
confidence = self.etos.events.send_confidence_level_modified(
"{}_OK".format(self.config.get("name")),
"SUCCESS" if test_results else "FAILURE",
links={
"CONTEXT": self.etos.config.get("context"),
"CAUSE": test_suite_started,
"SUBJECT": self.config.get("artifact"),
},
issuer=self.issuer,
)
print(confidence.pretty)
def environment(self, context):
"""Send out which environment we're executing within.
:param context: Context where this environment is used.
:type context: str
"""
# TODO: Get this from prepare
if os.getenv("HOSTNAME") is not None:
self.etos.events.send_environment_defined(
"ETR Hostname",
links={"CONTEXT": context},
host={"name": os.getenv("HOSTNAME"), "user": "******"},
)
def run_tests(self, log_handler):
"""Execute test recipes within a test executor.
:param log_handler: Log handler for gathering logs.
:type log_handler: :obj:`etr.lib.logs.LogHandler`
:return: Result of test execution.
:rtype: bool
"""
recipes = self.config.get("recipes")
result = True
for num, test in enumerate(recipes):
self.logger.info("Executing test %s/%s", num + 1, len(recipes))
with Executor(test, self.iut, self.etos) as executor:
self.logger.info("Starting test '%s'", executor.test_name)
executor.execute()
self.logger.info("Gather logs.")
log_handler.gather_logs_for_executor(executor)
if not executor.result:
result = executor.result
self.logger.info("Test finished. Result: %s.", executor.result)
return result
def outcome(self, result, executed, description):
"""Get outcome from test execution.
:param result: Result of execution.
:type result: bool
:param executed: Whether or not tests have successfully executed.
:type executed: bool
:param description: Optional description.
:type description: str
:return: Outcome of test execution.
:rtype: dict
"""
if executed:
conclusion = "SUCCESSFUL"
verdict = "PASSED" if result else "FAILED"
self.logger.info(
"Tests executed successfully. "
"Verdict set to '%s' due to result being '%s'",
verdict,
result,
)
else:
conclusion = "FAILED"
verdict = "INCONCLUSIVE"
self.logger.info(
"Tests did not execute successfully. " "Setting verdict to '%s'",
verdict,
)
suite_name = self.config.get("name")
if not description and not result:
self.logger.info(
"No description but result is a failure. " "At least some tests failed."
)
description = "At least some {} tests failed.".format(suite_name)
elif not description and result:
self.logger.info(
"No description and result is a success. "
"All tests executed successfully."
)
description = "All {} tests completed successfully.".format(suite_name)
else:
self.logger.info("Description was set. Probably due to an exception.")
return {
"verdict": verdict,
"description": description,
"conclusion": conclusion,
}
def execute(self): # pylint:disable=too-many-branches
"""Execute all tests in test suite.
:return: Result of execution. Linux exit code.
:rtype: int
"""
self.logger.info("Send test suite started event.")
test_suite_started = self.test_suite_started()
sub_suite_id = test_suite_started.meta.event_id
main_suite_id = self.main_suite_id(self.etos.config.get("context"))
self.logger.info("Send test environment events.")
self.environment(sub_suite_id)
self.logger.info("Initialize loghandler.")
log_handler = LogHandler(
main_suite_id,
sub_suite_id,
self.etos.config.get("context"),
self.iut,
self.etos,
)
self.logger.info("Start IUT monitoring.")
self.iut_monitoring.start_monitoring()
result = True
description = None
try:
self.logger.info("Starting test executor.")
result = self.run_tests(log_handler)
executed = True
except Exception as exception: # pylint:disable=broad-except
result = False
executed = False
description = str(exception)
raise
finally:
self.logger.info("Stop IUT monitoring.")
self.iut_monitoring.stop_monitoring()
self.logger.info("Gather global test logs.")
log_handler.gather_global_logs()
self.logger.info("Figure out test outcome.")
outcome = self.outcome(result, executed, description)
pprint(outcome)
self.logger.info("Send test suite finished and confidence events.")
self.etos.events.send_test_suite_finished(
test_suite_started,
links={"CONTEXT": self.etos.config.get("context")},
outcome=outcome,
persistentLogs=log_handler.persistent_logs,
)
self.confidence_level(result, test_suite_started)
timeout = time.time() + 30
self.logger.info("Waiting for eiffel publisher to deliver events (30s).")
# pylint:disable=len-as-condition, protected-access
while len(self.etos.publisher._deliveries):
if time.time() > timeout:
raise Exception("Eiffel publisher did not deliver all eiffel events.")
time.sleep(1)
self.logger.info("Tests finished executing.")
return 0 if result else outcome