def clear_etos_lib_configurations():
"""Make sure that etos library configuration is cleared after each test."""
config = Config()
debug = Debug()
yield
config.reset()
debug.events_received.clear()
debug.events_published.clear()
def setUp(self):
"""Create a script file."""
script = ["#!/bin/bash", "echo Hello $1 > $(pwd)/output"]
with open(self.script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config = Config()
self.files = [self.script, Path.cwd().joinpath("output")]
def __init__(self, iut):
"""Initialize monitoring.
:param iut: IUT object to monitor.
:type iut: :obj:`etr.lib.iut.Iut`
"""
self.iut = iut
self.processes = []
self.config = Config()
def __init__(self, product):
"""Initialize.
:param product: Dictionary to set attributes from.
Should be the response from pool plugin list.
:type product: dict
"""
self.test_runner = {}
self.config = Config()
self.config.set("scripts", [])
self.steps = {
"environment": self.load_environment,
"commands": self.commands
}
product["identity"] = PackageURL.from_string(product["identity"])
for key, value in product.items():
setattr(self, key, value)
self._product_dict = product
self.jsontas = JsonTas()
self.jsontas.dataset.add("iut", self._product_dict)
self.prepare()
class TestIutMonitoring(TestCase):
"""Tests for the IUT monitoring library."""
logger = logging.getLogger(__name__)
script = Path.cwd().joinpath("script.sh")
def setUp(self):
"""Create a script file."""
script = ["#!/bin/bash", "echo Hello $1 > $(pwd)/output"]
with open(self.script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config = Config()
self.files = [self.script, Path.cwd().joinpath("output")]
def tearDown(self):
"""Remove script file."""
for script in self.files:
self.logger.debug("Removing %r", script)
try:
script.unlink()
except FileNotFoundError:
pass
self.config.set("scripts", [])
def _wait_for_file(self, filename, timeout=5):
"""Wait for a file to exist.
:param filename: Absolute path to file.
:type filename: str
:param timeout: Maximum time to wait for file to exist. In seconds.
:type timeout: int
:return: True if the file exists, False if it does not.
:rtype: bool
"""
end = time.time() + timeout
while time.time() < end:
# Sleep is done before checking if the file exists.
# This adds 1 second to all tests that utilize this method, but
# it will assist with race conditions when the script writes to
# file. I.e. the file can exist, but no data in it yet.
time.sleep(1)
if os.path.exists(filename):
return True
self.logger.warning("File did not exist after %r seconds.", timeout)
return False
def test_start_monitoring_single_script(self):
"""Verify that the start monitoring method can execute a single script.
Approval criteria:
- Start monitoring shall execute a script to completion.
Test steps::
1. Initialize IUT monitoring.
2. Load a script to the config.
3. Start monitoring.
4. Verify that the script executed.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info("STEP: Load script to the config.")
self.config.set("scripts", [{
"name": str(self.script),
"parameters": ["world"]
}])
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
self.logger.info("STEP: Verify that the script executed.")
self._wait_for_file(Path.cwd().joinpath("output"))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
hello_world = output.read().strip()
self.logger.info(hello_world)
self.assertEqual(hello_world, "Hello world")
def test_start_monitoring_multiple_scripts(self):
"""Verify that the start monitoring method can execute multiple scripts.
Approval criteria:
- Start monitoring shall execute multiple scripts to completion.
Test steps::
1. Initialize IUT monitoring.
2. Load the scripts to the config.
3. Start monitoring.
4. Verify that the scripts executed.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info("STEP: Load the scripts to the config.")
script = ["#!/bin/bash", "echo Goodbye $1 > $(pwd)/output2"]
second_script = Path.cwd().joinpath("second.sh")
self.files.append(second_script)
self.files.append(Path.cwd().joinpath("output2"))
with open(second_script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config.set(
"scripts",
[
{
"name": str(self.script),
"parameters": ["world"]
},
{
"name": str(second_script),
"parameters": ["world"]
},
],
)
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
self.logger.info("STEP: Verify that the scripts executed.")
self._wait_for_file(Path.cwd().joinpath("output"))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
hello_world = output.read().strip()
self.logger.info(hello_world)
self._wait_for_file(Path.cwd().joinpath("output2"))
with open(Path.cwd().joinpath("output2"), encoding="utf-8") as output:
goodbye_world = output.read().strip()
self.logger.info(goodbye_world)
self.assertEqual(hello_world, "Hello world")
self.assertEqual(goodbye_world, "Goodbye world")
def test_stop_monitoring_single_script(self):
"""Verify that stop monitoring a single script interrupt it.
Approval criteria:
- stop monitoring shall send SIGINT to process.
Test steps::
1. Initialize IUT monitoring.
2. Create and add a script with an infinite loop.
3. Start monitoring.
4. Stop monitoring.
5. Verify that the script was interrupted with SIGINT.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info(
"STEP: Create and add a script with an infinite loop.")
script = [
"#!/bin/bash",
"int_handler() {",
" echo interrupted! > $(pwd)/output",
" exit 0",
"}",
"trap int_handler INT",
"while :",
"do",
" sleep 1",
"done",
]
interrupt_script = Path.cwd().joinpath("interrupt.sh")
self.files.append(interrupt_script)
self.files.append(Path.cwd().joinpath("output"))
with open(interrupt_script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config.set(
"scripts",
[
{
"name": str(interrupt_script)
},
],
)
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
time.sleep(1)
self.logger.info("STEP: Stop monitoring.")
iut_monitoring.stop_monitoring()
self.logger.info(
"STEP: Verify that the script was interrupted with SIGINT.")
self.assertTrue(self._wait_for_file(Path.cwd().joinpath("output")))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
text = output.read().strip()
self.assertEqual(text, "interrupted!")
def test_stop_monitoring_multiple_scripts(self):
"""Verify that stop monitoring multiple scripts interrupts them.
Approval criteria:
- stop monitoring shall send SIGINT to all processes.
Test steps::
1. Initialize IUT monitoring.
2. Create and add multiple scripts with an infinite loop.
3. Start monitoring.
4. Stop monitoring.
5. Verify that the scripts were interrupted with SIGINT.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info(
"STEP: Create and add multiple scripts with an infinite loop.")
script = [
"#!/bin/bash",
"int_handler() {",
" echo interrupted! > $(pwd)/output",
" exit 0",
"}",
"trap int_handler INT",
"while :",
"do",
" sleep 1",
"done",
]
first_interrupt_script = Path.cwd().joinpath("first_interrupt.sh")
self.files.append(first_interrupt_script)
self.files.append(Path.cwd().joinpath("output"))
with open(first_interrupt_script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
script = [
"#!/bin/bash",
"int_handler() {",
" echo interrupted! > $(pwd)/output2",
" exit 0",
"}",
"trap int_handler INT",
"while :",
"do",
" sleep 1",
"done",
]
second_interrupt_script = Path.cwd().joinpath("second_interrupt.sh")
self.files.append(second_interrupt_script)
self.files.append(Path.cwd().joinpath("output2"))
with open(second_interrupt_script, "w",
encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config.set(
"scripts",
[
{
"name": str(first_interrupt_script)
},
{
"name": str(second_interrupt_script)
},
],
)
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
time.sleep(1)
self.logger.info("STEP: Stop monitoring.")
iut_monitoring.stop_monitoring()
self.logger.info(
"STEP: Verify that the scripts were interrupted with SIGINT.")
self.assertTrue(self._wait_for_file(Path.cwd().joinpath("output")))
self.assertTrue(self._wait_for_file(Path.cwd().joinpath("output2")))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
text = output.read().strip()
self.assertEqual(text, "interrupted!")
with open(Path.cwd().joinpath("output2"), encoding="utf-8") as output:
text = output.read().strip()
self.assertEqual(text, "interrupted!")
def test_stop_monitoring_terminate(self):
"""Verify that stop monitoring will attempt to terminate script that does not interrupt.
Approval criteria:
- stop monitoring shall send SEGTERM to process if SIGINT fails.
Test steps::
1. Initialize IUT monitoring.
2. Create and add a script catching SIGINT.
3. Start monitoring.
4. Stop monitoring.
5. Verify that the script was interrupted with SIGTERM.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info("STEP: Create and add a script catching SIGINT.")
script = [
"#!/bin/bash",
"int_handler() {",
" echo interrupted! > $(pwd)/output",
"}",
"term_handler() {",
" echo terminated! >> $(pwd)/output",
" exit 0",
"}",
"trap int_handler INT",
"trap term_handler TERM",
"while :",
"do",
" sleep 1",
"done",
]
interrupt_script = Path.cwd().joinpath("terminate.sh")
self.files.append(interrupt_script)
self.files.append(Path.cwd().joinpath("output"))
with open(interrupt_script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config.set(
"scripts",
[
{
"name": str(interrupt_script)
},
],
)
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
time.sleep(1)
self.logger.info("STEP: Stop monitoring.")
iut_monitoring.stop_monitoring()
self.logger.info(
"STEP: Verify that the script was interrupted with SIGINT.")
self.assertTrue(self._wait_for_file(Path.cwd().joinpath("output")))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
lines = output.readlines()
interrupt = lines.pop(0).strip()
self.assertEqual(interrupt, "interrupted!")
terminate = lines.pop(0).strip()
self.assertEqual(terminate, "terminated!")
def test_stop_monitoring_kill(self):
"""Verify that stop monitoring will attempt to kill script that does not terminate.
Approval criteria:
- stop monitoring shall send SIGKILL to process if SIGTERM fails.
Test steps::
1. Initialize IUT monitoring.
2. Create and add a script catching SIGINT and SIGTERM.
3. Start monitoring.
4. Stop monitoring.
5. Verify that the script was killed.
"""
self.logger.info("STEP: Initialize IUT monitoring.")
iut_monitoring = IutMonitoring(None)
iut_monitoring.interrupt_timeout = 5
iut_monitoring.terminate_timeout = 5
iut_monitoring.kill_timeout = 5
self.logger.info(
"STEP: Create and add a script catching SIGINT and SIGTERM.")
script = [
"#!/bin/bash",
"int_handler() {",
" echo interrupted! > $(pwd)/output",
"}",
"term_handler() {",
" echo terminated! >> $(pwd)/output",
"}",
"trap int_handler INT",
"trap term_handler TERM",
"while :",
"do",
" sleep 1",
"done",
]
interrupt_script = Path.cwd().joinpath("kill.sh")
self.files.append(interrupt_script)
self.files.append(Path.cwd().joinpath("output"))
with open(interrupt_script, "w", encoding="utf-8") as scriptfile:
for line in script:
scriptfile.write(f"{line}\n")
self.config.set(
"scripts",
[
{
"name": str(interrupt_script)
},
],
)
self.logger.info("STEP: Start monitoring.")
iut_monitoring.start_monitoring()
time.sleep(1)
self.logger.info("STEP: Stop monitoring.")
iut_monitoring.stop_monitoring()
self.logger.info("STEP: Verify that the script was killed.")
self.assertTrue(self._wait_for_file(Path.cwd().joinpath("output")))
with open(Path.cwd().joinpath("output"), encoding="utf-8") as output:
lines = output.readlines()
interrupt = lines.pop(0).strip()
self.assertEqual(interrupt, "interrupted!")
terminate = lines.pop(0).strip()
self.assertEqual(terminate, "terminated!")
class IutMonitoring:
"""Helper class for monitoring IuT health statistics."""
logger = logging.getLogger("IUT Monitoring")
interrupt_timeout = 60 # Seconds
terminate_timeout = 30 # Seconds
kill_timeout = 30 # Seconds
monitoring = False
def __init__(self, iut):
"""Initialize monitoring.
:param iut: IUT object to monitor.
:type iut: :obj:`etr.lib.iut.Iut`
"""
self.iut = iut
self.processes = []
self.config = Config()
def _read_from_process(self, output):
"""Non-blocking read from a process output.
:param output: Output to read from.
:type output: filedescriptor
"""
self.logger.info("Reading output from %r", output)
for line in iter(output.readline, b""):
self.logger.info(line.decode("utf-8"))
output.close()
def start_monitoring(self):
"""Start monitoring IUT."""
if self.monitoring is True:
self.logger.info("Monitoring is already started.")
return
scripts = self.config.get("scripts") or []
for script in scripts:
self.logger.info(
"Starting script %r with parameters %r.",
script.get("name"),
script.get("parameters"),
)
# Make file executable.
filestat = os.stat(script.get("name"))
os.chmod(script.get("name"), filestat.st_mode | stat.S_IEXEC)
# These processes are closed elsewhere. pylint:disable=consider-using-with
process = Popen(
[script.get("name"), *script.get("parameters", [])],
stdout=PIPE,
stderr=STDOUT,
close_fds=ON_POSIX,
)
self.processes.append(process)
Thread(target=self._read_from_process,
daemon=True,
args=(process.stdout, )).start()
self.monitoring = True
def stop_monitoring(self):
"""Stop monitoring IUT."""
if self.monitoring is False:
self.logger.info("Monitoring is already stopped.")
return
self.monitoring = False
for process in self.processes:
self.logger.info(
"Interrupting process: %r (%rs timeout)",
self.interrupt_timeout,
process,
)
process.send_signal(SIGINT)
try:
try:
process.communicate(timeout=self.interrupt_timeout)
except TimeoutExpired:
self.logger.error(
"Unable to stop with SIGINT, terminating with SIGTERM (%rs timeout)",
self.terminate_timeout,
)
process.terminate()
try:
process.communicate(timeout=self.terminate_timeout)
except TimeoutExpired:
self.logger.error(
"Unable to stop with SIGTERM, killing with SIGKILL (%rs timeout).",
self.kill_timeout,
)
process.kill()
try:
process.communicate(timeout=self.kill_timeout)
except TimeoutExpired:
self.logger.error(
"Still unable to kill it. Return and have python clean up."
)
except OSError as exception:
self.logger.error("OS Error %r. Exiting.", exception)
def setUp(self):
os.environ["ETOS_CONFIGMAP"] = self.etos_configmap
os.environ["SUITE_RUNNER"] = self.suite_runner
Config().reset()
class Iut: # pylint: disable=too-few-public-methods
"""Data object for IUTs."""
logger = logging.getLogger(__name__)
def __init__(self, product):
"""Initialize.
:param product: Dictionary to set attributes from.
Should be the response from pool plugin list.
:type product: dict
"""
self.test_runner = {}
self.config = Config()
self.config.set("scripts", [])
self.steps = {
"environment": self.load_environment,
"commands": self.commands
}
product["identity"] = PackageURL.from_string(product["identity"])
for key, value in product.items():
setattr(self, key, value)
self._product_dict = product
self.jsontas = JsonTas()
self.jsontas.dataset.add("iut", self._product_dict)
self.prepare()
def prepare(self):
"""Prepare IUT for testing."""
self.logger.info("Preparing IUT %r", self)
for step, definition in self.test_runner.get("steps", {}).items():
step_method = self.steps.get(step)
if step_method is None:
self.logger.error("Step %r does not exist. Available %r", step,
self.steps)
continue
self.logger.info("Executing step %r", step)
self.logger.info("Definition: %r", definition)
if isinstance(definition, list):
for sub_definition in definition:
sub_definition = OrderedDict(**sub_definition)
step_result = self.jsontas.run(json_data=sub_definition)
step_method(step_result)
else:
definition = OrderedDict(**definition)
step_result = self.jsontas.run(json_data=definition)
step_method(step_result)
@staticmethod
def load_environment(environment):
"""Load and set environment variables from IUT definition.
:param environment: Environment variables to set.
:type environment: dict
"""
for key, value in environment.items():
os.environ[key] = value
def commands(self, command):
"""Create scripts from commands and add them to IUT monitoring.
:param command: A command dictionary to prepare.
:type command: dict
"""
script_name = str(Path.cwd().joinpath(command.get("name")))
parameters = command.get("parameters", [])
with open(script_name, "w", encoding="utf-8") as script:
for line in command.get("script"):
script.write(f"{line}\n")
self.config.get("scripts").append({
"name": script_name,
"parameters": parameters
})
@property
def as_dict(self):
"""Return IUT as a dictionary."""
return self._product_dict
def __repr__(self):
"""Represent IUT as string."""
try:
return self._product_dict.get("identity").to_string()
except: # noqa pylint:disable=bare-except
return "Unknown"