def __init__(self):
"""
Actions get added to pipelines by calling the
Pipeline.add_action function. Other Action
data comes from the parameters. Actions with
internal pipelines push parameters to actions
within those pipelines. Parameters are to be
treated as inmutable.
"""
# The level of this action within the pipeline. Levels start at one and
# each pipeline within an command uses a level within the level of the
# parent pipeline.
# First command in Outer pipeline: 1
# First command in pipeline within outer pipeline: 1.1
# Level is set during pipeline creation and must not be changed
# subsequently except by RetryCommand.
self.level = None
self.pipeline = None
self.__parameters__ = {}
self.__errors__ = []
self.job = None
self.logger = logging.getLogger("dispatcher")
self.__results__ = {}
self.timeout = Timeout(self.name, exception=self.timeout_exception)
# unless the strategy or the job parameters change this, do not retry
self.max_retries = 1
self.diagnostics = []
# list of protocol objects supported by this action, full list in job.protocols
self.protocols = []
self.connection_timeout = Timeout(self.name,
exception=self.timeout_exception)
self.character_delay = 0
self.force_prompt = False
def __init__(self, parameters, job_id):
self.logger = logging.getLogger("dispatcher")
self.poll_timeout = Timeout(self.name)
self.__errors__ = []
self.parameters = parameters
self.configured = False
self.job_id = job_id
def __init__(self, parameters, job_id):
super().__init__(parameters, job_id)
self.system_timeout = Timeout("system", LAVA_LXC_TIMEOUT)
self.persistence = parameters["protocols"][self.name].get(
"persist", False)
if self.persistence:
self.lxc_name = parameters["protocols"][self.name]["name"]
else:
self.lxc_name = "-".join(
[parameters["protocols"][self.name]["name"],
str(job_id)])
self.lxc_dist = parameters["protocols"][self.name]["distribution"]
self.lxc_release = parameters["protocols"][self.name]["release"]
self.lxc_arch = parameters["protocols"][self.name].get("arch")
self.lxc_template = parameters["protocols"][self.name].get(
"template", "download")
self.lxc_mirror = parameters["protocols"][self.name].get(
"mirror", None)
self.lxc_security_mirror = parameters["protocols"][self.name].get(
"security_mirror")
self.verbose = parameters["protocols"][self.name].get("verbose", False)
self.fastboot_reboot = parameters.get("reboot_to_fastboot", True)
self.custom_lxc_path = False
if LXC_PATH != lxc_path(parameters["dispatcher"]):
self.custom_lxc_path = True
self.logger = logging.getLogger("dispatcher")
self.job_prefix = parameters["dispatcher"].get("prefix", "")
def __init__(self, parameters, job_id):
super().__init__(parameters, job_id)
self.system_timeout = Timeout('system', LAVA_LXC_TIMEOUT)
self.persistence = parameters['protocols'][self.name].get('persist',
False)
if self.persistence:
self.lxc_name = parameters['protocols'][self.name]['name']
else:
self.lxc_name = '-'.join(
[parameters['protocols'][self.name]['name'], str(job_id)])
self.lxc_dist = parameters['protocols'][self.name]['distribution']
self.lxc_release = parameters['protocols'][self.name]['release']
self.lxc_arch = parameters['protocols'][self.name].get('arch')
self.lxc_template = parameters['protocols'][self.name].get(
'template', 'download')
self.lxc_mirror = parameters['protocols'][self.name].get('mirror',
None)
self.lxc_security_mirror = parameters['protocols'][self.name].get(
'security_mirror')
self.verbose = parameters['protocols'][self.name].get('verbose', False)
self.fastboot_reboot = parameters.get('reboot_to_fastboot', True)
self.custom_lxc_path = False
if LXC_PATH != lxc_path(parameters['dispatcher']):
self.custom_lxc_path = True
self.logger = logging.getLogger('dispatcher')
self.job_prefix = parameters["dispatcher"].get("prefix", "")
def setUp(self):
super().setUp()
self.parameters = {
"job_name":
"fakejob",
'timeouts': {
'job': {
'seconds': 3
}
},
"actions": [{
'deploy': {
'namespace': 'common',
'failure_retry': 3
},
'boot': {
'namespace': 'common',
'failure_retry': 4
},
'test': {
'namespace': 'common',
'failure_retry': 5
}
}]
}
self.fakejob = TestTimeout.FakeJob(self.parameters)
# copy of the _timeout function from parser.
if 'timeouts' in self.parameters:
if 'job' in self.parameters['timeouts']:
duration = Timeout.parse(self.parameters['timeouts']['job'])
self.fakejob.timeout = Timeout(self.parameters['job_name'],
duration)
def action(device, mocker):
a = CommandAction()
a.job = Job(42, {}, None)
a.job.timeout = Timeout("job")
a.job.device = device
a.run_cmd = mocker.MagicMock()
return a
def test_exception_raised(monkeypatch):
# 1/ default case
t = Timeout("name", 12)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([12, 0]))
with pytest.raises(JobError):
with t(None, None) as max_end_time:
t._timed_out(None, None)
# 2/ another exception
t = Timeout("name", 12, InfrastructureError)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([12, 0]))
with pytest.raises(InfrastructureError):
with t(None, None) as max_end_time:
t._timed_out(None, None)
def setUp(self):
super().setUp()
self.parameters = {
"job_name":
"fakejob",
"timeouts": {
"job": {
"seconds": 3
}
},
"actions": [{
"deploy": {
"namespace": "common",
"failure_retry": 3
},
"boot": {
"namespace": "common",
"failure_retry": 4
},
"test": {
"namespace": "common",
"failure_retry": 5
},
}],
}
self.fakejob = TestTimeout.FakeJob(self.parameters)
# copy of the _timeout function from parser.
if "timeouts" in self.parameters:
if "job" in self.parameters["timeouts"]:
duration = Timeout.parse(self.parameters["timeouts"]["job"])
self.fakejob.timeout = Timeout(self.parameters["job_name"],
duration)
def __init__(self, expect_final=True, method=None):
super().__init__()
self.params = None
self.timeout = Timeout(self.name,
BOOTLOADER_DEFAULT_CMD_TIMEOUT,
exception=self.timeout_exception)
self.method = method
self.expect_final = expect_final
def __init__(self, parameters, job_id):
super().__init__(parameters, job_id)
# timeout in utils.constants, default 10000
self.system_timeout = Timeout("system", XNBD_SYSTEM_TIMEOUT)
self.logger = logging.getLogger("dispatcher")
self.parameters = parameters
self.port = None
self.ports = []
def test_action_complete(self):
self.assertIsNotNone(self.fakejob.timeout)
seconds = 2
pipeline = TestTimeout.FakePipeline(job=self.fakejob)
action = TestTimeout.SafeAction()
action.timeout = Timeout(action.name, duration=seconds)
pipeline.add_action(action)
self.fakejob.pipeline = pipeline
self.fakejob.device = TestTimeout.FakeDevice()
self.fakejob.run()
def test_check_char(self):
shell = ShellCommand("%s\n" % "ls",
Timeout("fake", 30),
logger=logging.getLogger())
if shell.exitstatus:
raise JobError("%s command exited %d: %s" %
("ls", shell.exitstatus, shell.readlines()))
connection = ShellSession(self.job, shell)
self.assertFalse(hasattr(shell, "check_char"))
self.assertTrue(hasattr(connection, "check_char"))
self.assertIsNotNone(connection.check_char)
def test_action_timout_custom_exception(self):
seconds = 2
pipeline = TestTimeout.FakePipeline(job=self.fakejob)
action = TestTimeout.FakeAction()
action.timeout = Timeout(action.name,
duration=seconds,
exception=InfrastructureError)
pipeline.add_action(action)
self.fakejob.pipeline = pipeline
self.fakejob.device = TestTimeout.FakeDevice()
with self.assertRaises(InfrastructureError):
self.fakejob.run()
def test_action_timeout(self):
"""
Testing timeouts does mean that the tests do nothing until the timeout happens,
so the total length of time to run the tests has to increase...
"""
self.assertIsNotNone(self.fakejob.timeout)
seconds = 2
pipeline = TestTimeout.FakePipeline(job=self.fakejob)
action = TestTimeout.FakeAction()
action.timeout = Timeout(action.name, duration=seconds)
pipeline.add_action(action)
self.fakejob.pipeline = pipeline
self.fakejob.device = TestTimeout.FakeDevice()
with self.assertRaises(JobError):
self.fakejob.run()
def __init__(self, parameters, job_id):
super().__init__(parameters, job_id)
self.blocks = 4 * 1024
# how long between polls (in seconds)
self.system_timeout = Timeout('system', LAVA_MULTINODE_SYSTEM_TIMEOUT)
self.settings = None
self.sock = None
self.base_message = None
self.logger = logging.getLogger('dispatcher')
self.delayed_start = False
params = parameters['protocols'][self.name]
if 'request' in params and 'lava-start' == params['request'] and 'expect_role' in params:
if params['expect_role'] != params['role']:
self.delayed_start = True
self.system_timeout.duration = Timeout.parse(params['timeout'])
else:
self.errors = "expect_role must not match the role declaring lava_start"
self.logger.warning(self.errors)
def __init__(self, parameters, job_id):
super().__init__(parameters, job_id)
self.blocks = 4 * 1024
# how long between polls (in seconds)
self.system_timeout = Timeout("system", LAVA_MULTINODE_SYSTEM_TIMEOUT)
self.settings = None
self.sock = None
self.base_message = None
self.logger = logging.getLogger("dispatcher")
self.delayed_start = False
params = parameters["protocols"][self.name]
if ("request" in params and "lava-start" == params["request"]
and "expect_role" in params):
if params["expect_role"] != params["role"]:
self.delayed_start = True
self.system_timeout.duration = Timeout.parse(params["timeout"])
else:
self.errors = "expect_role must not match the role declaring lava_start"
self.logger.warning(self.errors)
def test_wait_for_board_id_is_optional(factory):
action = DockerTestAction()
action.job = Job("1234", {}, None)
rendered, _ = factory.create_device("hi6220-hikey-r2-01.jinja2")
action.job.device = NewDevice(yaml_load(rendered))
action.job.timeout = Timeout("blah")
action.level = 1
action.populate(
{
"namespace": "common",
"docker": {"image": "foobar", "wait": {"device": False}},
}
)
assert not any(
[a for a in action.pipeline.actions if a.name == "wait-device-boardid"]
)
docker_test_shell = action.pipeline.actions[-2]
assert not docker_test_shell.wait_for_device
def test_failure_retry_specified_interval(self):
self.parameters = {
"job_name":
"fakejob",
"timeouts": {
"job": {
"seconds": 3
}
},
"actions": [{
"deploy": {
"namespace": "common",
"failure_retry": 3,
"failure_retry_interval": 2,
},
"boot": {
"namespace": "common",
"failure_retry": 4
},
"test": {
"namespace": "common",
"failure_retry": 5
},
}],
}
self.fakejob = TestAction.FakeJob(self.parameters)
# copy of the _timeout function from parser.
if "timeouts" in self.parameters:
if "job" in self.parameters["timeouts"]:
duration = Timeout.parse(self.parameters["timeouts"]["job"])
self.fakejob.timeout = Timeout(self.parameters["job_name"],
duration)
pipeline = TestAction.FakePipeline(job=self.fakejob)
action = TestAction.InternalRetryAction()
for actions in self.lookup_deploy(self.parameters["actions"]):
action.parameters = actions
pipeline.add_action(action)
self.fakejob.pipeline = pipeline
self.fakejob.device = TestTimeout.FakeDevice()
with self.assertRaises(JobError):
self.fakejob.run()
self.assertEqual(action.sleep, 2)
def test_without_raising(monkeypatch):
# 1/ without parent
# 1.1/ without max_end_time
t = Timeout("timeout-name", 200)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([200, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with t(None, None) as max_end_time:
assert max_end_time == 200 # nosec - assert is part of the test process.
# signal.alarm and signal.signal were called once each
assert signal.alarm.data == [0] # nosec - assert is part of the test process.
assert signal.signal.data == [] # nosec - assert is part of the test process.
monkeypatch.setattr(time, "time", lambda: 23)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t.elapsed_time == 23 # nosec - assert is part of the test process.
# 1.1/ with a smaller max_end_time
t = Timeout("timeout-name", 200)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([125, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with t(None, 125) as max_end_time:
assert max_end_time == 125 # nosec - assert is part of the test process.
assert signal.alarm.data == [0] # nosec - assert is part of the test process.
assert signal.signal.data == [] # nosec - assert is part of the test process.
monkeypatch.setattr(time, "time", lambda: 109)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t.elapsed_time == 109 # nosec - assert is part of the test process.
# 1.2/ with a larger max_end_time
t = Timeout("timeout-name", 200)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([200, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with t(None, 201) as max_end_time:
assert max_end_time == 200 # nosec - assert is part of the test process.
assert signal.alarm.data == [0] # nosec - assert is part of the test process.
assert signal.signal.data == [] # nosec - assert is part of the test process.
monkeypatch.setattr(time, "time", lambda: 45)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t.elapsed_time == 45 # nosec - assert is part of the test process.
# 2/ with a parent
# 2.1/ with a larger max_end_time
t0 = Timeout("timeout-parent", 200)
parent = ParentAction(t0)
t1 = Timeout("timeout-child", 100)
monkeypatch.setattr(signal, "signal", DummySignal([t1._timed_out, t0._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([100, 177]))
monkeypatch.setattr(time, "time", lambda: 0)
with t1(parent, 200) as max_end_time:
assert max_end_time == 100 # nosec - assert is part of the test process.
assert signal.alarm.data == [177] # nosec - assert is part of the test process.
assert signal.signal.data == [ # nosec - assert is part of the test process.
t0._timed_out
]
monkeypatch.setattr(time, "time", lambda: 23)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t1.elapsed_time == 23 # nosec - assert is part of the test process.
# 2.2/ with a smaller max_end_time
t0 = Timeout("timeout-parent", 50)
parent = ParentAction(t0)
t1 = Timeout("timeout-child", 100)
monkeypatch.setattr(signal, "signal", DummySignal([t1._timed_out, t0._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([50, 27]))
monkeypatch.setattr(time, "time", lambda: 0)
with t1(parent, 50) as max_end_time:
assert max_end_time == 50 # nosec - assert is part of the test process.
assert signal.alarm.data == [27] # nosec - assert is part of the test process.
assert signal.signal.data == [ # nosec - assert is part of the test process.
t0._timed_out
]
monkeypatch.setattr(time, "time", lambda: 23)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t1.elapsed_time == 23 # nosec - assert is part of the test process.
def __init__(self, expect_final=True):
super().__init__()
self.params = None
self.timeout = Timeout(self.name, BOOTLOADER_DEFAULT_CMD_TIMEOUT)
self.method = ""
self.expect_final = expect_final
def __init__(self):
super().__init__()
self.start_message = None
self.timeout = Timeout(
self.name, BOOTLOADER_DEFAULT_CMD_TIMEOUT, exception=self.timeout_exception
)
def test_with_raising(monkeypatch):
# 1/ without parent
# 1.1/ without max_end_time
t = Timeout("timeout-name", 200)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([200, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with pytest.raises(JobError):
with t(None, None) as max_end_time:
assert max_end_time == 200 # nosec - assert is part of the test process.
assert signal.alarm.data == [ # nosec - assert is part of the test process.
0
]
assert ( # nosec - assert is part of the test process.
signal.signal.data == []
)
monkeypatch.setattr(time, "time", lambda: 200)
t._timed_out(None, None)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t.elapsed_time == 200 # nosec - assert is part of the test process.
# 1.1/ with a smaller max_end_time
t = Timeout("timeout-name", 200)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([125, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with pytest.raises(JobError):
with t(None, 125) as max_end_time:
assert max_end_time == 125 # nosec - assert is part of the test process.
assert signal.alarm.data == [ # nosec - assert is part of the test process.
0
]
assert ( # nosec - assert is part of the test process.
signal.signal.data == []
)
monkeypatch.setattr(time, "time", lambda: 126)
t._timed_out(None, None)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t.elapsed_time == 126 # nosec - assert is part of the test process.
# 1.2/ with a larger max_end_time
t = Timeout("timeout-name", 200)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([200, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with pytest.raises(JobError):
with t(None, 201) as max_end_time:
assert max_end_time == 200 # nosec - assert is part of the test process.
assert signal.alarm.data == [0] # nosec - test process.
assert signal.signal.data == [] # nosec - test process.
monkeypatch.setattr(time, "time", lambda: 200)
t._timed_out(None, None)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t.elapsed_time == 200 # nosec - assert is part of the test process.
# 1.3/ with max_end_time <= 0
t = Timeout("timeout-name", 200)
monkeypatch.setattr(signal, "signal", DummySignal([t._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([0]))
monkeypatch.setattr(time, "time", lambda: 0)
with pytest.raises(JobError):
with t(None, 0) as max_end_time:
# Check that the exception is raised before this line
assert 0 # nosec - assert is part of the test process.
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert t.elapsed_time == 0 # nosec - assert is part of the test process.
# 2/ with a parent
# 2.1/ with a larger max_end_time
t0 = Timeout("timeout-parent", 200)
parent = ParentAction(t0)
t1 = Timeout("timeout-child", 100)
monkeypatch.setattr(signal, "signal", DummySignal([t1._timed_out, t0._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([100, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with pytest.raises(JobError):
with t1(parent, 200) as max_end_time:
assert max_end_time == 100 # nosec - assert is part of the test process.
assert signal.alarm.data == [0] # nosec - test process.
assert signal.signal.data == [t0._timed_out] # nosec - test process.
monkeypatch.setattr(time, "time", lambda: 100)
t1._timed_out(None, None)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert signal.signal.data == [t0._timed_out] # nosec - test process.
assert t1.elapsed_time == 100 # nosec - assert is part of the test process.
# 2.2/ with a smaller max_end_time
t0 = Timeout("timeout-parent", 50)
parent = ParentAction(t0)
t1 = Timeout("timeout-child", 100)
monkeypatch.setattr(signal, "signal", DummySignal([t1._timed_out, t0._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([50, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with pytest.raises(JobError):
with t1(parent, 50) as max_end_time:
assert max_end_time == 50 # nosec - assert is part of the test process.
assert signal.alarm.data == [0] # nosec - test process.
assert signal.signal.data == [t0._timed_out] # nosec - test process.
monkeypatch.setattr(time, "time", lambda: 23)
t1._timed_out(None, None)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert signal.signal.data == [t0._timed_out] # nosec - test process.
assert t1.elapsed_time == 23 # nosec - assert is part of the test process.
# 2.3/ with max_end_time <= 0
t0 = Timeout("timeout-parent", 1)
parent = ParentAction(t0)
t1 = Timeout("timeout-child", 100)
monkeypatch.setattr(signal, "signal", DummySignal([t1._timed_out, t0._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([0]))
monkeypatch.setattr(time, "time", lambda: 0)
with pytest.raises(JobError):
with t1(parent, -1) as max_end_time:
assert 0 # nosec - assert is part of the test process.
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert signal.signal.data == [t1._timed_out, t0._timed_out] # nosec - test process.
assert t1.elapsed_time == 0 # nosec - assert is part of the test process.
# 2.4/ raising parent timeout
t0 = Timeout("timeout-parent", 50, InfrastructureError)
parent = ParentAction(t0)
t1 = Timeout("timeout-child", 100)
monkeypatch.setattr(signal, "signal", DummySignal([t1._timed_out, t0._timed_out]))
monkeypatch.setattr(signal, "alarm", DummyAlarm([50, 0, 0]))
monkeypatch.setattr(time, "time", lambda: 0)
with pytest.raises(InfrastructureError):
with t1(parent, 50) as max_end_time:
assert max_end_time == 50 # nosec - assert is part of the test process.
assert signal.alarm.data == [0, 0] # nosec - test
assert signal.signal.data == [t0._timed_out] # nosec - test
monkeypatch.setattr(time, "time", lambda: 50)
assert signal.alarm.data == [] # nosec - assert is part of the test process.
assert signal.signal.data == [] # nosec - assert is part of the test process.
assert t1.elapsed_time == 50 # nosec - assert is part of the test process.