def test_queue_timeout_raised(self):
def run(): # A dummy timeouting function
time.sleep(3)
queue_timeout = 1
execution_timeout = 1
controller_timeout = 1
timeout = Timeout(execution_timeout, queue_timeout, controller_timeout)
timeout.start_queue_timeout()
self.assertRaises(OtsQueueTimeoutError, run)
def test_queue_timeout_not_raised(self):
self.done = False
def run(): # A dummy not timeouting function
self.done = True
queue_timeout = 1
execution_timeout = 1
controller_timeout = 1
timeout = Timeout(execution_timeout, queue_timeout, controller_timeout)
timeout.start_queue_timeout()
run()
self.assertTrue(self.done)
def test_worker_alive_after_server_timeout_failing_task(self):
if not DEBUG:
self.worker_processes.start()
self.testrun_id = 111
self.testrun_id2 = 112
taskrunner1 = taskrunner_factory(
routing_key=ROUTING_KEY,
execution_timeout=10,
testrun_id=self.testrun_id,
config_file=_distributor_config_filename())
command = [
"sleep", "5", ";", "command_error_mock", "localhost",
str(self.testrun_id)
]
taskrunner1.add_task(command)
self.is_exception_raised = False
def cb_handler(signal, dto, **kwargs):
if isinstance(dto, Exception):
self.is_exception_raised = True
DTO_SIGNAL.connect(cb_handler)
# Overwrite server side timeout handler with a one that timeouts
from ots.server.distributor.timeout import Timeout
taskrunner1.timeout_handler = Timeout(1, 1, 1)
self.assertRaises(OtsExecutionTimeoutError, taskrunner1.run)
# self.assertTrue(self.is_exception_raised)
self.is_exception_raised = False
time.sleep(10) # Give worker time to reconnect
# Trigger another task to make sure worker is still alive
taskrunner2 = taskrunner_factory(
routing_key=ROUTING_KEY,
execution_timeout=10,
testrun_id=self.testrun_id2,
config_file=_distributor_config_filename())
taskrunner2.add_task(["echo", "foo"])
taskrunner2.run()
self.assertFalse(self.is_exception_raised)
self.send_quit()
time.sleep(1)
self.assertFalse(all(self.worker_processes.exitcodes))
def test_stop(self):
self.done = False
def run(): # A dummy timeouting function
time.sleep(2)
self.done = True
queue_timeout = 1
controller_timeout = 1
execution_timeout = 1
timeout = Timeout(execution_timeout, queue_timeout, controller_timeout)
timeout.start_queue_timeout()
timeout.stop()
run()
self.assertTrue(self.done)
def __init__(self,
username,
password,
host,
vhost,
services_exchange,
port,
routing_key,
testrun_id,
execution_timeout,
queue_timeout,
controller_timeout,
min_worker_version=None):
"""
@type username: C{str}
@param username: AMQP username
@type password: C{str}
@param password: AMQP password
@type host: C{str}
@param host: AMQP host
@type vhost: C{str}
@param vhost: AMQP vhost
@type services_exchange: C{str}
@param services_exchange: AMQP services exchange
@type port: C{int}
@param port: AMQP port (generally 5672)
@type routing_key: C{routing_key}
@param routing_key: AMQP routing_key (device group)
@type testrun_id: C{int}
@param testrun_id: The testrun id
@type execution_timeout: C{int}
@param execution_timeout: Time out in seconds for Task execution
@type queue_timeout: C{int}
@param queue_timeout: Time in seconds Tasks can wait on the queue
@type controller_timeout: C{int}
@param controller_timeout: The timeout for HW controllers
"""
#AMQP configuration
self._username = username
self._password = password
self._host = host
self._vhost = vhost
self._services_exchange = services_exchange
self._port = port
#
self._routing_key = routing_key
self._testrun_id = testrun_id
#
self._init_amqp()
#List of Tasks
self._tasks = []
self._is_run = False
#
self._min_worker_version = min_worker_version
#timeouts
self._execution_timeout = execution_timeout
self._queue_timeout = queue_timeout
self._controller_timeout = controller_timeout
self.timeout_handler = Timeout(execution_timeout, queue_timeout,
controller_timeout)
