def test_queue_timeout_raised(self):
def run(): # A dummy timeouting function
time.sleep(3)
queue_timeout = 1
execution_timeout = 1
controller_timeout = 1
timeout = Timeout(execution_timeout, queue_timeout, controller_timeout)
timeout.start_queue_timeout()
self.assertRaises(OtsQueueTimeoutError, run)
def test_queue_timeout_raised(self):
def run(): # A dummy timeouting function
time.sleep(3)
queue_timeout = 1
execution_timeout = 1
controller_timeout = 1
timeout = Timeout(execution_timeout,
queue_timeout,
controller_timeout)
timeout.start_queue_timeout()
self.assertRaises(OtsQueueTimeoutError, run)
def test_queue_timeout_not_raised(self):
self.done = False
def run(): # A dummy not timeouting function
self.done = True
queue_timeout = 1
execution_timeout = 1
controller_timeout = 1
timeout = Timeout(execution_timeout, queue_timeout, controller_timeout)
timeout.start_queue_timeout()
run()
self.assertTrue(self.done)
def test_queue_timeout_not_raised(self):
self.done = False
def run(): # A dummy not timeouting function
self.done = True
queue_timeout = 1
execution_timeout = 1
controller_timeout = 1
timeout = Timeout(execution_timeout,
queue_timeout,
controller_timeout)
timeout.start_queue_timeout()
run()
self.assertTrue(self.done)
def test_stop(self):
self.done = False
def run(): # A dummy timeouting function
time.sleep(2)
self.done = True
queue_timeout = 1
controller_timeout = 1
execution_timeout = 1
timeout = Timeout(execution_timeout,
queue_timeout,
controller_timeout)
timeout.start_queue_timeout()
timeout.stop()
run()
self.assertTrue(self.done)
def test_worker_alive_after_server_timeout_failing_task(self):
if not DEBUG:
self.worker_processes.start()
self.testrun_id = 111
self.testrun_id2 = 112
taskrunner1 = taskrunner_factory(
routing_key=ROUTING_KEY,
execution_timeout=10,
testrun_id=self.testrun_id,
config_file=_distributor_config_filename())
command = [
"sleep", "5", ";", "command_error_mock", "localhost",
str(self.testrun_id)
]
taskrunner1.add_task(command)
self.is_exception_raised = False
def cb_handler(signal, dto, **kwargs):
if isinstance(dto, Exception):
self.is_exception_raised = True
DTO_SIGNAL.connect(cb_handler)
# Overwrite server side timeout handler with a one that timeouts
from ots.server.distributor.timeout import Timeout
taskrunner1.timeout_handler = Timeout(1, 1, 1)
self.assertRaises(OtsExecutionTimeoutError, taskrunner1.run)
# self.assertTrue(self.is_exception_raised)
self.is_exception_raised = False
time.sleep(10) # Give worker time to reconnect
# Trigger another task to make sure worker is still alive
taskrunner2 = taskrunner_factory(
routing_key=ROUTING_KEY,
execution_timeout=10,
testrun_id=self.testrun_id2,
config_file=_distributor_config_filename())
taskrunner2.add_task(["echo", "foo"])
taskrunner2.run()
self.assertFalse(self.is_exception_raised)
self.send_quit()
time.sleep(1)
self.assertFalse(all(self.worker_processes.exitcodes))
def test_stop(self):
self.done = False
def run(): # A dummy timeouting function
time.sleep(2)
self.done = True
queue_timeout = 1
controller_timeout = 1
execution_timeout = 1
timeout = Timeout(execution_timeout, queue_timeout, controller_timeout)
timeout.start_queue_timeout()
timeout.stop()
run()
self.assertTrue(self.done)
class TaskRunner(object):
"""
Allows Tasks to be run remotely as a set associated with the Testrun
Each Task is sent individually to RabbitMQ
Wait for all tasks to comeback.
The TaskRunner can only be run once.
Results from the tasks are signalled with names
associated with MESSAGE_TYPES.
"""
def __init__(self, username, password, host, vhost,
services_exchange, port,
routing_key, testrun_id,
execution_timeout, queue_timeout, controller_timeout,
min_worker_version = None):
"""
@type username: C{str}
@param username: AMQP username
@type password: C{str}
@param password: AMQP password
@type host: C{str}
@param host: AMQP host
@type vhost: C{str}
@param vhost: AMQP vhost
@type services_exchange: C{str}
@param services_exchange: AMQP services exchange
@type port: C{int}
@param port: AMQP port (generally 5672)
@type routing_key: C{routing_key}
@param routing_key: AMQP routing_key (device group)
@type testrun_id: C{int}
@param testrun_id: The testrun id
@type execution_timeout: C{int}
@param execution_timeout: Time out in seconds for Task execution
@type queue_timeout: C{int}
@param queue_timeout: Time in seconds Tasks can wait on the queue
@type controller_timeout: C{int}
@param controller_timeout: The timeout for HW controllers
"""
#AMQP configuration
self._username = username
self._password = password
self._host = host
self._vhost = vhost
self._services_exchange = services_exchange
self._port = port
#
self._routing_key = routing_key
self._testrun_id = testrun_id
#
self._init_amqp()
#List of Tasks
self._tasks = []
self._is_run = False
#
self._min_worker_version = min_worker_version
#timeouts
self._execution_timeout = execution_timeout
self._queue_timeout = queue_timeout
self._controller_timeout = controller_timeout
self.timeout_handler = Timeout(execution_timeout,
queue_timeout,
controller_timeout)
#############################################
# MESSAGE HANDLING
#############################################
def _on_message(self, amqp_message):
"""
Handler for AMQP messages
Two types of messages:
1. Indication of state changes on the Task
2. Feedback from the Task itself
State change messages trigger a state transition.
Everything else fires a signal
@type amqp_message: amqplib.client_0_8.basic_message.Message
@param amqp_message: AMQP message
"""
msg = unpack_message(amqp_message)
if isinstance(msg, StateChangeMessage):
LOGGER.debug("Received state change message %s, task %s "\
% (msg.condition, msg.task_id))
self._task_transition(msg)
else:
#The message is data. Relay using a signal
# If message is monitor message, make received timestamp
if isinstance(msg, Monitor):
msg.set_received()
DTO_SIGNAL.send(sender = "TaskRunner", dto = msg)
def _task_transition(self, message):
"""
Processes state change message
"""
if message.is_start:
self.timeout_handler.task_started()
task = self._get_task(message.task_id)
task.transition(message.condition)
if task.is_finished:
self._tasks.remove(task)
##########################################
# OTHER HELPERS
##########################################
def _init_amqp(self):
"""
Make the AMQP connection.
Prepare a Queue
Start Consuming
"""
# Disabling "Attribute defined outside __init__" because this method is
# called from the __init__
#pylint: disable=W0201
self._testrun_queue = testrun_queue_name(self._testrun_id)
self._connection = amqp.Connection(host = self._host,
userid = self._username,
password = self._password,
virtual_host = self._vhost,
insist = False)
self._channel = self._connection.channel()
_init_queue(self._channel,
self._testrun_queue,
self._testrun_queue,
self._testrun_queue)
self._channel.basic_consume(queue = self._testrun_queue,
callback = self._on_message,
no_ack = True)
def _get_task(self, task_id):
"""
Get the Task for the given task_id
@type task_id: string
@param task_id: The ID of the Task
@rtype: L{Task}
@return: A Task
"""
return dict([(task.task_id, task) for task in self._tasks])[task_id]
def _dispatch_tasks(self):
"""
Publish the Tasks to the RabbitMQ
"""
self.timeout_handler.start_queue_timeout()
for task in self._tasks:
log_msg = "Sending command '%s' with key '%s'" \
% (task.command, self._routing_key)
LOGGER.debug(log_msg)
#Send task in queue event with task id
send_monitor_event(MonitorType.TASK_INQUEUE,
__name__,
task.task_id)
cmd_msg = CommandMessage(task.command,
self._testrun_queue,
task.task_id,
timeout = self._execution_timeout,
xml_file = task.xml_file,
min_worker_version =
self._min_worker_version)
message = pack_message(cmd_msg)
self._channel.basic_publish(message,
exchange = self._services_exchange,
routing_key = self._routing_key)
def _wait_for_all_tasks(self):
"""
Block until all Tasks are complete
"""
while 1:
try:
self._channel.wait()
except socket.error, error:
# interrupted system call exception need to be ignored so that
# testruns don't fail on apache graceful restart
if error[0] == errno.EINTR:
LOGGER.debug("Interrupted system call. Ignoring...")
else:
raise
if len(self._tasks) == 0:
break
def __init__(self, username, password, host, vhost,
services_exchange, port,
routing_key, testrun_id,
execution_timeout, queue_timeout, controller_timeout,
min_worker_version = None):
"""
@type username: C{str}
@param username: AMQP username
@type password: C{str}
@param password: AMQP password
@type host: C{str}
@param host: AMQP host
@type vhost: C{str}
@param vhost: AMQP vhost
@type services_exchange: C{str}
@param services_exchange: AMQP services exchange
@type port: C{int}
@param port: AMQP port (generally 5672)
@type routing_key: C{routing_key}
@param routing_key: AMQP routing_key (device group)
@type testrun_id: C{int}
@param testrun_id: The testrun id
@type execution_timeout: C{int}
@param execution_timeout: Time out in seconds for Task execution
@type queue_timeout: C{int}
@param queue_timeout: Time in seconds Tasks can wait on the queue
@type controller_timeout: C{int}
@param controller_timeout: The timeout for HW controllers
"""
#AMQP configuration
self._username = username
self._password = password
self._host = host
self._vhost = vhost
self._services_exchange = services_exchange
self._port = port
#
self._routing_key = routing_key
self._testrun_id = testrun_id
#
self._init_amqp()
#List of Tasks
self._tasks = []
self._is_run = False
#
self._min_worker_version = min_worker_version
#timeouts
self._execution_timeout = execution_timeout
self._queue_timeout = queue_timeout
self._controller_timeout = controller_timeout
self.timeout_handler = Timeout(execution_timeout,
queue_timeout,
controller_timeout)
class TaskRunner(object):
"""
Allows Tasks to be run remotely as a set associated with the Testrun
Each Task is sent individually to RabbitMQ
Wait for all tasks to comeback.
The TaskRunner can only be run once.
Results from the tasks are signalled with names
associated with MESSAGE_TYPES.
"""
def __init__(self,
username,
password,
host,
vhost,
services_exchange,
port,
routing_key,
testrun_id,
execution_timeout,
queue_timeout,
controller_timeout,
min_worker_version=None):
"""
@type username: C{str}
@param username: AMQP username
@type password: C{str}
@param password: AMQP password
@type host: C{str}
@param host: AMQP host
@type vhost: C{str}
@param vhost: AMQP vhost
@type services_exchange: C{str}
@param services_exchange: AMQP services exchange
@type port: C{int}
@param port: AMQP port (generally 5672)
@type routing_key: C{routing_key}
@param routing_key: AMQP routing_key (device group)
@type testrun_id: C{int}
@param testrun_id: The testrun id
@type execution_timeout: C{int}
@param execution_timeout: Time out in seconds for Task execution
@type queue_timeout: C{int}
@param queue_timeout: Time in seconds Tasks can wait on the queue
@type controller_timeout: C{int}
@param controller_timeout: The timeout for HW controllers
"""
#AMQP configuration
self._username = username
self._password = password
self._host = host
self._vhost = vhost
self._services_exchange = services_exchange
self._port = port
#
self._routing_key = routing_key
self._testrun_id = testrun_id
#
self._init_amqp()
#List of Tasks
self._tasks = []
self._is_run = False
#
self._min_worker_version = min_worker_version
#timeouts
self._execution_timeout = execution_timeout
self._queue_timeout = queue_timeout
self._controller_timeout = controller_timeout
self.timeout_handler = Timeout(execution_timeout, queue_timeout,
controller_timeout)
#############################################
# MESSAGE HANDLING
#############################################
def _on_message(self, amqp_message):
"""
Handler for AMQP messages
Two types of messages:
1. Indication of state changes on the Task
2. Feedback from the Task itself
State change messages trigger a state transition.
Everything else fires a signal
@type amqp_message: amqplib.client_0_8.basic_message.Message
@param amqp_message: AMQP message
"""
msg = unpack_message(amqp_message)
if isinstance(msg, StateChangeMessage):
LOGGER.debug("Received state change message %s, task %s "\
% (msg.condition, msg.task_id))
self._task_transition(msg)
else:
#The message is data. Relay using a signal
# If message is monitor message, make received timestamp
if isinstance(msg, Monitor):
msg.set_received()
DTO_SIGNAL.send(sender="TaskRunner", dto=msg)
def _task_transition(self, message):
"""
Processes state change message
"""
if message.is_start:
self.timeout_handler.task_started()
task = self._get_task(message.task_id)
task.transition(message.condition)
if task.is_finished:
self._tasks.remove(task)
##########################################
# OTHER HELPERS
##########################################
def _init_amqp(self):
"""
Make the AMQP connection.
Prepare a Queue
Start Consuming
"""
# Disabling "Attribute defined outside __init__" because this method is
# called from the __init__
#pylint: disable=W0201
self._testrun_queue = testrun_queue_name(self._testrun_id)
self._connection = amqp.Connection(host=self._host,
userid=self._username,
password=self._password,
virtual_host=self._vhost,
insist=False)
self._channel = self._connection.channel()
_init_queue(self._channel, self._testrun_queue, self._testrun_queue,
self._testrun_queue)
self._channel.basic_consume(queue=self._testrun_queue,
callback=self._on_message,
no_ack=True)
def _get_task(self, task_id):
"""
Get the Task for the given task_id
@type task_id: string
@param task_id: The ID of the Task
@rtype: L{Task}
@return: A Task
"""
return dict([(task.task_id, task) for task in self._tasks])[task_id]
def _dispatch_tasks(self):
"""
Publish the Tasks to the RabbitMQ
"""
self.timeout_handler.start_queue_timeout()
for task in self._tasks:
log_msg = "Sending command '%s' with key '%s'" \
% (task.command, self._routing_key)
LOGGER.debug(log_msg)
#Send task in queue event with task id
send_monitor_event(MonitorType.TASK_INQUEUE, __name__,
task.task_id)
cmd_msg = CommandMessage(
task.command,
self._testrun_queue,
task.task_id,
timeout=self._execution_timeout,
xml_file=task.xml_file,
min_worker_version=self._min_worker_version)
message = pack_message(cmd_msg)
self._channel.basic_publish(message,
exchange=self._services_exchange,
routing_key=self._routing_key)
def _wait_for_all_tasks(self):
"""
Block until all Tasks are complete
"""
while 1:
try:
self._channel.wait()
except socket.error, error:
# interrupted system call exception need to be ignored so that
# testruns don't fail on apache graceful restart
if error[0] == errno.EINTR:
LOGGER.debug("Interrupted system call. Ignoring...")
else:
raise
if len(self._tasks) == 0:
break
def __init__(self,
username,
password,
host,
vhost,
services_exchange,
port,
routing_key,
testrun_id,
execution_timeout,
queue_timeout,
controller_timeout,
min_worker_version=None):
"""
@type username: C{str}
@param username: AMQP username
@type password: C{str}
@param password: AMQP password
@type host: C{str}
@param host: AMQP host
@type vhost: C{str}
@param vhost: AMQP vhost
@type services_exchange: C{str}
@param services_exchange: AMQP services exchange
@type port: C{int}
@param port: AMQP port (generally 5672)
@type routing_key: C{routing_key}
@param routing_key: AMQP routing_key (device group)
@type testrun_id: C{int}
@param testrun_id: The testrun id
@type execution_timeout: C{int}
@param execution_timeout: Time out in seconds for Task execution
@type queue_timeout: C{int}
@param queue_timeout: Time in seconds Tasks can wait on the queue
@type controller_timeout: C{int}
@param controller_timeout: The timeout for HW controllers
"""
#AMQP configuration
self._username = username
self._password = password
self._host = host
self._vhost = vhost
self._services_exchange = services_exchange
self._port = port
#
self._routing_key = routing_key
self._testrun_id = testrun_id
#
self._init_amqp()
#List of Tasks
self._tasks = []
self._is_run = False
#
self._min_worker_version = min_worker_version
#timeouts
self._execution_timeout = execution_timeout
self._queue_timeout = queue_timeout
self._controller_timeout = controller_timeout
self.timeout_handler = Timeout(execution_timeout, queue_timeout,
controller_timeout)
