def test_queue_timeout_raised(self): def run(): # A dummy timeouting function time.sleep(3) queue_timeout = 1 execution_timeout = 1 controller_timeout = 1 timeout = Timeout(execution_timeout, queue_timeout, controller_timeout) timeout.start_queue_timeout() self.assertRaises(OtsQueueTimeoutError, run)
def test_queue_timeout_not_raised(self): self.done = False def run(): # A dummy not timeouting function self.done = True queue_timeout = 1 execution_timeout = 1 controller_timeout = 1 timeout = Timeout(execution_timeout, queue_timeout, controller_timeout) timeout.start_queue_timeout() run() self.assertTrue(self.done)
def test_stop(self): self.done = False def run(): # A dummy timeouting function time.sleep(2) self.done = True queue_timeout = 1 controller_timeout = 1 execution_timeout = 1 timeout = Timeout(execution_timeout, queue_timeout, controller_timeout) timeout.start_queue_timeout() timeout.stop() run() self.assertTrue(self.done)
def test_worker_alive_after_server_timeout_failing_task(self): if not DEBUG: self.worker_processes.start() self.testrun_id = 111 self.testrun_id2 = 112 taskrunner1 = taskrunner_factory( routing_key=ROUTING_KEY, execution_timeout=10, testrun_id=self.testrun_id, config_file=_distributor_config_filename()) command = [ "sleep", "5", ";", "command_error_mock", "localhost", str(self.testrun_id) ] taskrunner1.add_task(command) self.is_exception_raised = False def cb_handler(signal, dto, **kwargs): if isinstance(dto, Exception): self.is_exception_raised = True DTO_SIGNAL.connect(cb_handler) # Overwrite server side timeout handler with a one that timeouts from ots.server.distributor.timeout import Timeout taskrunner1.timeout_handler = Timeout(1, 1, 1) self.assertRaises(OtsExecutionTimeoutError, taskrunner1.run) # self.assertTrue(self.is_exception_raised) self.is_exception_raised = False time.sleep(10) # Give worker time to reconnect # Trigger another task to make sure worker is still alive taskrunner2 = taskrunner_factory( routing_key=ROUTING_KEY, execution_timeout=10, testrun_id=self.testrun_id2, config_file=_distributor_config_filename()) taskrunner2.add_task(["echo", "foo"]) taskrunner2.run() self.assertFalse(self.is_exception_raised) self.send_quit() time.sleep(1) self.assertFalse(all(self.worker_processes.exitcodes))
class TaskRunner(object): """ Allows Tasks to be run remotely as a set associated with the Testrun Each Task is sent individually to RabbitMQ Wait for all tasks to comeback. The TaskRunner can only be run once. Results from the tasks are signalled with names associated with MESSAGE_TYPES. """ def __init__(self, username, password, host, vhost, services_exchange, port, routing_key, testrun_id, execution_timeout, queue_timeout, controller_timeout, min_worker_version = None): """ @type username: C{str} @param username: AMQP username @type password: C{str} @param password: AMQP password @type host: C{str} @param host: AMQP host @type vhost: C{str} @param vhost: AMQP vhost @type services_exchange: C{str} @param services_exchange: AMQP services exchange @type port: C{int} @param port: AMQP port (generally 5672) @type routing_key: C{routing_key} @param routing_key: AMQP routing_key (device group) @type testrun_id: C{int} @param testrun_id: The testrun id @type execution_timeout: C{int} @param execution_timeout: Time out in seconds for Task execution @type queue_timeout: C{int} @param queue_timeout: Time in seconds Tasks can wait on the queue @type controller_timeout: C{int} @param controller_timeout: The timeout for HW controllers """ #AMQP configuration self._username = username self._password = password self._host = host self._vhost = vhost self._services_exchange = services_exchange self._port = port # self._routing_key = routing_key self._testrun_id = testrun_id # self._init_amqp() #List of Tasks self._tasks = [] self._is_run = False # self._min_worker_version = min_worker_version #timeouts self._execution_timeout = execution_timeout self._queue_timeout = queue_timeout self._controller_timeout = controller_timeout self.timeout_handler = Timeout(execution_timeout, queue_timeout, controller_timeout) ############################################# # MESSAGE HANDLING ############################################# def _on_message(self, amqp_message): """ Handler for AMQP messages Two types of messages: 1. Indication of state changes on the Task 2. Feedback from the Task itself State change messages trigger a state transition. Everything else fires a signal @type amqp_message: amqplib.client_0_8.basic_message.Message @param amqp_message: AMQP message """ msg = unpack_message(amqp_message) if isinstance(msg, StateChangeMessage): LOGGER.debug("Received state change message %s, task %s "\ % (msg.condition, msg.task_id)) self._task_transition(msg) else: #The message is data. Relay using a signal # If message is monitor message, make received timestamp if isinstance(msg, Monitor): msg.set_received() DTO_SIGNAL.send(sender = "TaskRunner", dto = msg) def _task_transition(self, message): """ Processes state change message """ if message.is_start: self.timeout_handler.task_started() task = self._get_task(message.task_id) task.transition(message.condition) if task.is_finished: self._tasks.remove(task) ########################################## # OTHER HELPERS ########################################## def _init_amqp(self): """ Make the AMQP connection. Prepare a Queue Start Consuming """ # Disabling "Attribute defined outside __init__" because this method is # called from the __init__ #pylint: disable=W0201 self._testrun_queue = testrun_queue_name(self._testrun_id) self._connection = amqp.Connection(host = self._host, userid = self._username, password = self._password, virtual_host = self._vhost, insist = False) self._channel = self._connection.channel() _init_queue(self._channel, self._testrun_queue, self._testrun_queue, self._testrun_queue) self._channel.basic_consume(queue = self._testrun_queue, callback = self._on_message, no_ack = True) def _get_task(self, task_id): """ Get the Task for the given task_id @type task_id: string @param task_id: The ID of the Task @rtype: L{Task} @return: A Task """ return dict([(task.task_id, task) for task in self._tasks])[task_id] def _dispatch_tasks(self): """ Publish the Tasks to the RabbitMQ """ self.timeout_handler.start_queue_timeout() for task in self._tasks: log_msg = "Sending command '%s' with key '%s'" \ % (task.command, self._routing_key) LOGGER.debug(log_msg) #Send task in queue event with task id send_monitor_event(MonitorType.TASK_INQUEUE, __name__, task.task_id) cmd_msg = CommandMessage(task.command, self._testrun_queue, task.task_id, timeout = self._execution_timeout, xml_file = task.xml_file, min_worker_version = self._min_worker_version) message = pack_message(cmd_msg) self._channel.basic_publish(message, exchange = self._services_exchange, routing_key = self._routing_key) def _wait_for_all_tasks(self): """ Block until all Tasks are complete """ while 1: try: self._channel.wait() except socket.error, error: # interrupted system call exception need to be ignored so that # testruns don't fail on apache graceful restart if error[0] == errno.EINTR: LOGGER.debug("Interrupted system call. Ignoring...") else: raise if len(self._tasks) == 0: break
def __init__(self, username, password, host, vhost, services_exchange, port, routing_key, testrun_id, execution_timeout, queue_timeout, controller_timeout, min_worker_version = None): """ @type username: C{str} @param username: AMQP username @type password: C{str} @param password: AMQP password @type host: C{str} @param host: AMQP host @type vhost: C{str} @param vhost: AMQP vhost @type services_exchange: C{str} @param services_exchange: AMQP services exchange @type port: C{int} @param port: AMQP port (generally 5672) @type routing_key: C{routing_key} @param routing_key: AMQP routing_key (device group) @type testrun_id: C{int} @param testrun_id: The testrun id @type execution_timeout: C{int} @param execution_timeout: Time out in seconds for Task execution @type queue_timeout: C{int} @param queue_timeout: Time in seconds Tasks can wait on the queue @type controller_timeout: C{int} @param controller_timeout: The timeout for HW controllers """ #AMQP configuration self._username = username self._password = password self._host = host self._vhost = vhost self._services_exchange = services_exchange self._port = port # self._routing_key = routing_key self._testrun_id = testrun_id # self._init_amqp() #List of Tasks self._tasks = [] self._is_run = False # self._min_worker_version = min_worker_version #timeouts self._execution_timeout = execution_timeout self._queue_timeout = queue_timeout self._controller_timeout = controller_timeout self.timeout_handler = Timeout(execution_timeout, queue_timeout, controller_timeout)
class TaskRunner(object): """ Allows Tasks to be run remotely as a set associated with the Testrun Each Task is sent individually to RabbitMQ Wait for all tasks to comeback. The TaskRunner can only be run once. Results from the tasks are signalled with names associated with MESSAGE_TYPES. """ def __init__(self, username, password, host, vhost, services_exchange, port, routing_key, testrun_id, execution_timeout, queue_timeout, controller_timeout, min_worker_version=None): """ @type username: C{str} @param username: AMQP username @type password: C{str} @param password: AMQP password @type host: C{str} @param host: AMQP host @type vhost: C{str} @param vhost: AMQP vhost @type services_exchange: C{str} @param services_exchange: AMQP services exchange @type port: C{int} @param port: AMQP port (generally 5672) @type routing_key: C{routing_key} @param routing_key: AMQP routing_key (device group) @type testrun_id: C{int} @param testrun_id: The testrun id @type execution_timeout: C{int} @param execution_timeout: Time out in seconds for Task execution @type queue_timeout: C{int} @param queue_timeout: Time in seconds Tasks can wait on the queue @type controller_timeout: C{int} @param controller_timeout: The timeout for HW controllers """ #AMQP configuration self._username = username self._password = password self._host = host self._vhost = vhost self._services_exchange = services_exchange self._port = port # self._routing_key = routing_key self._testrun_id = testrun_id # self._init_amqp() #List of Tasks self._tasks = [] self._is_run = False # self._min_worker_version = min_worker_version #timeouts self._execution_timeout = execution_timeout self._queue_timeout = queue_timeout self._controller_timeout = controller_timeout self.timeout_handler = Timeout(execution_timeout, queue_timeout, controller_timeout) ############################################# # MESSAGE HANDLING ############################################# def _on_message(self, amqp_message): """ Handler for AMQP messages Two types of messages: 1. Indication of state changes on the Task 2. Feedback from the Task itself State change messages trigger a state transition. Everything else fires a signal @type amqp_message: amqplib.client_0_8.basic_message.Message @param amqp_message: AMQP message """ msg = unpack_message(amqp_message) if isinstance(msg, StateChangeMessage): LOGGER.debug("Received state change message %s, task %s "\ % (msg.condition, msg.task_id)) self._task_transition(msg) else: #The message is data. Relay using a signal # If message is monitor message, make received timestamp if isinstance(msg, Monitor): msg.set_received() DTO_SIGNAL.send(sender="TaskRunner", dto=msg) def _task_transition(self, message): """ Processes state change message """ if message.is_start: self.timeout_handler.task_started() task = self._get_task(message.task_id) task.transition(message.condition) if task.is_finished: self._tasks.remove(task) ########################################## # OTHER HELPERS ########################################## def _init_amqp(self): """ Make the AMQP connection. Prepare a Queue Start Consuming """ # Disabling "Attribute defined outside __init__" because this method is # called from the __init__ #pylint: disable=W0201 self._testrun_queue = testrun_queue_name(self._testrun_id) self._connection = amqp.Connection(host=self._host, userid=self._username, password=self._password, virtual_host=self._vhost, insist=False) self._channel = self._connection.channel() _init_queue(self._channel, self._testrun_queue, self._testrun_queue, self._testrun_queue) self._channel.basic_consume(queue=self._testrun_queue, callback=self._on_message, no_ack=True) def _get_task(self, task_id): """ Get the Task for the given task_id @type task_id: string @param task_id: The ID of the Task @rtype: L{Task} @return: A Task """ return dict([(task.task_id, task) for task in self._tasks])[task_id] def _dispatch_tasks(self): """ Publish the Tasks to the RabbitMQ """ self.timeout_handler.start_queue_timeout() for task in self._tasks: log_msg = "Sending command '%s' with key '%s'" \ % (task.command, self._routing_key) LOGGER.debug(log_msg) #Send task in queue event with task id send_monitor_event(MonitorType.TASK_INQUEUE, __name__, task.task_id) cmd_msg = CommandMessage( task.command, self._testrun_queue, task.task_id, timeout=self._execution_timeout, xml_file=task.xml_file, min_worker_version=self._min_worker_version) message = pack_message(cmd_msg) self._channel.basic_publish(message, exchange=self._services_exchange, routing_key=self._routing_key) def _wait_for_all_tasks(self): """ Block until all Tasks are complete """ while 1: try: self._channel.wait() except socket.error, error: # interrupted system call exception need to be ignored so that # testruns don't fail on apache graceful restart if error[0] == errno.EINTR: LOGGER.debug("Interrupted system call. Ignoring...") else: raise if len(self._tasks) == 0: break
def __init__(self, username, password, host, vhost, services_exchange, port, routing_key, testrun_id, execution_timeout, queue_timeout, controller_timeout, min_worker_version=None): """ @type username: C{str} @param username: AMQP username @type password: C{str} @param password: AMQP password @type host: C{str} @param host: AMQP host @type vhost: C{str} @param vhost: AMQP vhost @type services_exchange: C{str} @param services_exchange: AMQP services exchange @type port: C{int} @param port: AMQP port (generally 5672) @type routing_key: C{routing_key} @param routing_key: AMQP routing_key (device group) @type testrun_id: C{int} @param testrun_id: The testrun id @type execution_timeout: C{int} @param execution_timeout: Time out in seconds for Task execution @type queue_timeout: C{int} @param queue_timeout: Time in seconds Tasks can wait on the queue @type controller_timeout: C{int} @param controller_timeout: The timeout for HW controllers """ #AMQP configuration self._username = username self._password = password self._host = host self._vhost = vhost self._services_exchange = services_exchange self._port = port # self._routing_key = routing_key self._testrun_id = testrun_id # self._init_amqp() #List of Tasks self._tasks = [] self._is_run = False # self._min_worker_version = min_worker_version #timeouts self._execution_timeout = execution_timeout self._queue_timeout = queue_timeout self._controller_timeout = controller_timeout self.timeout_handler = Timeout(execution_timeout, queue_timeout, controller_timeout)