Exemple #1
0
    def test_global_timeout_raised(self):
        def run():  # A dummy timeouting function
            time.sleep(3)

        queue_timeout = 1
        controller_timeout = 1
        execution_timeout = 1
        timeout = Timeout(execution_timeout, queue_timeout, controller_timeout)
        timeout.start_queue_timeout()
        timeout.task_started()
        self.assertRaises(OtsExecutionTimeoutError, run)
Exemple #2
0
 def test_global_timeout_raised(self):
     def run(): # A dummy timeouting function
         time.sleep(3)
     queue_timeout = 1
     controller_timeout = 1
     execution_timeout = 1
     timeout = Timeout(execution_timeout, 
                       queue_timeout, 
                       controller_timeout)
     timeout.start_queue_timeout()
     timeout.task_started()
     self.assertRaises(OtsExecutionTimeoutError, run)
Exemple #3
0
class TaskRunner(object):
    """
    Allows Tasks to be run remotely as a set associated with the Testrun 
    
    Each Task is sent individually to RabbitMQ 
    Wait for all tasks to comeback.

    The TaskRunner can only be run once.

    Results from the tasks are signalled with names 
    associated with MESSAGE_TYPES.
    """

    def __init__(self, username, password, host, vhost, 
                 services_exchange, port, 
                 routing_key, testrun_id, 
                 execution_timeout, queue_timeout, controller_timeout,
                 min_worker_version = None):
        """
        @type username: C{str}
        @param username: AMQP username 

        @type password: C{str}
        @param password: AMQP password 

        @type host: C{str}
        @param host: AMQP host 

        @type vhost: C{str}
        @param vhost: AMQP vhost

        @type services_exchange: C{str}
        @param services_exchange: AMQP services exchange

        @type port: C{int}
        @param port: AMQP port (generally 5672)

        @type routing_key: C{routing_key}
        @param routing_key: AMQP routing_key (device group) 
 
        @type testrun_id: C{int}
        @param testrun_id: The testrun id

        @type execution_timeout: C{int}
        @param execution_timeout: Time out in seconds for Task execution

        @type queue_timeout: C{int}
        @param queue_timeout: Time in seconds Tasks can wait on the queue 

        @type controller_timeout: C{int}
        @param controller_timeout: The timeout for HW controllers         
        """
        #AMQP configuration
        self._username = username
        self._password = password
        self._host = host
        self._vhost = vhost
        self._services_exchange = services_exchange
        self._port = port
        #
        self._routing_key = routing_key
        self._testrun_id = testrun_id
        #
        self._init_amqp()
        #List of Tasks
        self._tasks = []
        self._is_run = False

        #
        self._min_worker_version = min_worker_version

        #timeouts
        self._execution_timeout = execution_timeout
        self._queue_timeout = queue_timeout
        self._controller_timeout = controller_timeout
        self.timeout_handler = Timeout(execution_timeout, 
                                       queue_timeout, 
                                       controller_timeout)


    #############################################
    # MESSAGE HANDLING 
    #############################################
        
    def _on_message(self, amqp_message):
        """
        Handler for AMQP messages   
        
        Two types of messages:
        
            1. Indication of state changes on the Task 
            2. Feedback from the Task itself

        State change messages trigger a state transition.
        Everything else fires a signal

        @type amqp_message: amqplib.client_0_8.basic_message.Message 
        @param amqp_message: AMQP message
        """
        msg = unpack_message(amqp_message)
        if isinstance(msg, StateChangeMessage):
            LOGGER.debug("Received state change message %s, task %s "\
                             % (msg.condition, msg.task_id))
            self._task_transition(msg)
        else:
            #The message is data. Relay using a signal
            # If message is monitor message, make received timestamp
            if isinstance(msg, Monitor):
                msg.set_received()
            DTO_SIGNAL.send(sender = "TaskRunner", dto = msg)
  
    def _task_transition(self, message):
        """
        Processes state change message 
        """
        if message.is_start:
            self.timeout_handler.task_started()
        task = self._get_task(message.task_id)
        task.transition(message.condition)
        if task.is_finished:
            self._tasks.remove(task)
    
    ##########################################
    # OTHER HELPERS 
    ##########################################

    def _init_amqp(self):
        """
        Make the AMQP connection. 
        Prepare a Queue
        Start Consuming 
        """
        # Disabling "Attribute defined outside __init__" because this method is
        # called from the __init__
        #pylint: disable=W0201

        self._testrun_queue = testrun_queue_name(self._testrun_id)
        self._connection = amqp.Connection(host = self._host, 
                                           userid = self._username,
                                           password = self._password,
                                           virtual_host = self._vhost, 
                                           insist = False)
        self._channel = self._connection.channel()
        _init_queue(self._channel, 
                    self._testrun_queue, 
                    self._testrun_queue, 
                    self._testrun_queue)
        self._channel.basic_consume(queue = self._testrun_queue, 
                                    callback = self._on_message,
                                    no_ack = True)


    def _get_task(self, task_id):
        """
        Get the Task for the given task_id

        @type task_id: string
        @param task_id: The ID of the Task

        @rtype: L{Task}  
        @return: A Task
        """
        return dict([(task.task_id, task) for task in self._tasks])[task_id]
        
    def _dispatch_tasks(self):
        """
        Publish the Tasks to the RabbitMQ
        """
        self.timeout_handler.start_queue_timeout()
        for task in self._tasks:
            log_msg = "Sending command '%s' with key '%s'" \
                          % (task.command, self._routing_key)
            LOGGER.debug(log_msg)
            
            #Send task in queue event with task id
            send_monitor_event(MonitorType.TASK_INQUEUE,
                               __name__,
                               task.task_id)
            
            cmd_msg = CommandMessage(task.command, 
                                     self._testrun_queue,
                                     task.task_id,
                                     timeout = self._execution_timeout,
                                     xml_file = task.xml_file,
                                     min_worker_version = 
                                       self._min_worker_version)
            message = pack_message(cmd_msg)
            self._channel.basic_publish(message, 
                                        exchange = self._services_exchange,
                                        routing_key = self._routing_key)

    def _wait_for_all_tasks(self):
        """
        Block until all Tasks are complete
        """
        while 1:
            try:
                self._channel.wait()
            except socket.error, error:
                # interrupted system call exception need to be ignored so that
                # testruns don't fail on apache graceful restart
                if error[0] == errno.EINTR:
                    LOGGER.debug("Interrupted system call. Ignoring...")
                else:
                    raise
            if len(self._tasks) == 0:
                break
Exemple #4
0
class TaskRunner(object):
    """
    Allows Tasks to be run remotely as a set associated with the Testrun 
    
    Each Task is sent individually to RabbitMQ 
    Wait for all tasks to comeback.

    The TaskRunner can only be run once.

    Results from the tasks are signalled with names 
    associated with MESSAGE_TYPES.
    """
    def __init__(self,
                 username,
                 password,
                 host,
                 vhost,
                 services_exchange,
                 port,
                 routing_key,
                 testrun_id,
                 execution_timeout,
                 queue_timeout,
                 controller_timeout,
                 min_worker_version=None):
        """
        @type username: C{str}
        @param username: AMQP username 

        @type password: C{str}
        @param password: AMQP password 

        @type host: C{str}
        @param host: AMQP host 

        @type vhost: C{str}
        @param vhost: AMQP vhost

        @type services_exchange: C{str}
        @param services_exchange: AMQP services exchange

        @type port: C{int}
        @param port: AMQP port (generally 5672)

        @type routing_key: C{routing_key}
        @param routing_key: AMQP routing_key (device group) 
 
        @type testrun_id: C{int}
        @param testrun_id: The testrun id

        @type execution_timeout: C{int}
        @param execution_timeout: Time out in seconds for Task execution

        @type queue_timeout: C{int}
        @param queue_timeout: Time in seconds Tasks can wait on the queue 

        @type controller_timeout: C{int}
        @param controller_timeout: The timeout for HW controllers         
        """
        #AMQP configuration
        self._username = username
        self._password = password
        self._host = host
        self._vhost = vhost
        self._services_exchange = services_exchange
        self._port = port
        #
        self._routing_key = routing_key
        self._testrun_id = testrun_id
        #
        self._init_amqp()
        #List of Tasks
        self._tasks = []
        self._is_run = False

        #
        self._min_worker_version = min_worker_version

        #timeouts
        self._execution_timeout = execution_timeout
        self._queue_timeout = queue_timeout
        self._controller_timeout = controller_timeout
        self.timeout_handler = Timeout(execution_timeout, queue_timeout,
                                       controller_timeout)

    #############################################
    # MESSAGE HANDLING
    #############################################

    def _on_message(self, amqp_message):
        """
        Handler for AMQP messages   
        
        Two types of messages:
        
            1. Indication of state changes on the Task 
            2. Feedback from the Task itself

        State change messages trigger a state transition.
        Everything else fires a signal

        @type amqp_message: amqplib.client_0_8.basic_message.Message 
        @param amqp_message: AMQP message
        """
        msg = unpack_message(amqp_message)
        if isinstance(msg, StateChangeMessage):
            LOGGER.debug("Received state change message %s, task %s "\
                             % (msg.condition, msg.task_id))
            self._task_transition(msg)
        else:
            #The message is data. Relay using a signal
            # If message is monitor message, make received timestamp
            if isinstance(msg, Monitor):
                msg.set_received()
            DTO_SIGNAL.send(sender="TaskRunner", dto=msg)

    def _task_transition(self, message):
        """
        Processes state change message 
        """
        if message.is_start:
            self.timeout_handler.task_started()
        task = self._get_task(message.task_id)
        task.transition(message.condition)
        if task.is_finished:
            self._tasks.remove(task)

    ##########################################
    # OTHER HELPERS
    ##########################################

    def _init_amqp(self):
        """
        Make the AMQP connection. 
        Prepare a Queue
        Start Consuming 
        """
        # Disabling "Attribute defined outside __init__" because this method is
        # called from the __init__
        #pylint: disable=W0201

        self._testrun_queue = testrun_queue_name(self._testrun_id)
        self._connection = amqp.Connection(host=self._host,
                                           userid=self._username,
                                           password=self._password,
                                           virtual_host=self._vhost,
                                           insist=False)
        self._channel = self._connection.channel()
        _init_queue(self._channel, self._testrun_queue, self._testrun_queue,
                    self._testrun_queue)
        self._channel.basic_consume(queue=self._testrun_queue,
                                    callback=self._on_message,
                                    no_ack=True)

    def _get_task(self, task_id):
        """
        Get the Task for the given task_id

        @type task_id: string
        @param task_id: The ID of the Task

        @rtype: L{Task}  
        @return: A Task
        """
        return dict([(task.task_id, task) for task in self._tasks])[task_id]

    def _dispatch_tasks(self):
        """
        Publish the Tasks to the RabbitMQ
        """
        self.timeout_handler.start_queue_timeout()
        for task in self._tasks:
            log_msg = "Sending command '%s' with key '%s'" \
                          % (task.command, self._routing_key)
            LOGGER.debug(log_msg)

            #Send task in queue event with task id
            send_monitor_event(MonitorType.TASK_INQUEUE, __name__,
                               task.task_id)

            cmd_msg = CommandMessage(
                task.command,
                self._testrun_queue,
                task.task_id,
                timeout=self._execution_timeout,
                xml_file=task.xml_file,
                min_worker_version=self._min_worker_version)
            message = pack_message(cmd_msg)
            self._channel.basic_publish(message,
                                        exchange=self._services_exchange,
                                        routing_key=self._routing_key)

    def _wait_for_all_tasks(self):
        """
        Block until all Tasks are complete
        """
        while 1:
            try:
                self._channel.wait()
            except socket.error, error:
                # interrupted system call exception need to be ignored so that
                # testruns don't fail on apache graceful restart
                if error[0] == errno.EINTR:
                    LOGGER.debug("Interrupted system call. Ignoring...")
                else:
                    raise
            if len(self._tasks) == 0:
                break