Пример #1
0
    def start_command(self, schedule=False):
        """Initialize progress bars and variables for this command execution.

        Executed at the start of each command.

        Arguments:
            schedule (bool, optional): whether the next command should be sent to ClusterShell for execution or not.

        """
        self.counters['success'] = 0

        self.progress.init(self.counters['total'])

        # Schedule the next command, the first was already scheduled by ClusterShellWorker.execute()
        if schedule:
            with self.lock:  # Avoid modifications of the same data from other callbacks triggered by ClusterShell
                # Available nodes for the next command execution were already update back to the pending state
                remaining_nodes = [node.name for node in self.nodes.values() if node.state.is_pending]
                first_batch = remaining_nodes[:self.target.batch_size]
                first_batch_set = nodeset_fromlist(first_batch)
                for node_name in first_batch:
                    self.nodes[node_name].state.update(State.scheduled)

            command = self.commands[self.current_command_index]
            self.logger.debug(
                "command='%s', timeout=%s, first_batch=%s", command.command, command.timeout, first_batch_set)

            # Schedule the command for execution in ClusterShell
            Task.task_self().flush_buffers()
            Task.task_self().shell(command.command, nodes=first_batch_set, handler=self, timeout=command.timeout)
Пример #2
0
    def ev_timer(self, timer):
        """Schedule the current command on the next node or the next command on the first batch of nodes.

        This callback is triggered by `ClusterShell` when a scheduled `Task.timer()` goes off.

        :Parameters:
            according to parent :py:meth:`ClusterShell.Event.EventHandler.ev_timer`.
        """
        success_ratio = 1 - ((self.counters['failed'] + self.counters['timeout']) / self.counters['total'])

        node = None
        if success_ratio >= self.success_threshold:
            # Success ratio is still good, looking for the next node
            with self.lock:  # Avoid modifications of the same data from other callbacks triggered by ClusterShell
                for new_node in self.nodes.values():
                    if new_node.state.is_pending:
                        # Found the next node where to execute all the commands
                        node = new_node
                        node.state.update(State.scheduled)
                        break

        if node is not None:
            # Schedule the exeuction of the first command to the next node with ClusterShell
            command = node.commands[0]
            self.logger.debug("next_node=%s, timeout=%s, command='%s'", node.name, command.command, command.timeout)
            Task.task_self().shell(
                command.command, handler=timer.eh, timeout=command.timeout, nodes=nodeset(node.name))
        else:
            self.logger.debug('No more nodes left')
Пример #3
0
    def testThreadTaskWaitWhenRunning(self):
        """test task_wait() when workers are running"""

        for i in range(1, 5):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        task_wait()
Пример #4
0
    def testThreadTaskWaitWhenSomeFinished(self):
        """test task_wait() when some workers finished"""

        for i in range(1, 5):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        time.sleep(2)
        task_wait()
Пример #5
0
    def testThreadTaskWaitWhenAllFinished(self):
        """test task_wait() when all workers finished"""

        for i in range(1, 3):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        time.sleep(4)
        task_wait()
Пример #6
0
    def end_command(self):
        """Command terminated, print the result and schedule the next command if criteria are met.

        Executed at the end of each command inside a lock.

        Returns:
            bool: :py:data:`True` if the next command should be scheduled, :py:data:`False` otherwise.

        """
        self._commands_output_report(Task.task_self(), command=self.commands[self.current_command_index].command)

        self.progress.close()

        self._failed_commands_report(filter_command_index=self.current_command_index)
        self._success_nodes_report(command=self.commands[self.current_command_index].command)

        success_ratio = self.counters['success'] / self.counters['total']

        # Abort on failure
        if success_ratio < self.success_threshold:
            self.return_value = 2
            self.aborted = True  # Tells other timers that might trigger after that the abort is already in progress
            return False

        if success_ratio == 1:
            self.return_value = 0
        else:
            self.return_value = 1

        if self.current_command_index == (len(self.commands) - 1):
            self.logger.debug('This was the last command')
            return False  # This was the last command

        return True
Пример #7
0
    def testThreadTaskUnhandledException(self):
        """test task unhandled exception in thread"""
        class TestUnhandledException(Exception):
            """test exception"""
        class RaiseOnRead(EventHandler):
            def ev_read(self, worker):
                raise TestUnhandledException("you should see this exception")

        task = Task()
        # test data access from main thread
        task.shell("echo raisefoobar", key=1, handler=RaiseOnRead())
        task.resume()
        task.join()
        self.assertEqual(task.key_buffer(1), b"raisefoobar")
        time.sleep(1) # for pretty display, because unhandled exception
                      # traceback may be sent to stderr after the join()
        self.assertFalse(task.running())
    def testTaskInNewThread2(self):
        """test task in new thread 2"""
        # create a task in a new thread
        task = Task()
        self.assert_(task != None)

        match = "again"

        # schedule a command in that task
        worker = task.shell("/bin/echo %s" % match)

        # run this task
        task.resume()

        # wait for the task to complete
        task_wait()

        # verify that the worker has completed
        self.assertEqual(worker.read(), match)
Пример #9
0
    def testTaskInNewThread2(self):
        """test task in new thread 2"""
        # create a task in a new thread
        task = Task()
        self.assert_(task != None)

        match = "again"

        # schedule a command in that task
        worker = task.shell("/bin/echo %s" % match)

        # run this task
        task.resume()

        # wait for the task to complete
        task_wait()

        # verify that the worker has completed
        self.assertEqual(worker.read(), match)
Пример #10
0
    def testThreadTaskUnhandledException(self):
        """test task unhandled exception in thread"""
        class TestUnhandledException(Exception):
            """test exception"""
        class RaiseOnRead(EventHandler):
            def ev_read(self, worker):
                raise TestUnhandledException("you should see this exception")

        task = Task()
        # test data access from main thread
        task.shell("echo raisefoobar", key=1, handler=RaiseOnRead())
        task.resume()
        task.join()
        self.assertEqual(task.key_buffer(1), "raisefoobar")
        time.sleep(1) # for pretty display, because unhandled exception
                      # traceback may be sent to stderr after the join()
        self.assertFalse(task.running())
Пример #11
0
    def testPortRemove(self):
        """test remove_port()"""
        class PortHandler(EventHandler):
            def ev_msg(self, port, msg):
                pass

        task = Task()  # new thread
        port = task.port(handler=PortHandler(), autoclose=True)
        task.resume()
        task.remove_port(port)
        task_wait()
Пример #12
0
    def testPortMsg1(self):
        """test port msg from main thread to task"""

        TaskPortTest.got_msg = False

        # create task in new thread
        task = Task()

        class PortHandler(EventHandler):
            def ev_msg(self, port, msg):
                # receive msg
                assert msg == "toto"
                assert port.task.thread == threading.currentThread()
                TaskPortTest.got_msg = True
                port.task.abort()

        # create non-autoclosing port
        port = task.port(handler=PortHandler())
        task.resume()
        # send msg from main thread
        port.msg("toto")
        task_wait()
        self.assertTrue(TaskPortTest.got_msg)
Пример #13
0
    def testPortMsg1(self):
        """test port msg from main thread to task"""
        
        TaskPortTest.got_msg = False

        # create task in new thread
        task = Task()

        class PortHandler(EventHandler):
            def ev_msg(self, port, msg):
                # receive msg
                assert msg == "toto"
                assert port.task.thread == threading.currentThread()
                TaskPortTest.got_msg = True
                port.task.abort()

        # create non-autoclosing port
        port = task.port(handler=PortHandler())
        task.resume()
        # send msg from main thread
        port.msg("toto")
        task_wait()
        self.assert_(TaskPortTest.got_msg)
Пример #14
0
    def testThreadTaskWaitWhenRunning(self):
        """test task_wait() when workers are running"""

        for i in range(1, 5):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        task_wait()
Пример #15
0
    def testPortRemove(self):
        """test port remove [private as of 1.2]"""

        task = Task()

        class PortHandler(EventHandler):
            def ev_msg(self, port, msg):
                pass

        port = task.port(handler=PortHandler(), autoclose=True)
        task.resume()
        task._remove_port(port)
        task_wait()
Пример #16
0
    def testThreadTaskWaitWhenAllFinished(self):
        """test task_wait() when all workers finished"""

        for i in range(1, 3):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        time.sleep(4)
        task_wait()
Пример #17
0
    def testThreadTaskWaitWhenSomeFinished(self):
        """test task_wait() when some workers finished"""

        for i in range(1, 5):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        time.sleep(2)
        task_wait()
Пример #18
0
    def __init__(self, config, target):
        """Worker ClusterShell constructor.

        :Parameters:
            according to parent :py:meth:`cumin.transports.BaseWorker.__init__`.
        """
        super().__init__(config, target)
        self.task = Task.task_self()  # Initialize a ClusterShell task
        self._handler_instance = None

        # Set any ClusterShell task options
        for key, value in config.get('clustershell', {}).items():
            if isinstance(value, list):
                self.task.set_info(key, ' '.join(value))
            else:
                self.task.set_info(key, value)
Пример #19
0
    def testPortRemove(self):
        """test remove_port()"""

        class PortHandler(EventHandler):
            def ev_msg(self, port, msg):
                pass

        task = Task() # new thread
        port = task.port(handler=PortHandler(), autoclose=True)
        task.resume()
        task.remove_port(port)
        task_wait()
Пример #20
0
 def tasks_run(self, arg_tasks, action):
     """
     Prepare and run tasks
     """
     print "TASKS RUN =>", action
     workers = []
     task = Task.task_self()
     for script in arg_tasks:
         groupedNodes = Node.Node.group_by_manager(arg_tasks[script])
         for manager in groupedNodes:
             nodesList = ",".join([node.name for node in groupedNodes[manager]])
             command = managers.get_command(manager=manager, service=script, action=action)
             print "Task run: " + command + ", nodes: " + nodesList
             worker = task.shell(command, nodes=nodesList)
             workers.append((worker, script))
     task.run()
     task.join()
     return workers
Пример #21
0
    def testPortRemove(self):
        """test port remove [private as of 1.2]"""
        
        task = Task()

        class PortHandler(EventHandler):
            def ev_msg(self, port, msg):
                pass

        port = task.port(handler=PortHandler(), autoclose=True)
        task.resume()
        task._remove_port(port)
        task_wait()
Пример #22
0
    def testTaskInNewThread2(self):
        # create a task in a new thread
        task = Task()

        match = "again"

        # schedule a command in that task
        worker = task.shell("/bin/echo %s" % match)

        # run this task
        task.resume()

        # wait for the task to complete
        task_wait()

        # verify that the worker has completed
        self.assertEqual(worker.read(), match.encode('ascii'))

        # stop task
        task.abort()
Пример #23
0
    def testTaskInNewThread3(self):
        # create a task in a new thread
        task = Task()

        match = "once again"

        # schedule a command in that task
        worker = task.shell("/bin/echo %s" % match)

        # run this task
        task.resume()

        # wait for the task to complete
        task_wait()

        # verify that the worker has completed
        self.assertEqual(worker.read(), match.encode('ascii'))

        # stop task
        task.abort()
    def testSuspendMiscTwoTasks(self):
        """test task suspend/resume (2 tasks)"""
        task = task_self()
        task2 = Task()

        task2.shell("sleep 4 && echo thr1")
        task2.resume()
        w = task.shell("sleep 1 && echo thr0", key=0)
        task.resume()
        self.assertEqual(task.key_buffer(0), "thr0")
        self.assertEqual(w.read(), "thr0")

        assert task2 != task
        task2.suspend()
        time.sleep(10)
        task2.resume()

        task_wait()

        task2.shell("echo suspend_test", key=1)
        task2.resume()

        task_wait()
        self.assertEqual(task2.key_buffer(1), "suspend_test")
Пример #25
0
    def ev_timer(self, timer):  # noqa, mccabe: MC0001 too complex (15) FIXME
        """Schedule the current command on the next node or the next command on the first batch of nodes.

        This callback is triggered by `ClusterShell` when a scheduled `Task.timer()` goes off.

        :Parameters:
            according to parent :py:meth:`ClusterShell.Event.EventHandler.ev_timer`.
        """
        success_ratio = 1 - ((self.counters['failed'] + self.counters['timeout']) / self.counters['total'])

        node = None
        if success_ratio >= self.success_threshold:
            # Success ratio is still good, looking for the next node
            with self.lock:  # Avoid modifications of the same data from other callbacks triggered by ClusterShell
                for new_node in self.nodes.values():
                    if new_node.state.is_pending:
                        # Found the next node where to execute the command
                        node = new_node
                        node.state.update(State.scheduled)
                        break

        if node is not None:
            # Schedule the execution with ClusterShell of the current command to the next node found above
            command = self.nodes[node.name].commands[self.nodes[node.name].running_command_index + 1]
            self.logger.debug("next_node=%s, timeout=%s, command='%s'", node.name, command.command, command.timeout)
            Task.task_self().shell(command.command, handler=timer.eh, timeout=command.timeout, nodes=nodeset(node.name))
            return

        # No more nodes were left for the execution of the current command
        with self.lock:  # Avoid modifications of the same data from other callbacks triggered by ClusterShell
            try:
                command = self.commands[self.current_command_index].command
            except IndexError:
                command = None  # Last command reached

            # Get a list of the nodes still in pending state
            pending = [pending_node.name for pending_node in self.nodes.values() if pending_node.state.is_pending]
            # Nodes in running are still running the command and nodes in scheduled state will execute the command
            # anyway, they were already offloaded to ClusterShell
            accounted = len(pending) + self.counters['failed'] + self.counters['success'] + self.counters['timeout']

            # Avoid race conditions
            if self.aborted or accounted != self.counters['total'] or command is None or self.global_timedout:
                self.logger.debug("Skipped timer")
                return

            if pending:
                # This usually happens when executing in batches
                self.logger.warning("Command '%s' was not executed on: %s", command, nodeset_fromlist(pending))

            self.logger.info("Completed command '%s'", command)
            restart = self.end_command()
            self.current_command_index += 1  # Move the global pointer of the command in execution

            if restart:
                for node in self.nodes.values():
                    if node.state.is_success:
                        # Only nodes in pending state will be scheduled for the next command
                        node.state.update(State.pending)

        if restart:
            self.start_command(schedule=True)
Пример #26
0
    def testThreadTaskWaitWithSuspend(self):
        """test task_wait() with suspended tasks"""
        task = Task()
        self.resumed = False
        threading.Thread(None,
                         self._thread_delayed_unsuspend_func,
                         args=(task, )).start()
        time_sh = int(random.random() * 4)
        #print "TIME shell=%d" % time_sh
        task.shell("sleep %d" % time_sh)
        task.resume()
        time.sleep(1)
        suspended = task.suspend()

        for i in range(1, 4):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        time.sleep(1)
        task_wait()
        self.assertTrue(self.resumed or suspended == False)
Пример #27
0
 def testThreadSimpleTaskSupervisor(self):
     """test task methods from another thread"""
     #print "PASS 1"
     task = Task()
     task.shell("sleep 3")
     task.shell("echo testing", key=1)
     task.resume()
     task.join()
     self.assertEqual(task.key_buffer(1), "testing")
     #print "PASS 2"
     task.shell("echo ok", key=2)
     task.resume()
     task.join()
     #print "PASS 3"
     self.assertEqual(task.key_buffer(2), "ok")
     task.shell("sleep 1 && echo done", key=3)
     task.resume()
     task.join()
     #print "PASS 4"
     self.assertEqual(task.key_buffer(3), "done")
     task.abort()
    def testThreadTaskWaitWithSuspend(self):
        """test task_wait() with suspended tasks"""
        task = Task()
        self.resumed = False
        thread.start_new_thread(TaskThreadSuspendTest._thread_delayed_unsuspend_func, (self, task))
        time_sh = int(random.random()*4)
        #print "TIME shell=%d" % time_sh
        task.shell("sleep %d" % time_sh)
        task.resume()
        time.sleep(1)
        suspended = task.suspend()

        for i in range(1, 4):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        time.sleep(1)
        task_wait()
        self.assert_(self.resumed or suspended == False)
    def testSuspendMiscTwoTasks(self):
        """test task suspend/resume (2 tasks)"""
        task = task_self()
        task2 = Task()

        task2.shell("sleep 4 && echo thr1")
        task2.resume()
        w = task.shell("sleep 1 && echo thr0", key=0)
        task.resume()
        self.assertEqual(task.key_buffer(0), "thr0")
        self.assertEqual(w.read(), "thr0")

        assert task2 != task
        task2.suspend()
        time.sleep(10)
        task2.resume()

        task_wait()

        task2.shell("echo suspend_test", key=1)
        task2.resume()

        task_wait()
        self.assertEqual(task2.key_buffer(1), "suspend_test")
Пример #30
0
    def testThreadTaskBuffers(self):
        """test task data access methods after join()"""
        task = Task()
        # test data access from main thread

        # test stderr separated
        task.set_default("stderr", True)
        task.shell("echo foobar", key="OUT")
        task.shell("echo raboof 1>&2", key="ERR")
        task.resume()
        task.join()
        self.assertEqual(task.key_buffer("OUT"), "foobar")
        self.assertEqual(task.key_error("OUT"), "")
        self.assertEqual(task.key_buffer("ERR"), "")
        self.assertEqual(task.key_error("ERR"), "raboof")

        # test stderr merged
        task.set_default("stderr", False)
        task.shell("echo foobar", key="OUT")
        task.shell("echo raboof 1>&2", key="ERR")
        task.resume()
        task.join()
        self.assertEqual(task.key_buffer("OUT"), "foobar")
        self.assertEqual(task.key_error("OUT"), "")
        self.assertEqual(task.key_buffer("ERR"), "raboof")
        self.assertEqual(task.key_error("ERR"), "")
Пример #31
0
    def testThreadTaskWaitWithSuspend(self):
        """test task_wait() with suspended tasks"""
        task = Task()
        self.resumed = False
        threading.Thread(None, self._thread_delayed_unsuspend_func,
                         args=(task,)).start()
        time_sh = int(random.random()*4)
        #print "TIME shell=%d" % time_sh
        task.shell("sleep %d" % time_sh)
        task.resume()
        time.sleep(1)
        suspended = task.suspend()

        for i in range(1, 4):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        time.sleep(1)
        task_wait()
        self.assertTrue(self.resumed or suspended == False)
Пример #32
0
 def testThreadSimpleTaskSupervisor(self):
     """test task methods from another thread"""
     #print "PASS 1"
     task = Task()
     task.shell("sleep 3")
     task.shell("echo testing", key=1)
     task.resume()
     task.join()
     self.assertEqual(task.key_buffer(1), b"testing")
     #print "PASS 2"
     task.shell("echo ok", key=2)
     task.resume()
     task.join()
     #print "PASS 3"
     self.assertEqual(task.key_buffer(2), b"ok")
     task.shell("sleep 1 && echo done", key=3)
     task.resume()
     task.join()
     #print "PASS 4"
     self.assertEqual(task.key_buffer(3), b"done")
     task.abort()
Пример #33
0
    def testThreadTaskBuffers(self):
        """test task data access methods after join()"""
        task = Task()
        # test data access from main thread

        # test stderr separated
        task.set_default("stderr", True)
        task.shell("echo foobar", key="OUT")
        task.shell("echo raboof 1>&2", key="ERR")
        task.resume()
        task.join()
        self.assertEqual(task.key_buffer("OUT"), b"foobar")
        self.assertEqual(task.key_error("OUT"), b"")
        self.assertEqual(task.key_buffer("ERR"), b"")
        self.assertEqual(task.key_error("ERR"), b"raboof")

        # test stderr merged
        task.set_default("stderr", False)
        task.shell("echo foobar", key="OUT")
        task.shell("echo raboof 1>&2", key="ERR")
        task.resume()
        task.join()
        self.assertEqual(task.key_buffer("OUT"), b"foobar")
        self.assertEqual(task.key_error("OUT"), b"")
        self.assertEqual(task.key_buffer("ERR"), b"raboof")
        self.assertEqual(task.key_error("ERR"), b"")
    def testThreadTaskWaitWithSuspend(self):
        """test task_wait() with suspended tasks"""
        task = Task()
        self.resumed = False
        thread.start_new_thread(
            TaskThreadSuspendTest._thread_delayed_unsuspend_func, (self, task))
        time_sh = int(random.random() * 4)
        #print "TIME shell=%d" % time_sh
        task.shell("sleep %d" % time_sh)
        task.resume()
        time.sleep(1)
        suspended = task.suspend()

        for i in range(1, 4):
            task = Task()
            task.shell("sleep %d" % i)
            task.resume()

        time.sleep(1)
        task_wait()
        self.assert_(self.resumed or suspended == False)