def start_command(self, schedule=False):
"""Initialize progress bars and variables for this command execution.
Executed at the start of each command.
Arguments:
schedule (bool, optional): whether the next command should be sent to ClusterShell for execution or not.
"""
self.counters['success'] = 0
self.progress.init(self.counters['total'])
# Schedule the next command, the first was already scheduled by ClusterShellWorker.execute()
if schedule:
with self.lock: # Avoid modifications of the same data from other callbacks triggered by ClusterShell
# Available nodes for the next command execution were already update back to the pending state
remaining_nodes = [node.name for node in self.nodes.values() if node.state.is_pending]
first_batch = remaining_nodes[:self.target.batch_size]
first_batch_set = nodeset_fromlist(first_batch)
for node_name in first_batch:
self.nodes[node_name].state.update(State.scheduled)
command = self.commands[self.current_command_index]
self.logger.debug(
"command='%s', timeout=%s, first_batch=%s", command.command, command.timeout, first_batch_set)
# Schedule the command for execution in ClusterShell
Task.task_self().flush_buffers()
Task.task_self().shell(command.command, nodes=first_batch_set, handler=self, timeout=command.timeout)
def ev_timer(self, timer):
"""Schedule the current command on the next node or the next command on the first batch of nodes.
This callback is triggered by `ClusterShell` when a scheduled `Task.timer()` goes off.
:Parameters:
according to parent :py:meth:`ClusterShell.Event.EventHandler.ev_timer`.
"""
success_ratio = 1 - ((self.counters['failed'] + self.counters['timeout']) / self.counters['total'])
node = None
if success_ratio >= self.success_threshold:
# Success ratio is still good, looking for the next node
with self.lock: # Avoid modifications of the same data from other callbacks triggered by ClusterShell
for new_node in self.nodes.values():
if new_node.state.is_pending:
# Found the next node where to execute all the commands
node = new_node
node.state.update(State.scheduled)
break
if node is not None:
# Schedule the exeuction of the first command to the next node with ClusterShell
command = node.commands[0]
self.logger.debug("next_node=%s, timeout=%s, command='%s'", node.name, command.command, command.timeout)
Task.task_self().shell(
command.command, handler=timer.eh, timeout=command.timeout, nodes=nodeset(node.name))
else:
self.logger.debug('No more nodes left')
def testThreadTaskWaitWhenRunning(self):
"""test task_wait() when workers are running"""
for i in range(1, 5):
task = Task()
task.shell("sleep %d" % i)
task.resume()
task_wait()
def testThreadTaskWaitWhenSomeFinished(self):
"""test task_wait() when some workers finished"""
for i in range(1, 5):
task = Task()
task.shell("sleep %d" % i)
task.resume()
time.sleep(2)
task_wait()
def testThreadTaskWaitWhenAllFinished(self):
"""test task_wait() when all workers finished"""
for i in range(1, 3):
task = Task()
task.shell("sleep %d" % i)
task.resume()
time.sleep(4)
task_wait()
def end_command(self):
"""Command terminated, print the result and schedule the next command if criteria are met.
Executed at the end of each command inside a lock.
Returns:
bool: :py:data:`True` if the next command should be scheduled, :py:data:`False` otherwise.
"""
self._commands_output_report(Task.task_self(), command=self.commands[self.current_command_index].command)
self.progress.close()
self._failed_commands_report(filter_command_index=self.current_command_index)
self._success_nodes_report(command=self.commands[self.current_command_index].command)
success_ratio = self.counters['success'] / self.counters['total']
# Abort on failure
if success_ratio < self.success_threshold:
self.return_value = 2
self.aborted = True # Tells other timers that might trigger after that the abort is already in progress
return False
if success_ratio == 1:
self.return_value = 0
else:
self.return_value = 1
if self.current_command_index == (len(self.commands) - 1):
self.logger.debug('This was the last command')
return False # This was the last command
return True
def testThreadTaskUnhandledException(self):
"""test task unhandled exception in thread"""
class TestUnhandledException(Exception):
"""test exception"""
class RaiseOnRead(EventHandler):
def ev_read(self, worker):
raise TestUnhandledException("you should see this exception")
task = Task()
# test data access from main thread
task.shell("echo raisefoobar", key=1, handler=RaiseOnRead())
task.resume()
task.join()
self.assertEqual(task.key_buffer(1), b"raisefoobar")
time.sleep(1) # for pretty display, because unhandled exception
# traceback may be sent to stderr after the join()
self.assertFalse(task.running())
def testTaskInNewThread2(self):
"""test task in new thread 2"""
# create a task in a new thread
task = Task()
self.assert_(task != None)
match = "again"
# schedule a command in that task
worker = task.shell("/bin/echo %s" % match)
# run this task
task.resume()
# wait for the task to complete
task_wait()
# verify that the worker has completed
self.assertEqual(worker.read(), match)
def testTaskInNewThread2(self):
"""test task in new thread 2"""
# create a task in a new thread
task = Task()
self.assert_(task != None)
match = "again"
# schedule a command in that task
worker = task.shell("/bin/echo %s" % match)
# run this task
task.resume()
# wait for the task to complete
task_wait()
# verify that the worker has completed
self.assertEqual(worker.read(), match)
def testThreadTaskUnhandledException(self):
"""test task unhandled exception in thread"""
class TestUnhandledException(Exception):
"""test exception"""
class RaiseOnRead(EventHandler):
def ev_read(self, worker):
raise TestUnhandledException("you should see this exception")
task = Task()
# test data access from main thread
task.shell("echo raisefoobar", key=1, handler=RaiseOnRead())
task.resume()
task.join()
self.assertEqual(task.key_buffer(1), "raisefoobar")
time.sleep(1) # for pretty display, because unhandled exception
# traceback may be sent to stderr after the join()
self.assertFalse(task.running())
def testPortRemove(self):
"""test remove_port()"""
class PortHandler(EventHandler):
def ev_msg(self, port, msg):
pass
task = Task() # new thread
port = task.port(handler=PortHandler(), autoclose=True)
task.resume()
task.remove_port(port)
task_wait()
def testPortMsg1(self):
"""test port msg from main thread to task"""
TaskPortTest.got_msg = False
# create task in new thread
task = Task()
class PortHandler(EventHandler):
def ev_msg(self, port, msg):
# receive msg
assert msg == "toto"
assert port.task.thread == threading.currentThread()
TaskPortTest.got_msg = True
port.task.abort()
# create non-autoclosing port
port = task.port(handler=PortHandler())
task.resume()
# send msg from main thread
port.msg("toto")
task_wait()
self.assertTrue(TaskPortTest.got_msg)
def testPortMsg1(self):
"""test port msg from main thread to task"""
TaskPortTest.got_msg = False
# create task in new thread
task = Task()
class PortHandler(EventHandler):
def ev_msg(self, port, msg):
# receive msg
assert msg == "toto"
assert port.task.thread == threading.currentThread()
TaskPortTest.got_msg = True
port.task.abort()
# create non-autoclosing port
port = task.port(handler=PortHandler())
task.resume()
# send msg from main thread
port.msg("toto")
task_wait()
self.assert_(TaskPortTest.got_msg)
def testThreadTaskWaitWhenRunning(self):
"""test task_wait() when workers are running"""
for i in range(1, 5):
task = Task()
task.shell("sleep %d" % i)
task.resume()
task_wait()
def testPortRemove(self):
"""test port remove [private as of 1.2]"""
task = Task()
class PortHandler(EventHandler):
def ev_msg(self, port, msg):
pass
port = task.port(handler=PortHandler(), autoclose=True)
task.resume()
task._remove_port(port)
task_wait()
def testThreadTaskWaitWhenAllFinished(self):
"""test task_wait() when all workers finished"""
for i in range(1, 3):
task = Task()
task.shell("sleep %d" % i)
task.resume()
time.sleep(4)
task_wait()
def testThreadTaskWaitWhenSomeFinished(self):
"""test task_wait() when some workers finished"""
for i in range(1, 5):
task = Task()
task.shell("sleep %d" % i)
task.resume()
time.sleep(2)
task_wait()
def __init__(self, config, target):
"""Worker ClusterShell constructor.
:Parameters:
according to parent :py:meth:`cumin.transports.BaseWorker.__init__`.
"""
super().__init__(config, target)
self.task = Task.task_self() # Initialize a ClusterShell task
self._handler_instance = None
# Set any ClusterShell task options
for key, value in config.get('clustershell', {}).items():
if isinstance(value, list):
self.task.set_info(key, ' '.join(value))
else:
self.task.set_info(key, value)
def testPortRemove(self):
"""test remove_port()"""
class PortHandler(EventHandler):
def ev_msg(self, port, msg):
pass
task = Task() # new thread
port = task.port(handler=PortHandler(), autoclose=True)
task.resume()
task.remove_port(port)
task_wait()
def tasks_run(self, arg_tasks, action):
"""
Prepare and run tasks
"""
print "TASKS RUN =>", action
workers = []
task = Task.task_self()
for script in arg_tasks:
groupedNodes = Node.Node.group_by_manager(arg_tasks[script])
for manager in groupedNodes:
nodesList = ",".join([node.name for node in groupedNodes[manager]])
command = managers.get_command(manager=manager, service=script, action=action)
print "Task run: " + command + ", nodes: " + nodesList
worker = task.shell(command, nodes=nodesList)
workers.append((worker, script))
task.run()
task.join()
return workers
def testPortRemove(self):
"""test port remove [private as of 1.2]"""
task = Task()
class PortHandler(EventHandler):
def ev_msg(self, port, msg):
pass
port = task.port(handler=PortHandler(), autoclose=True)
task.resume()
task._remove_port(port)
task_wait()
def testTaskInNewThread2(self):
# create a task in a new thread
task = Task()
match = "again"
# schedule a command in that task
worker = task.shell("/bin/echo %s" % match)
# run this task
task.resume()
# wait for the task to complete
task_wait()
# verify that the worker has completed
self.assertEqual(worker.read(), match.encode('ascii'))
# stop task
task.abort()
def testTaskInNewThread3(self):
# create a task in a new thread
task = Task()
match = "once again"
# schedule a command in that task
worker = task.shell("/bin/echo %s" % match)
# run this task
task.resume()
# wait for the task to complete
task_wait()
# verify that the worker has completed
self.assertEqual(worker.read(), match.encode('ascii'))
# stop task
task.abort()
def testSuspendMiscTwoTasks(self):
"""test task suspend/resume (2 tasks)"""
task = task_self()
task2 = Task()
task2.shell("sleep 4 && echo thr1")
task2.resume()
w = task.shell("sleep 1 && echo thr0", key=0)
task.resume()
self.assertEqual(task.key_buffer(0), "thr0")
self.assertEqual(w.read(), "thr0")
assert task2 != task
task2.suspend()
time.sleep(10)
task2.resume()
task_wait()
task2.shell("echo suspend_test", key=1)
task2.resume()
task_wait()
self.assertEqual(task2.key_buffer(1), "suspend_test")
def ev_timer(self, timer): # noqa, mccabe: MC0001 too complex (15) FIXME
"""Schedule the current command on the next node or the next command on the first batch of nodes.
This callback is triggered by `ClusterShell` when a scheduled `Task.timer()` goes off.
:Parameters:
according to parent :py:meth:`ClusterShell.Event.EventHandler.ev_timer`.
"""
success_ratio = 1 - ((self.counters['failed'] + self.counters['timeout']) / self.counters['total'])
node = None
if success_ratio >= self.success_threshold:
# Success ratio is still good, looking for the next node
with self.lock: # Avoid modifications of the same data from other callbacks triggered by ClusterShell
for new_node in self.nodes.values():
if new_node.state.is_pending:
# Found the next node where to execute the command
node = new_node
node.state.update(State.scheduled)
break
if node is not None:
# Schedule the execution with ClusterShell of the current command to the next node found above
command = self.nodes[node.name].commands[self.nodes[node.name].running_command_index + 1]
self.logger.debug("next_node=%s, timeout=%s, command='%s'", node.name, command.command, command.timeout)
Task.task_self().shell(command.command, handler=timer.eh, timeout=command.timeout, nodes=nodeset(node.name))
return
# No more nodes were left for the execution of the current command
with self.lock: # Avoid modifications of the same data from other callbacks triggered by ClusterShell
try:
command = self.commands[self.current_command_index].command
except IndexError:
command = None # Last command reached
# Get a list of the nodes still in pending state
pending = [pending_node.name for pending_node in self.nodes.values() if pending_node.state.is_pending]
# Nodes in running are still running the command and nodes in scheduled state will execute the command
# anyway, they were already offloaded to ClusterShell
accounted = len(pending) + self.counters['failed'] + self.counters['success'] + self.counters['timeout']
# Avoid race conditions
if self.aborted or accounted != self.counters['total'] or command is None or self.global_timedout:
self.logger.debug("Skipped timer")
return
if pending:
# This usually happens when executing in batches
self.logger.warning("Command '%s' was not executed on: %s", command, nodeset_fromlist(pending))
self.logger.info("Completed command '%s'", command)
restart = self.end_command()
self.current_command_index += 1 # Move the global pointer of the command in execution
if restart:
for node in self.nodes.values():
if node.state.is_success:
# Only nodes in pending state will be scheduled for the next command
node.state.update(State.pending)
if restart:
self.start_command(schedule=True)
def testThreadTaskWaitWithSuspend(self):
"""test task_wait() with suspended tasks"""
task = Task()
self.resumed = False
threading.Thread(None,
self._thread_delayed_unsuspend_func,
args=(task, )).start()
time_sh = int(random.random() * 4)
#print "TIME shell=%d" % time_sh
task.shell("sleep %d" % time_sh)
task.resume()
time.sleep(1)
suspended = task.suspend()
for i in range(1, 4):
task = Task()
task.shell("sleep %d" % i)
task.resume()
time.sleep(1)
task_wait()
self.assertTrue(self.resumed or suspended == False)
def testThreadSimpleTaskSupervisor(self):
"""test task methods from another thread"""
#print "PASS 1"
task = Task()
task.shell("sleep 3")
task.shell("echo testing", key=1)
task.resume()
task.join()
self.assertEqual(task.key_buffer(1), "testing")
#print "PASS 2"
task.shell("echo ok", key=2)
task.resume()
task.join()
#print "PASS 3"
self.assertEqual(task.key_buffer(2), "ok")
task.shell("sleep 1 && echo done", key=3)
task.resume()
task.join()
#print "PASS 4"
self.assertEqual(task.key_buffer(3), "done")
task.abort()
def testThreadTaskWaitWithSuspend(self):
"""test task_wait() with suspended tasks"""
task = Task()
self.resumed = False
thread.start_new_thread(TaskThreadSuspendTest._thread_delayed_unsuspend_func, (self, task))
time_sh = int(random.random()*4)
#print "TIME shell=%d" % time_sh
task.shell("sleep %d" % time_sh)
task.resume()
time.sleep(1)
suspended = task.suspend()
for i in range(1, 4):
task = Task()
task.shell("sleep %d" % i)
task.resume()
time.sleep(1)
task_wait()
self.assert_(self.resumed or suspended == False)
def testSuspendMiscTwoTasks(self):
"""test task suspend/resume (2 tasks)"""
task = task_self()
task2 = Task()
task2.shell("sleep 4 && echo thr1")
task2.resume()
w = task.shell("sleep 1 && echo thr0", key=0)
task.resume()
self.assertEqual(task.key_buffer(0), "thr0")
self.assertEqual(w.read(), "thr0")
assert task2 != task
task2.suspend()
time.sleep(10)
task2.resume()
task_wait()
task2.shell("echo suspend_test", key=1)
task2.resume()
task_wait()
self.assertEqual(task2.key_buffer(1), "suspend_test")
def testThreadTaskBuffers(self):
"""test task data access methods after join()"""
task = Task()
# test data access from main thread
# test stderr separated
task.set_default("stderr", True)
task.shell("echo foobar", key="OUT")
task.shell("echo raboof 1>&2", key="ERR")
task.resume()
task.join()
self.assertEqual(task.key_buffer("OUT"), "foobar")
self.assertEqual(task.key_error("OUT"), "")
self.assertEqual(task.key_buffer("ERR"), "")
self.assertEqual(task.key_error("ERR"), "raboof")
# test stderr merged
task.set_default("stderr", False)
task.shell("echo foobar", key="OUT")
task.shell("echo raboof 1>&2", key="ERR")
task.resume()
task.join()
self.assertEqual(task.key_buffer("OUT"), "foobar")
self.assertEqual(task.key_error("OUT"), "")
self.assertEqual(task.key_buffer("ERR"), "raboof")
self.assertEqual(task.key_error("ERR"), "")
def testThreadTaskWaitWithSuspend(self):
"""test task_wait() with suspended tasks"""
task = Task()
self.resumed = False
threading.Thread(None, self._thread_delayed_unsuspend_func,
args=(task,)).start()
time_sh = int(random.random()*4)
#print "TIME shell=%d" % time_sh
task.shell("sleep %d" % time_sh)
task.resume()
time.sleep(1)
suspended = task.suspend()
for i in range(1, 4):
task = Task()
task.shell("sleep %d" % i)
task.resume()
time.sleep(1)
task_wait()
self.assertTrue(self.resumed or suspended == False)
def testThreadSimpleTaskSupervisor(self):
"""test task methods from another thread"""
#print "PASS 1"
task = Task()
task.shell("sleep 3")
task.shell("echo testing", key=1)
task.resume()
task.join()
self.assertEqual(task.key_buffer(1), b"testing")
#print "PASS 2"
task.shell("echo ok", key=2)
task.resume()
task.join()
#print "PASS 3"
self.assertEqual(task.key_buffer(2), b"ok")
task.shell("sleep 1 && echo done", key=3)
task.resume()
task.join()
#print "PASS 4"
self.assertEqual(task.key_buffer(3), b"done")
task.abort()
def testThreadTaskBuffers(self):
"""test task data access methods after join()"""
task = Task()
# test data access from main thread
# test stderr separated
task.set_default("stderr", True)
task.shell("echo foobar", key="OUT")
task.shell("echo raboof 1>&2", key="ERR")
task.resume()
task.join()
self.assertEqual(task.key_buffer("OUT"), b"foobar")
self.assertEqual(task.key_error("OUT"), b"")
self.assertEqual(task.key_buffer("ERR"), b"")
self.assertEqual(task.key_error("ERR"), b"raboof")
# test stderr merged
task.set_default("stderr", False)
task.shell("echo foobar", key="OUT")
task.shell("echo raboof 1>&2", key="ERR")
task.resume()
task.join()
self.assertEqual(task.key_buffer("OUT"), b"foobar")
self.assertEqual(task.key_error("OUT"), b"")
self.assertEqual(task.key_buffer("ERR"), b"raboof")
self.assertEqual(task.key_error("ERR"), b"")
def testThreadTaskWaitWithSuspend(self):
"""test task_wait() with suspended tasks"""
task = Task()
self.resumed = False
thread.start_new_thread(
TaskThreadSuspendTest._thread_delayed_unsuspend_func, (self, task))
time_sh = int(random.random() * 4)
#print "TIME shell=%d" % time_sh
task.shell("sleep %d" % time_sh)
task.resume()
time.sleep(1)
suspended = task.suspend()
for i in range(1, 4):
task = Task()
task.shell("sleep %d" % i)
task.resume()
time.sleep(1)
task_wait()
self.assert_(self.resumed or suspended == False)