def test_get_results(self):
"""Test retrieving results from the results queue."""
# Insert results for the same drone twice into the results queue
# and confirm that an exception is raised.
task_queue = thread_lib.ThreadedTaskQueue()
drone1 = self.create_remote_drone('fakehostname1')
drone2 = self.create_remote_drone('fakehostname2')
task_queue.results_queue.put(
thread_lib.ThreadedTaskQueue.result(drone1, self.mock_return))
task_queue.results_queue.put(
thread_lib.ThreadedTaskQueue.result(drone1, self.mock_return))
self.god.stub_function(task_queue, 'wait_on_drones')
task_queue.wait_on_drones.expect_call()
self.assertRaises(drone_task_queue.DroneTaskQueueException,
task_queue.get_results)
# Insert results for different drones and check that they're returned
# in a drone results dict.
self.assertTrue(task_queue.results_queue.empty())
task_queue.results_queue.put(
thread_lib.ThreadedTaskQueue.result(drone1, self.mock_return))
task_queue.results_queue.put(
thread_lib.ThreadedTaskQueue.result(drone2, self.mock_return))
task_queue.wait_on_drones.expect_call()
results = task_queue.get_results()
self.assertTrue(results[drone1] == self.mock_return
and results[drone2] == self.mock_return)
self.god.check_playback()
def __init__(self):
# absolute path of base results dir
self._results_dir = None
# holds Process objects
self._process_set = set()
# holds the list of all processes running on all drones
self._all_processes = {}
# maps PidfileId to PidfileContents
self._pidfiles = {}
# same as _pidfiles
self._pidfiles_second_read = {}
# maps PidfileId to _PidfileInfo
self._registered_pidfile_info = {}
# used to generate unique temporary paths
self._temporary_path_counter = 0
# maps hostname to Drone object
self._drones = {}
self._results_drone = None
# maps results dir to dict mapping file path to contents
self._attached_files = {}
# heapq of _DroneHeapWrappers
self._drone_queue = []
# map drone hostname to time stamp of email that
# has been sent about the drone hitting process limit.
self._notify_record = {}
# A threaded task queue used to refresh drones asynchronously.
if _THREADED_DRONE_MANAGER:
self._refresh_task_queue = thread_lib.ThreadedTaskQueue(
name='%s.refresh_queue' % self._STATS_KEY)
else:
self._refresh_task_queue = drone_task_queue.DroneTaskQueue()
def test_execute(self):
"""Test task queue execute."""
drone1 = self.create_remote_drone('fakehostname1')
drone2 = self.create_remote_drone('fakehostname2')
drone3 = self.create_remote_drone('fakehostname3')
# Check task queue exception conditions.
task_queue = thread_lib.ThreadedTaskQueue()
task_queue.results_queue.put(1)
self.assertRaises(drone_task_queue.DroneTaskQueueException,
task_queue.execute, [])
task_queue.results_queue.get()
task_queue.drone_threads[drone1] = None
self.assertRaises(drone_task_queue.DroneTaskQueueException,
task_queue.execute, [])
task_queue.drone_threads = {}
# Queue 2 calls against each drone, and confirm that the host's
# run method is called 3 times. Then check the threads created,
# and finally compare results returned by the task queue against
# the mock results.
drones = [drone1, drone2, drone3]
for drone in drones:
drone.queue_call('foo')
drone.queue_call('bar')
mock_result = utils.CmdResult(
stdout=cPickle.dumps(self.mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
task_queue.execute(drones, wait=False)
self.assertTrue(set(task_queue.drone_threads.keys()) == set(drones))
for drone, thread in task_queue.drone_threads.iteritems():
self.assertTrue(drone.hostname in thread.getName())
self.assertTrue(thread.isDaemon())
self.assertRaises(RuntimeError, thread.start)
results = task_queue.get_results()
for drone, result in results.iteritems():
self.assertTrue(result == self.mock_return['results'])
# Test synchronous execute
drone1.queue_call('foo')
mock_result = utils.CmdResult(stdout=cPickle.dumps(self.mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone1.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
self.assertTrue(
task_queue.execute(drones, wait=True)[drone1] ==
self.mock_return['results'])
self.god.check_playback()
def execute_actions(self):
"""
Called at the end of a scheduler cycle to execute all queued actions
on drones.
"""
# Invoke calls queued on all drones since the last call to execute
# and wait for them to return.
if _THREADED_DRONE_MANAGER:
thread_lib.ThreadedTaskQueue(name='%s.execute_queue' %
self._STATS_KEY).execute(
self._drones.values())
else:
drone_task_queue.DroneTaskQueue().execute(self._drones.values())
try:
self._results_drone.execute_queued_calls()
except error.AutoservError:
m = 'chromeos/autotest/errors/results_repository_failed'
metrics.Counter(m).increment(
fields={'drone_hostname': self._results_drone.hostname})
self._results_drone.clear_call_queue()
def execute_actions(self):
"""
Called at the end of a scheduler cycle to execute all queued actions
on drones.
"""
# Invoke calls queued on all drones since the last call to execute
# and wait for them to return.
if _THREADED_DRONE_MANAGER:
thread_lib.ThreadedTaskQueue(
name='%s.execute_queue' % self._STATS_KEY).execute(
self._drones.values())
else:
drone_task_queue.DroneTaskQueue().execute(self._drones.values())
try:
self._results_drone.execute_queued_calls()
except error.AutoservError:
warning = ('Results repository failed to execute calls:\n' +
traceback.format_exc())
email_manager.manager.enqueue_notify_email(
'Results repository error', warning)
self._results_drone.clear_call_queue()
def test_worker(self):
"""Test the worker method of a ThreadedTaskQueue."""
# Invoke the worker method with a drone that has a queued call and check
# that the drones host.run method is invoked for the call, and the
# results queue contains the expected results.
drone = self.create_remote_drone('fakehostname')
task_queue = thread_lib.ThreadedTaskQueue()
drone.queue_call('foo')
mock_result = utils.CmdResult(stdout=cPickle.dumps(self.mock_return))
self._mock_host.run.expect_call(
'python %s' % self.drone_utility_path,
stdin=cPickle.dumps(drone.get_calls()),
stdout_tee=None,
connect_timeout=mock.is_instance_comparator(int)).and_return(
mock_result)
task_queue.worker(drone, task_queue.results_queue)
result = task_queue.results_queue.get()
self.assertTrue(task_queue.results_queue.empty()
and result.drone == drone
and result.results == self.mock_return['results'])
self.god.check_playback()
def test_wait_on_drones(self):
"""Test waiting on drone threads."""
def waiting_func(queue):
while len(queue.queue) < 2:
continue
logging.warning('Consuming thread finished.')
queue.put(3)
def exception_func(queue):
while queue.empty():
continue
queue.put(2)
logging.warning('Failing thread raising error.')
raise ValueError('Value error')
def quick_func():
return
# Create 2 threads, one of which raises an exception while the other
# just exits normally. Insert both threads into the thread_queue against
# mock drones and confirm that:
# a. The task queue waits for both threads, though the first one fails.
# b. The task queue records the right DroneTaskQueueException, which
# contains the original exception.
# c. The failing thread records its own exception instead of raising it.
task_queue = thread_lib.ThreadedTaskQueue()
drone1 = self.create_remote_drone('fakehostname1')
drone2 = self.create_remote_drone('fakehostname2')
sync_queue = Queue.Queue()
waiting_worker = thread_lib.ExceptionRememberingThread(
target=waiting_func, args=(sync_queue, ))
failing_worker = thread_lib.ExceptionRememberingThread(
target=exception_func, args=(sync_queue, ))
task_queue.drone_threads[drone1] = waiting_worker
task_queue.drone_threads[drone2] = failing_worker
master_thread = thread_lib.ExceptionRememberingThread(
target=task_queue.wait_on_drones)
thread_list = [failing_worker, waiting_worker, master_thread]
for thread in thread_list:
thread.setDaemon(True)
thread.start()
sync_queue.put(1)
master_thread.join()
self.assertTrue(
isinstance(master_thread.err,
drone_task_queue.DroneTaskQueueException))
self.assertTrue(isinstance(failing_worker.err, ValueError))
self.assertTrue(str(failing_worker.err) in str(master_thread.err))
self.assertTrue(3 in list(sync_queue.queue))
self.assertTrue(task_queue.drone_threads == {})
# Call wait_on_drones after the child thread has exited.
quick_worker = thread_lib.ExceptionRememberingThread(target=quick_func)
task_queue.drone_threads[drone1] = quick_worker
quick_worker.start()
while quick_worker.isAlive():
continue
task_queue.wait_on_drones()
self.assertTrue(task_queue.drone_threads == {})
