def test_async_push_upload_errors(self):
chunk = 'data_chunk'
def _generator(_chunk_size):
yield chunk
def push_side_effect():
raise IOError('Nope')
# TODO(vadimsh): Retrying push when fetching data from a generator is
# broken now (it reuses same generator instance when retrying).
content_sources = (
# generator(),
lambda _chunk_size: [chunk], )
for use_zip in (False, True):
for source in content_sources:
item = FakeItem(chunk)
self.mock(item, 'content', source)
storage_api = self.mock_push(push_side_effect)
storage = isolateserver.Storage(storage_api, use_zip)
channel = threading_utils.TaskChannel()
storage.async_push(channel, 0, item)
with self.assertRaises(IOError):
channel.pull()
# First initial attempt + all retries.
attempts = 1 + isolateserver.WorkerPool.RETRIES
# Single push attempt parameters.
expected_push = (item, item.zipped if use_zip else item.data)
# Ensure all pushes are attempted.
self.assertEqual([expected_push] * attempts,
storage_api.pushed)
def test_async_push_upload_errors(self):
chunk = 'data_chunk'
def push_side_effect():
raise IOError('Nope')
content_sources = (
lambda: [chunk],
lambda: [(yield chunk)],
)
for use_zip in (False, True):
for source in content_sources:
item = FakeItem(chunk)
self.mock(item, 'content', source)
server_ref = isolate_storage.ServerRef(
'http://localhost:1', 'default-gzip' if use_zip else 'default')
storage_api = MockedStorageApi(
server_ref, {item.digest: 'push_state'}, push_side_effect)
storage = isolateserver.Storage(storage_api)
channel = threading_utils.TaskChannel()
storage._async_push(channel, item, self.get_push_state(storage, item))
with self.assertRaises(IOError):
next(channel)
# First initial attempt + all retries.
attempts = 1 + storage.net_thread_pool.RETRIES
# Single push attempt call arguments.
expected_push = (
item, 'push_state', item.zipped if use_zip else item.data)
# Ensure all pushes are attempted.
self.assertEqual(
[expected_push] * attempts, storage_api.push_calls)
def test_add_task_with_channel_retryable_error(self):
with threading_utils.AutoRetryThreadPool([OSError], 2, 1, 1, 0) as pool:
channel = threading_utils.TaskChannel()
def throw(exc):
raise exc
pool.add_task_with_channel(channel, 0, throw, OSError())
with self.assertRaises(OSError):
channel.pull()
def test_send_exception_raises_exception(self):
class CustomError(Exception):
pass
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
tp.add_task(0, lambda: channel.send_exception(CustomError()))
with self.assertRaises(CustomError):
channel.pull()
def test_timeout_exception_from_task(self):
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
def task_func():
raise threading_utils.TaskChannel.Timeout()
tp.add_task(0, channel.wrap_task(task_func))
# 'Timeout' raised by task gets transformed into 'RuntimeError'.
with self.assertRaises(RuntimeError):
channel.next()
def test_wrap_task_raises_exception(self):
class CustomError(Exception):
pass
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
def task_func():
raise CustomError()
tp.add_task(0, channel.wrap_task(task_func))
with self.assertRaises(CustomError):
channel.pull()
def test_send_exception_raises_exception(self):
class CustomError(Exception):
pass
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
exc_info = (CustomError, CustomError(), None)
tp.add_task(0, lambda: channel.send_exception(exc_info))
with self.assertRaises(CustomError):
next(channel)
def test_add_task_with_channel_fatal_error(self):
with threading_utils.AutoRetryThreadPool([OSError], 2, 1, 1,
0) as pool:
channel = threading_utils.TaskChannel()
def throw(exc):
raise exc
pool.add_task_with_channel(channel, 0, throw, ValueError())
with self.assertRaises(ValueError):
next(channel)
def test_next_timeout(self):
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
def task_func():
# This test ultimately relies on the condition variable primitive
# provided by pthreads. There's no easy way to mock time for it.
# Increase this duration if the test is flaky.
time.sleep(0.2)
return 123
tp.add_task(0, channel.wrap_task(task_func))
with self.assertRaises(threading_utils.TaskChannel.Timeout):
channel.next(timeout=0.001)
self.assertEqual(123, channel.next())
def test_async_push(self):
for use_zip in (False, True):
item = FakeItem('1234567')
storage_api = self.mock_push()
storage = isolateserver.Storage(storage_api, use_zip)
channel = threading_utils.TaskChannel()
storage.async_push(channel, 0, item)
# Wait for push to finish.
pushed_item = channel.pull()
self.assertEqual(item, pushed_item)
# StorageApi.push was called with correct arguments.
self.assertEqual([(item, item.zipped if use_zip else item.data)],
storage_api.pushed)
def test_add_task_with_channel_captures_stack_trace(self):
with threading_utils.AutoRetryThreadPool([OSError], 2, 1, 1, 0) as pool:
channel = threading_utils.TaskChannel()
def throw(exc):
def function_with_some_unusual_name():
raise exc
function_with_some_unusual_name()
pool.add_task_with_channel(channel, 0, throw, OSError())
exc_traceback = ''
try:
channel.next()
except OSError:
exc_traceback = traceback.format_exc()
self.assertIn('function_with_some_unusual_name', exc_traceback)
def test_generator(self):
channel = threading_utils.TaskChannel()
channel.send_result(1)
channel.send_result(2)
channel.send_done()
channel.send_done()
channel.send_result(3)
channel.send_done()
actual = list(channel)
self.assertEqual([1, 2], actual)
actual = list(channel)
self.assertEqual([], actual)
actual = list(channel)
self.assertEqual([3], actual)
def test_wrap_task_exception_captures_stack_trace(self):
class CustomError(Exception):
pass
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
def task_func():
def function_with_some_unusual_name():
raise CustomError()
function_with_some_unusual_name()
tp.add_task(0, channel.wrap_task(task_func))
exc_traceback = ''
try:
channel.next()
except CustomError:
exc_traceback = traceback.format_exc()
self.assertIn('function_with_some_unusual_name', exc_traceback)
def test_async_push(self):
for use_zip in (False, True):
item = FakeItem('1234567')
server_ref = isolate_storage.ServerRef(
'http://localhost:1', 'default-gzip' if use_zip else 'default')
storage_api = MockedStorageApi(server_ref, {item.digest: 'push_state'})
storage = isolateserver.Storage(storage_api)
channel = threading_utils.TaskChannel()
storage._async_push(channel, item, self.get_push_state(storage, item))
# Wait for push to finish.
pushed_item = next(channel)
self.assertEqual(item, pushed_item)
# StorageApi.push was called with correct arguments.
self.assertEqual(
[(item, 'push_state', item.zipped if use_zip else item.data)],
storage_api.push_calls)
def test_async_push_generator_errors(self):
class FakeException(Exception):
pass
def faulty_generator(_chunk_size):
yield 'Hi!'
raise FakeException('fake exception')
for use_zip in (False, True):
item = FakeItem('')
self.mock(item, 'content', faulty_generator)
storage_api = self.mock_push()
storage = isolateserver.Storage(storage_api, use_zip)
channel = threading_utils.TaskChannel()
storage.async_push(channel, 0, item)
with self.assertRaises(FakeException):
channel.pull()
# StorageApi's push should never complete when data can not be read.
self.assertEqual(0, len(storage_api.pushed))
def test_async_push_generator_errors(self):
class FakeException(Exception):
pass
def faulty_generator():
yield 'Hi!'
raise FakeException('fake exception')
for use_zip in (False, True):
item = FakeItem('')
self.mock(item, 'content', faulty_generator)
server_ref = isolate_storage.ServerRef(
'http://localhost:1', 'default-gzip' if use_zip else 'default')
storage_api = MockedStorageApi(server_ref, {item.digest: 'push_state'})
storage = isolateserver.Storage(storage_api)
channel = threading_utils.TaskChannel()
storage._async_push(channel, item, self.get_push_state(storage, item))
with self.assertRaises(FakeException):
next(channel)
# StorageApi's push should never complete when data can not be read.
self.assertEqual(0, len(storage_api.push_calls))
def test_async_push_upload_errors(self):
chunk = 'data_chunk'
def _generator():
yield chunk
def push_side_effect():
raise IOError('Nope')
# TODO(vadimsh): Retrying push when fetching data from a generator is
# broken now (it reuses same generator instance when retrying).
content_sources = (
# generator(),
lambda: [chunk], )
for use_zip in (False, True):
for source in content_sources:
item = FakeItem(chunk)
self.mock(item, 'content', source)
storage_api = MockedStorageApi(
{item.digest: 'push_state'},
push_side_effect,
namespace='default-gzip' if use_zip else 'default')
storage = isolateserver.Storage(storage_api)
channel = threading_utils.TaskChannel()
storage.async_push(channel, item,
self.get_push_state(storage, item))
with self.assertRaises(IOError):
channel.pull()
# First initial attempt + all retries.
attempts = 1 + storage.net_thread_pool.RETRIES
# Single push attempt call arguments.
expected_push = (item, 'push_state',
item.zipped if use_zip else item.data)
# Ensure all pushes are attempted.
self.assertEqual([expected_push] * attempts,
storage_api.push_calls)
def test_wrap_task_passes_exception_value(self):
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
tp.add_task(0, channel.wrap_task(lambda: Exception()))
self.assertTrue(isinstance(channel.pull(), Exception))
def test_wrap_task_passes_simple_value(self):
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
tp.add_task(0, channel.wrap_task(lambda: 0))
self.assertEqual(0, channel.pull())
def test_add_task_with_channel_success(self):
with threading_utils.AutoRetryThreadPool([OSError], 2, 1, 1,
0) as pool:
channel = threading_utils.TaskChannel()
pool.add_task_with_channel(channel, 0, lambda: 0)
self.assertEqual(0, channel.pull())
def yield_results(swarm_base_url, task_keys, timeout, max_threads,
print_status_updates, output_collector):
"""Yields swarming task results from the swarming server as (index, result).
Duplicate shards are ignored. Shards are yielded in order of completion.
Timed out shards are NOT yielded at all. Caller can compare number of yielded
shards with len(task_keys) to verify all shards completed.
max_threads is optional and is used to limit the number of parallel fetches
done. Since in general the number of task_keys is in the range <=10, it's not
worth normally to limit the number threads. Mostly used for testing purposes.
output_collector is an optional instance of TaskOutputCollector that will be
used to fetch files produced by a task from isolate server to the local disk.
Yields:
(index, result). In particular, 'result' is defined as the
GetRunnerResults() function in services/swarming/server/test_runner.py.
"""
number_threads = (min(max_threads, len(task_keys))
if max_threads else len(task_keys))
should_stop = threading.Event()
results_channel = threading_utils.TaskChannel()
with threading_utils.ThreadPool(number_threads, number_threads, 0) as pool:
try:
# Adds a task to the thread pool to call 'retrieve_results' and return
# the results together with shard_index that produced them (as a tuple).
def enqueue_retrieve_results(shard_index, task_key):
task_fn = lambda *args: (shard_index, retrieve_results(*args))
pool.add_task(0, results_channel.wrap_task(task_fn),
swarm_base_url, shard_index, task_key, timeout,
should_stop, output_collector)
# Enqueue 'retrieve_results' calls for each shard key to run in parallel.
for shard_index, task_key in enumerate(task_keys):
enqueue_retrieve_results(shard_index, task_key)
# Wait for all of them to finish.
shards_remaining = range(len(task_keys))
active_task_count = len(task_keys)
while active_task_count:
shard_index, result = None, None
try:
shard_index, result = results_channel.pull(
timeout=STATUS_UPDATE_INTERVAL)
except threading_utils.TaskChannel.Timeout:
if print_status_updates:
print(
'Waiting for results from the following shards: %s'
% ', '.join(map(str, shards_remaining)))
sys.stdout.flush()
continue
except Exception:
logging.exception(
'Unexpected exception in retrieve_results')
# A call to 'retrieve_results' finished (successfully or not).
active_task_count -= 1
if not result:
logging.error(
'Failed to retrieve the results for a swarming key')
continue
# Yield back results to the caller.
assert shard_index in shards_remaining
shards_remaining.remove(shard_index)
yield shard_index, result
finally:
# Done or aborted with Ctrl+C, kill the remaining threads.
should_stop.set()
def test_passes_simple_value(self):
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
tp.add_task(0, lambda: channel.send_result(0))
self.assertEqual(0, channel.next())
def test_passes_exception_value(self):
with threading_utils.ThreadPool(1, 1, 0) as tp:
channel = threading_utils.TaskChannel()
tp.add_task(0, lambda: channel.send_result(Exception()))
self.assertTrue(isinstance(channel.next(), Exception))
