def test_concurrent_ignore_empty():
runner = concurrent(ignore_empty=True)
done, pending = run_in_loop(runner.run())
assert len(done) == 0
assert len(pending) == 0
runner = concurrent()
done, pending = run_in_loop(runner.run(ignore_empty=True))
assert len(done) == 0
assert len(pending) == 0
def test_concurrent_single():
@asyncio.coroutine
def coro(num):
return num * 2
p = concurrent(10)
p.add(coro, 2)
assert len(p) == 1
assert p.__len__() == 1
assert len(p.pool) == 1
done, pending = run_in_loop(p.run())
assert len(done) == 1
assert len(pending) == 0
for future in done:
assert future.result() == 4
def test_concurrent_observe_exception():
start = []
error = []
finish = []
@asyncio.coroutine
def coro(num):
if num > 4:
raise ValueError('invalid number')
return num * 2
@asyncio.coroutine
def on_start(task):
start.append(task)
@asyncio.coroutine
def on_error(task, err):
error.append(err)
@asyncio.coroutine
def on_finish(task, result):
finish.append(result)
p = concurrent(1)
p.on('task.start', on_start)
p.on('task.error', on_error)
p.on('task.finish', on_finish)
p.add(coro, 2)
p.add(coro, 4)
p.add(coro, 6)
assert len(p.pool) == 3
with pytest.raises(ValueError):
run_in_loop(p.run(return_exceptions=False))
# Assert event calls
assert len(start) == 3
assert len(error) == 1
assert len(finish) == 2
finish.sort()
assert finish == [4, 8]
def test_concurrent_observe():
start = []
finish = []
@asyncio.coroutine
def coro(num):
return num * 2
@asyncio.coroutine
def on_start(task):
start.append(task)
@asyncio.coroutine
def on_finish(task, result):
finish.append(result)
p = concurrent(10)
p.on('task.start', on_start)
p.on('task.finish', on_finish)
p.add(coro, 2)
p.add(coro, 4)
p.add(coro, 6)
assert len(p.pool) == 3
done, pending = run_in_loop(p.run())
assert len(done) == 3
assert len(pending) == 0
# Assert event calls
assert len(start) == 3
assert len(finish) == 3
results = [future.result() for future in done]
results.sort()
assert results == [4, 8, 12]
finish.sort()
assert finish == [4, 8, 12]
def run_test(limit=3, times=10, timespan=0.1):
p = concurrent(limit)
for i in range(times):
p.add(sleep_coro, timespan)
return (yield from p.run())
def test_concurrent_empty_error():
with pytest.raises(ValueError):
run_in_loop(concurrent().run())
async def transition(self, wanted_state: State) -> TransitionResult:
self.check_transition(wanted_state)
self._wanted_state = wanted_state
coro_names: List[str] = [i.name for i in self.__workers]
coro_results: List[Optional[TransitionResult]] = [
None for _ in self.__workers
]
def pretty_result(
result: Union[TransitionResult, Exception, None]) -> str:
result_names = {
TransitionResult.DONE:
Color.YELLOW + "Action performed" + Color.RESET,
TransitionResult.NOOP: Color.GREEN + "Ok" + Color.RESET,
TransitionResult.ERROR: Color.RED + "ERROR" + Color.RESET,
None: "Pending"
}
if isinstance(result, Exception):
return Color.RED + "ERROR! Shutting down" + Color.RESET
return result_names[result]
def print_progress_update(
task: asyncio.Task = None,
result: Optional[TransitionResult] = None) -> None:
longest_name_len = max(map(len, coro_names)) + 1
if not task:
return
name = coro_names[task.index]
print(
f"{Color.CYAN}{(name + ':').ljust(longest_name_len)}{Color.RESET} {pretty_result(result)}"
)
def register_result(
task: asyncio.Task, result: Union[TransitionResult,
Exception]) -> None:
coro_results[task.index] = result
print_progress_update(task, result)
message = f"Current state is {self.cur_state}, next step: {wanted_state}"
print(Color.BROWN + ('-' * len(message)))
print(message)
print("Trying to reach next state, waiting for worker rendezvous")
print("Memory usage:")
subprocess.Popen(['free', '-h']).wait()
balloon_stat()
print(('-' * len(message)) + Color.RESET)
print_progress_update()
try:
concurrent = paco.concurrent(limit=len(self.__workers))
# noinspection PyTypeChecker
concurrent.on('task.finish', register_result)
# noinspection PyTypeChecker
concurrent.on('task.error', register_result)
for worker in self.__workers:
concurrent.add(worker.transition, wanted_state)
# If we don't retrieve the exception manually, two exceptions are thrown making the error more confusing
done, pending = await concurrent.run(return_when='FIRST_EXCEPTION',
return_exceptions=True)
for task in list(done) + list(pending):
exc = task.exception() or isinstance(
task.result(), Exception) and task.result() or None
if exc:
raise exc
self._cur_state = wanted_state
except Exception:
# If state is JOB, next state will be either LOGCAT or STOP, so let's not die yet
if wanted_state == State.JOB:
print(
Color.RED +
"Error in JOB state. Ignoring error state transition and proceeding to next state"
)
traceback.print_exc()
print(Color.RESET, end="")
self._cur_state = wanted_state
return TransitionResult.ERROR
self._cur_state = State.UNKNOWN
try:
await concurrent.cancel()
except (AttributeError, TypeError):
pass
raise
finally:
self._wanted_state = None
return TransitionResult.NOOP