def cpu_bound(task_id: Optional[int] = None,
send_process_id: bool = False) -> str:
"""
CPU-bound operations will block the event loop:
in general it is preferable to run them in a process pool.
"""
try:
logger = getLogger(__name__)
def hander(_signum: int, _frame: Optional[Any]) -> NoReturn:
logger.info("CPU-bound: Terminate")
raise Terminated()
signal(SIGTERM, hander)
process_id = os.getpid()
print(process_id)
logger.info("CPU-bound: process id = %d", process_id)
if send_process_id:
time.sleep(SECOND_SLEEP_FOR_TEST_MIDDLE)
logger.info("CPU-bound: Send process id")
LocalSocket.send(str(process_id))
logger.info("CPU-bound: Start")
result = sum(i * i for i in range(10**7))
logger.debug("CPU-bound: Finish")
logger.debug("%d %s", task_id, datetime.now())
return ("" if task_id is None else
f"task_id: {task_id}, ") + f"result: {result}"
except KeyboardInterrupt:
logger.info("CPU-bound: KeyboardInterupt")
raise
def test_sigterm(manager_queue: "queue.Queue[LogRecord]") -> None:
"""ProcessTaskPoolExecutor should raise keyboard interrupt."""
process = Process(target=example_use_case_cancel_repost_process_id, kwargs={"queue_main": manager_queue})
process.start()
LocalSocket.receive()
time.sleep(SECOND_SLEEP_FOR_TEST_SHORT)
psutil_process = psutil.Process(process.pid)
psutil_process.send_signal(SIGTERM)
psutil_process.wait()
# Reason: Requires to enhance types-psutil
assert not psutil_process.is_running() # type: ignore
assert_graceful_shutdown(manager_queue)
def test_keyboard_interrupt(self) -> None:
"""Tests keyboard interrupt and send response to pytest by socket when succeed."""
try:
self.test()
except BaseException as error:
self.logger.exception(error)
traceback.print_exc()
LocalSocket.send("Test failed")
raise
else:
LocalSocket.send("Test succeed")
finally:
asyncio.run(asyncio.sleep(10))
def test_keyboard_interrupt_on_linux(self) -> None:
"""
- Keyboard interrupt should reach to all descendant processes.
- Keyboard interrupt should shutdown ProcessTaskPoolExecutor gracefully.
"""
process = Process(target=self.report_raises_keyboard_interrupt)
process.start()
LocalSocket.receive()
time.sleep(SECOND_SLEEP_FOR_TEST_SHORT)
self.simulate_ctrl_c_in_posix(process)
assert LocalSocket.receive() == "Test succeed"
process.join()
assert process.exitcode == 0
assert not process.is_alive()
def example_use_case_cancel_repost_process_id(
queue_sub: Optional["queue.Queue[LogRecord]"] = None,
queue_main: Optional["queue.Queue[LogRecord]"] = None) -> None:
"""The example use case of ProcessTaskPoolExecutor for E2E testing in case of cancel."""
time.sleep(SECOND_SLEEP_FOR_TEST_SHORT)
LocalSocket.send(str(os.getpid()))
if queue_main:
logger = getLogger()
logger.addHandler(QueueHandler(queue_main))
logger.setLevel(DEBUG)
results = example_use_case_method(queue_sub)
results_string = repr(results)
LocalSocket.send(results_string)
pytest.fail(results_string)
async def get_process_id() -> int:
print("Await socket")
process_id = int(LocalSocket.receive())
print("Await sleep")
await asyncio.sleep(SECOND_SLEEP_FOR_TEST_WINDOWS_NEW_WINDOW)
print("Kill group lead process")
return process_id
def test_keyboard_interrupt_ctrl_c_new_process_group() -> None:
"""
see:
- On Windows, what is the python launcher 'py' doing that lets control-C cross between process groups?
https://stackoverflow.com/q/42180468/12721873
https://github.com/njsmith/appveyor-ctrl-c-test/blob/34e13fab9be56d59c3eba566e26d80505c309438/a.py
https://github.com/njsmith/appveyor-ctrl-c-test/blob/34e13fab9be56d59c3eba566e26d80505c309438/run-a.py
"""
with Popen(
f"{sys.executable} tests\\testlibraries\\keyboaard_interrupt_in_windows.py",
# Reason: Definition of following constant is Windows only
creationflags=CREATE_NEW_PROCESS_GROUP, # type: ignore
) as popen:
asyncio.run(asyncio.sleep(SECOND_SLEEP_FOR_TEST_MIDDLE))
LocalSocket.send(str(popen.pid))
assert LocalSocket.receive() == "Test succeed"
assert popen.wait() == 0
def test_keyboard_interrupt_ctrl_c_new_window() -> None:
"""
see:
- Answer: Sending ^C to Python subprocess objects on Windows
https://stackoverflow.com/a/7980368/12721873
"""
with Popen(f"start {sys.executable} tests\\testlibraries\\subprocess_wrapper_windows.py", shell=True) as popen:
assert LocalSocket.receive() == "Test succeed"
assert popen.wait() == 0
def test_coroutine() -> None:
"""CPU bound should be terminated when coroutine."""
with ProcessPoolExecutor() as executor:
future = executor.submit(cpu_bound, 1, True)
pid = LocalSocket.receive()
print(pid)
psutil_process = psutil.Process(int(pid))
psutil_process.terminate()
assert isinstance(future.exception(), Terminated)
assert future.done()
def test_terminate(manager_queue: "queue.Queue[LogRecord]") -> None:
"""
Can't test in case process.kill() since it sends signal.SIGKILL and Python can't trap it.
Function process.kill() stops pytest process.
see:
- https://psutil.readthedocs.io/en/latest/#psutil.Process.terminate
- https://psutil.readthedocs.io/en/latest/#psutil.Process.kill
- https://docs.python.org/ja/3/library/signal.html#signal.SIGKILL
"""
process = Process(target=example_use_case_cancel_repost_process_id, kwargs={"queue_main": manager_queue})
process.start()
LocalSocket.receive()
time.sleep(SECOND_SLEEP_FOR_TEST_SHORT)
psutil_process = psutil.Process(process.pid)
psutil_process.terminate()
psutil_process.wait()
# Reason: Requires to enhance types-psutil
assert not psutil_process.is_running() # type: ignore
assert_graceful_shutdown(manager_queue)
async def execute_test(future: "Future[Any]", signal_number: int,
expect: bool,
expect_type_error: Type[BaseException]) -> None:
"""Executes test."""
process_task = ProcessTask(int(LocalSocket.receive()))
assert future.done() == expect
assert future.cancelled() == expect
await asyncio.sleep(SECOND_SLEEP_FOR_TEST_SHORT)
with pytest.raises(expect_type_error):
process_task.send_signal(signal_number)
await future
def test_method() -> None:
"""CPU bound should be terminated when method."""
parent_conn, child_conn = multiprocessing.Pipe()
process = Process(target=process_cpu_bound_method,
args=(1, True, child_conn))
process.start()
pid = LocalSocket.receive()
print(pid)
psutil_process = psutil.Process(int(pid))
psutil_process.terminate()
process.join()
assert not parent_conn.poll()
async def test_future() -> None:
"""
Can't test in case process.kill() since it sends signal.SIGKILL and Python can't trap it.
Function process.kill() stops pytest process.
see:
- https://psutil.readthedocs.io/en/latest/#psutil.Process.terminate
- https://psutil.readthedocs.io/en/latest/#psutil.Process.kill
- https://docs.python.org/ja/3/library/signal.html#signal.SIGKILL
"""
with ProcessTaskPoolExecutor(cancel_tasks_when_shutdown=True) as executor:
future = executor.create_process_task(process_cpu_bound, 1, True)
pid = LocalSocket.receive()
process = psutil.Process(int(pid))
process.terminate()
with pytest.raises(Terminated):
await future
assert future.done()
assert isinstance(future.exception(), Terminated)
def report_raises_keyboard_interrupt() -> None:
with pytest.raises(KeyboardInterrupt):
example_use_case_cancel_repost_process_id()
LocalSocket.send("Test succeed")