def main():
with kiwipy.connect(
'amqp://127.0.0.1') as communicator, tempfile.TemporaryDirectory(
) as tmpdir:
loop = asyncio.get_event_loop()
persister = plumpy.PicklePersister(tmpdir)
task_receiver = plumpy.ProcessLauncher(loop=loop, persister=persister)
def callback(*args, **kwargs):
fut = plumpy.create_task(functools.partial(task_receiver, *args,
**kwargs),
loop=loop)
return fut
communicator.add_task_subscriber(callback)
process_controller = plumpy.RemoteProcessThreadController(communicator)
future = process_controller.launch_process(DummyProcessWithOutput)
async def task():
result = await asyncio.wrap_future(future.result())
print(result)
loop.run_until_complete(task())
async def test_execute_action_nowait(self, loop_communicator,
async_controller, persister):
""" Test the process execute action """
loop = asyncio.get_event_loop()
loop_communicator.add_task_subscriber(
plumpy.ProcessLauncher(loop, persister=persister))
pid = await async_controller.execute_process(
utils.DummyProcessWithOutput, nowait=True)
assert isinstance(pid, uuid.UUID)
async def test_launch_nowait(self, loop_communicator, async_controller,
persister):
""" Testing launching but don't wait, just get the pid """
loop = asyncio.get_event_loop()
loop_communicator.add_task_subscriber(
plumpy.ProcessLauncher(loop, persister=persister))
pid = await async_controller.launch_process(utils.DummyProcess,
nowait=True)
assert isinstance(pid, uuid.UUID)
async def test_execute_action(self, loop_communicator, async_controller,
persister):
""" Test the process execute action """
loop = asyncio.get_event_loop()
loop_communicator.add_task_subscriber(
plumpy.ProcessLauncher(loop, persister=persister))
result = await async_controller.execute_process(
utils.DummyProcessWithOutput)
assert utils.DummyProcessWithOutput.EXPECTED_OUTPUTS == result
async def test_launch(self, loop_communicator, async_controller,
persister):
# Let the process run to the end
loop = asyncio.get_event_loop()
loop_communicator.add_task_subscriber(
plumpy.ProcessLauncher(loop, persister=persister))
result = await async_controller.launch_process(utils.DummyProcess)
# Check that we got a result
assert result == utils.DummyProcess.EXPECTED_OUTPUTS
def test_loader_is_used(self):
""" Make sure that the provided class loader is used by the process launcher """
loader = CustomObjectLoader()
proc = Process()
persister = plumpy.InMemoryPersister(loader=loader)
persister.save_checkpoint(proc)
launcher = plumpy.ProcessLauncher(persister=persister, loader=loader)
continue_task = plumpy.create_continue_body(proc.pid)
yield launcher._continue(None,
**continue_task[process_comms.TASK_ARGS])
async def test_loader_is_used():
""" Make sure that the provided class loader is used by the process launcher """
loader = CustomObjectLoader()
proc = Process()
persister = plumpy.InMemoryPersister(loader=loader)
persister.save_checkpoint(proc)
launcher = plumpy.ProcessLauncher(persister=persister, loader=loader)
continue_task = plumpy.create_continue_body(proc.pid)
result = await launcher._continue(None,
**continue_task[process_comms.TASK_ARGS])
assert result == utils.DummyProcess.EXPECTED_OUTPUTS
async def test_continue(self, loop_communicator, async_controller,
persister):
""" Test continuing a saved process """
loop = asyncio.get_event_loop()
loop_communicator.add_task_subscriber(
plumpy.ProcessLauncher(loop, persister=persister))
process = utils.DummyProcessWithOutput()
persister.save_checkpoint(process)
pid = process.pid
del process
# Let the process run to the end
result = await async_controller.continue_process(pid)
assert result, utils.DummyProcessWithOutput.EXPECTED_OUTPUTS
def test_continue(self):
persister = plumpy.InMemoryPersister()
load_context = plumpy.LoadSaveContext(loop=self.loop)
launcher = plumpy.ProcessLauncher(persister=persister,
load_context=load_context)
process = test_utils.DummyProcess(loop=self.loop)
pid = process.pid
persister.save_checkpoint(process)
del process
process = None
result = yield launcher._continue(
None,
**plumpy.create_continue_body(pid)[process_comms.TASK_ARGS])
self.assertEqual(test_utils.DummyProcess.EXPECTED_OUTPUTS, result)
async def test_continue():
persister = plumpy.InMemoryPersister()
load_context = plumpy.LoadSaveContext()
launcher = plumpy.ProcessLauncher(persister=persister,
load_context=load_context)
process = utils.DummyProcess()
pid = process.pid
persister.save_checkpoint(process)
del process
process = None
result = await launcher._continue(
None,
**plumpy.create_continue_body(pid)[process_comms.TASK_ARGS])
assert result == utils.DummyProcess.EXPECTED_OUTPUTS
async def test_launch_many(self, loop_communicator, async_controller,
persister):
"""Test launching multiple processes"""
loop = asyncio.get_event_loop()
loop_communicator.add_task_subscriber(
plumpy.ProcessLauncher(loop, persister=persister))
num_to_launch = 10
launch_futures = []
for _ in range(num_to_launch):
launch = async_controller.launch_process(utils.DummyProcess,
nowait=True)
launch_futures.append(launch)
results = await asyncio.gather(*launch_futures)
for result in results:
assert isinstance(result, uuid.UUID)