def main(): with kiwipy.connect( 'amqp://127.0.0.1') as communicator, tempfile.TemporaryDirectory( ) as tmpdir: loop = asyncio.get_event_loop() persister = plumpy.PicklePersister(tmpdir) task_receiver = plumpy.ProcessLauncher(loop=loop, persister=persister) def callback(*args, **kwargs): fut = plumpy.create_task(functools.partial(task_receiver, *args, **kwargs), loop=loop) return fut communicator.add_task_subscriber(callback) process_controller = plumpy.RemoteProcessThreadController(communicator) future = process_controller.launch_process(DummyProcessWithOutput) async def task(): result = await asyncio.wrap_future(future.result()) print(result) loop.run_until_complete(task())
async def test_execute_action_nowait(self, loop_communicator, async_controller, persister): """ Test the process execute action """ loop = asyncio.get_event_loop() loop_communicator.add_task_subscriber( plumpy.ProcessLauncher(loop, persister=persister)) pid = await async_controller.execute_process( utils.DummyProcessWithOutput, nowait=True) assert isinstance(pid, uuid.UUID)
async def test_launch_nowait(self, loop_communicator, async_controller, persister): """ Testing launching but don't wait, just get the pid """ loop = asyncio.get_event_loop() loop_communicator.add_task_subscriber( plumpy.ProcessLauncher(loop, persister=persister)) pid = await async_controller.launch_process(utils.DummyProcess, nowait=True) assert isinstance(pid, uuid.UUID)
async def test_execute_action(self, loop_communicator, async_controller, persister): """ Test the process execute action """ loop = asyncio.get_event_loop() loop_communicator.add_task_subscriber( plumpy.ProcessLauncher(loop, persister=persister)) result = await async_controller.execute_process( utils.DummyProcessWithOutput) assert utils.DummyProcessWithOutput.EXPECTED_OUTPUTS == result
async def test_launch(self, loop_communicator, async_controller, persister): # Let the process run to the end loop = asyncio.get_event_loop() loop_communicator.add_task_subscriber( plumpy.ProcessLauncher(loop, persister=persister)) result = await async_controller.launch_process(utils.DummyProcess) # Check that we got a result assert result == utils.DummyProcess.EXPECTED_OUTPUTS
def test_loader_is_used(self): """ Make sure that the provided class loader is used by the process launcher """ loader = CustomObjectLoader() proc = Process() persister = plumpy.InMemoryPersister(loader=loader) persister.save_checkpoint(proc) launcher = plumpy.ProcessLauncher(persister=persister, loader=loader) continue_task = plumpy.create_continue_body(proc.pid) yield launcher._continue(None, **continue_task[process_comms.TASK_ARGS])
async def test_loader_is_used(): """ Make sure that the provided class loader is used by the process launcher """ loader = CustomObjectLoader() proc = Process() persister = plumpy.InMemoryPersister(loader=loader) persister.save_checkpoint(proc) launcher = plumpy.ProcessLauncher(persister=persister, loader=loader) continue_task = plumpy.create_continue_body(proc.pid) result = await launcher._continue(None, **continue_task[process_comms.TASK_ARGS]) assert result == utils.DummyProcess.EXPECTED_OUTPUTS
async def test_continue(self, loop_communicator, async_controller, persister): """ Test continuing a saved process """ loop = asyncio.get_event_loop() loop_communicator.add_task_subscriber( plumpy.ProcessLauncher(loop, persister=persister)) process = utils.DummyProcessWithOutput() persister.save_checkpoint(process) pid = process.pid del process # Let the process run to the end result = await async_controller.continue_process(pid) assert result, utils.DummyProcessWithOutput.EXPECTED_OUTPUTS
def test_continue(self): persister = plumpy.InMemoryPersister() load_context = plumpy.LoadSaveContext(loop=self.loop) launcher = plumpy.ProcessLauncher(persister=persister, load_context=load_context) process = test_utils.DummyProcess(loop=self.loop) pid = process.pid persister.save_checkpoint(process) del process process = None result = yield launcher._continue( None, **plumpy.create_continue_body(pid)[process_comms.TASK_ARGS]) self.assertEqual(test_utils.DummyProcess.EXPECTED_OUTPUTS, result)
async def test_continue(): persister = plumpy.InMemoryPersister() load_context = plumpy.LoadSaveContext() launcher = plumpy.ProcessLauncher(persister=persister, load_context=load_context) process = utils.DummyProcess() pid = process.pid persister.save_checkpoint(process) del process process = None result = await launcher._continue( None, **plumpy.create_continue_body(pid)[process_comms.TASK_ARGS]) assert result == utils.DummyProcess.EXPECTED_OUTPUTS
async def test_launch_many(self, loop_communicator, async_controller, persister): """Test launching multiple processes""" loop = asyncio.get_event_loop() loop_communicator.add_task_subscriber( plumpy.ProcessLauncher(loop, persister=persister)) num_to_launch = 10 launch_futures = [] for _ in range(num_to_launch): launch = async_controller.launch_process(utils.DummyProcess, nowait=True) launch_futures.append(launch) results = await asyncio.gather(*launch_futures) for result in results: assert isinstance(result, uuid.UUID)