def start_listening(self, listener, event='*'):
if not event.startswith("process."):
raise ValueError(
"This emitter only knows about process.[pid[.event]] events")
super(ProcessMonitorEmitter, self).start_listening(listener, event)
if self.num_listening() > 0:
MONITOR.start_listening(self)
def __init__(self,
auto_persist=False,
running_directory=_RUNNING_DIRECTORY,
finished_directory=_FINISHED_DIRECTORY,
failed_directory=_FAILED_DIRECTORY):
"""
Create the pickle persistence object. If auto_persist is True then
this object will automatically persist any Processes that are created
and will keep their persisted state up to date as they run. By default
this is turned off as the user may prefer to manually specify which
process should be persisted.
The directory structure that will be used is:
running_directory/[pid].pickle - Currently active processes
finished_directory/[pid].pickle - Finished processes
failed_directory/[pid].pickle - Failed processes
:param auto_persist: Will automatically persist Processes if True.
:type auto_persist: bool
:param running_directory: The base directory to store all pickles in.
:type running_directory: str
:param finished_directory: The (relative) subdirectory to put finished
Process pickles in. If None they will be deleted when finished.
:type finished_directory: str
:param failed_directory: The (relative) subdirectory to put failed
Process pickles in. If None they will be deleted on fail.
:type failed_directory: str
"""
self._running_directory = running_directory
self._finished_directory = finished_directory
self._failed_directory = failed_directory
self._auto_persist = auto_persist
MONITOR.start_listening(self)
def play(self):
"""
Play the process.
"""
assert not self.__playing, \
"Cannot execute a process twice simultaneously"
try:
try:
MONITOR.register_process(self)
with self.__state_lock:
self._call_with_super_check(self.on_playing)
# Keep going until we run out of tasks
fn = self._next()
while fn is not None:
with self.__state_lock:
self._next_transition = None
fn()
fn = self._next()
except BaseException as e:
exc_type, value, tb = sys.exc_info()
self._perform_fail_noraise(e)
return
finally:
try:
MONITOR.deregister_process(self)
self._call_with_super_check(self.on_done_playing)
except BaseException as e:
if self.state != ProcessState.FAILED:
exc_type, value, tb = sys.exc_info()
self._perform_fail_noraise(e)
return self._outputs
def setUp(self):
import tempfile
self.assertEqual(len(MONITOR.get_pids()), 0)
self.store_dir = tempfile.mkdtemp()
self.pickle_persistence = PicklePersistence(
running_directory=self.store_dir)
def test_create_stop(self):
l = EventTracker()
with MONITOR.listen(l):
self.assertFalse(l.created_called)
self.assertFalse(l.stopped_called)
self.assertFalse(l.failed_called)
DummyProcess.run()
self.assertTrue(l.created_called)
self.assertTrue(l.stopped_called)
self.assertFalse(l.failed_called)
def _on_stop_playing(self):
"""
WARNING: No state changes should be made after this call.
"""
MONITOR.deregister_process(self)
stack.pop(self)
self.__play_lock.release()
try:
self._call_with_super_check(self.on_done_playing)
except BaseException:
# Only set failed if it hasn't already failed, otherwise
# we could obscure the real reason
if self.state != ProcessState.FAILED:
self._set_and_execute_state(Failed(self, sys.exc_info()))
raise
self.__event_helper.fire_event(ProcessListener.on_process_done_playing,
self)
if self.has_terminated():
# There will be no more messages so remove the listeners. Otherwise we
# may continue to hold references to them and stop them being garbage
# collected
self.__event_helper.remove_all_listeners()
def test_create_fail(self):
l = EventTracker()
with MONITOR.listen(l):
self.assertFalse(l.created_called)
self.assertFalse(l.stopped_called)
self.assertFalse(l.failed_called)
try:
ExceptionProcess.run()
except RuntimeError:
pass
except BaseException as e:
print(e.message)
self.assertTrue(l.created_called)
self.assertFalse(l.stopped_called)
self.assertTrue(l.failed_called)
def test_launch_simple(self):
class Tester(ProcessMonitorListener):
def __init__(self):
self.proc_class = None
self.stopped = False
def on_monitored_process_registered(self, process):
self.proc_class = process.__class__
def on_monitored_process_stopped(self, process):
self.stopped = True
t = Tester()
with MONITOR.listen(t):
self.manager.launch(DummyProcess)
while not t.stopped:
pass
self.assertIs(t.proc_class, DummyProcess)
def get_parent_calc(self):
from aiida.orm import load_node
# Can't get it if we don't know our parent
if self._parent_pid is None:
return None
# First try and get the process from the registry in case it is running
try:
return MONITOR.get_process(self._parent_pid).calc
except ValueError:
pass
# Ok, maybe the pid is actually a pk...
try:
return load_node(pk=self._parent_pid)
except exceptions.NotExistent:
pass
# Out of options
return None
def test_send(self):
class RanLogger(ProcessMonitorListener):
def __init__(self):
self.ran = []
def on_monitored_process_registered(self, process):
self.ran.append(process.__class__)
l = RanLogger()
with MONITOR.listen(l):
# Try sending some processes
for ProcClass in TEST_PROCESSES:
self.sender.send(ProcClass)
# Now make them run
num_ran = 0
while num_ran < len(TEST_PROCESSES):
num_ran += self.runner.poll(0.2)
self.assertEqual(num_ran, len(TEST_PROCESSES))
self.assertListEqual(TEST_PROCESSES, l.ran)
def test_launch(self):
class RanLogger(ProcessMonitorListener):
def __init__(self):
self.ran = []
def on_monitored_process_registered(self, process):
self.ran.append(process.__class__)
# Try launching some processes
for proc_class in TEST_PROCESSES:
self.publisher.launch(proc_class)
l = RanLogger()
with MONITOR.listen(l):
# Now make them run
num_ran = 0
for _ in range(0, 10):
num_ran += self.subscriber.poll(0.2)
if num_ran >= len(TEST_PROCESSES):
break
self.assertEqual(num_ran, len(TEST_PROCESSES))
self.assertListEqual(TEST_PROCESSES, l.ran)
def __init__(self):
MONITOR.add_monitor_listener(self)
def disable_persist_all(self):
"""
Stop persisting all ran processes
"""
MONITOR.stop_listening(self)
def tearDown(self):
self.assertEqual(len(MONITOR.get_pids()), 0)
self._empty_directory()
def tearDown(self):
self.manager.shutdown()
self.assertEqual(len(MONITOR.get_pids()), 0)
def setUp(self):
self.assertEqual(len(MONITOR.get_pids()), 0)
self.manager = ProcessManager()
def stop_listening(self, listener, event=None):
super(ProcessMonitorEmitter, self).stop_listening(listener, event)
if self.num_listening() == 0:
MONITOR.stop_listening(self)
def setUp(self):
self.assertEqual(len(MONITOR.get_pids()), 0)
self.executor = ThreadExecutor()
def disable_publish_all(self):
"""
Stop publishing messages from all run processes.
"""
MONITOR.stop_listening(self)
def enable_publish_all(self):
"""
Publish event messages from all run processes.
"""
MONITOR.start_listening(self)
def _on_start_playing(self):
self.__play_lock.acquire()
self.__interrupt_action = None
self._call_with_super_check(self.on_playing)
stack.push(self)
MONITOR.register_process(self)
def setUp(self):
self.assertEqual(len(MONITOR.get_pids()), 0,
"One or more processes are still running")
self.assertEqual(len(stack.stack()), 0, "The stack is not empty")
def enable_persist_all(self):
"""
Persist all processes that run
"""
MONITOR.start_listening(self)
def setUp(self):
self.assertEqual(
len(MONITOR.get_pids()), 0,
"One or more processes are still running")
def tearDown(self):
self.assertEqual(len(MONITOR.get_pids()), 0)
def setUp(self):
self.assertEqual(len(MONITOR.get_pids()), 0)