def send_exception(g, exc):
# note: send_exception(g, exc) can be now done with g.throw(exc).
# the purpose of this test is to explicitely check the propagation rules.
def crasher(exc):
raise exc
g1 = Fiber(target=crasher, args=(exc, ), parent=g)
g1.switch()
def test_finished_parent(self):
def f():
return 42
g = Fiber(f)
g.switch()
self.assertFalse(g.is_alive())
self.assertRaises(ValueError, Fiber, parent=g)
def test_finished_parent(self):
def f():
return 42
g = Fiber(f)
g.switch()
self.assertFalse(g.is_alive())
self.assertRaises(ValueError, Fiber, parent=g)
def send_exception(g, exc):
# note: send_exception(g, exc) can be now done with g.throw(exc).
# the purpose of this test is to explicitely check the propagation rules.
def crasher(exc):
raise exc
g1 = Fiber(target=crasher, args=(exc, ), parent=g)
g1.switch()
def test_arg_refs(self):
args = ('a', 'b', 'c')
refcount_before = sys.getrefcount(args)
g = Fiber(target=lambda *x: None, args=args)
self.assertEqual(sys.getrefcount(args), refcount_before+1)
g.switch()
self.assertEqual(sys.getrefcount(args), refcount_before)
del g
self.assertEqual(sys.getrefcount(args), refcount_before)
def f():
try:
raise ValueError('fun')
except:
exc_info = sys.exc_info()
t = Fiber(h)
t.switch()
self.assertEqual(exc_info, sys.exc_info())
del t
def test_kwarg_refs(self):
kwargs = {'a': 1234}
refcount_before = sys.getrefcount(kwargs)
g = Fiber(lambda **x: None, kwargs=kwargs)
self.assertEqual(sys.getrefcount(kwargs), refcount_before+1)
g.switch()
self.assertEqual(sys.getrefcount(kwargs), refcount_before)
del g
self.assertEqual(sys.getrefcount(kwargs), refcount_before)
def f():
try:
raise ValueError('fun')
except:
exc_info = sys.exc_info()
t = Fiber(h)
t.switch()
self.assertEqual(exc_info, sys.exc_info())
del t
def test_exception(self):
seen = []
g1 = Fiber(target=fmain, args=(seen, ))
g2 = Fiber(target=fmain, args=(seen, ))
g1.switch()
g2.switch()
g2.parent = g1
self.assertEqual(seen, [])
self.assertRaises(SomeError, g2.switch)
self.assertEqual(seen, [SomeError])
def test_exception(self):
seen = []
g1 = Fiber(target=fmain, args=(seen, ))
g2 = Fiber(target=fmain, args=(seen, ))
g1.switch()
g2.switch()
g2.parent = g1
self.assertEqual(seen, [])
self.assertRaises(SomeError, g2.switch)
self.assertEqual(seen, [SomeError])
def test_kwarg_refs(self):
if not has_refcount:
return
kwargs = {'a': 1234}
refcount_before = sys.getrefcount(kwargs)
g = Fiber(lambda **x: None, kwargs=kwargs)
self.assertEqual(sys.getrefcount(kwargs), refcount_before + 1)
g.switch()
self.assertEqual(sys.getrefcount(kwargs), refcount_before)
del g
self.assertEqual(sys.getrefcount(kwargs), refcount_before)
def test_arg_refs(self):
if not has_refcount:
return
args = ('a', 'b', 'c')
refcount_before = sys.getrefcount(args)
g = Fiber(target=lambda *x: None, args=args)
self.assertEqual(sys.getrefcount(args), refcount_before + 1)
g.switch()
self.assertEqual(sys.getrefcount(args), refcount_before)
del g
self.assertEqual(sys.getrefcount(args), refcount_before)
def test_simple2(self):
lst = []
def f():
lst.append(1)
current().parent.switch()
lst.append(3)
g = Fiber(f)
lst.append(0)
g.switch()
lst.append(2)
g.switch()
lst.append(4)
self.assertEqual(lst, list(range(5)))
def test_simple2(self):
lst = []
def f():
lst.append(1)
current().parent.switch()
lst.append(3)
g = Fiber(f)
lst.append(0)
g.switch()
lst.append(2)
g.switch()
lst.append(4)
self.assertEqual(lst, list(range(5)))
def test_two_recursive_children(self):
lst = []
def f():
lst.append(1)
current().parent.switch()
def h():
lst.append(1)
i = Fiber(f)
i.switch()
lst.append(1)
g = Fiber(h)
g.switch()
self.assertEqual(len(lst), 3)
def test_two_recursive_children(self):
lst = []
def f():
lst.append(1)
current().parent.switch()
def h():
lst.append(1)
i = Fiber(f)
i.switch()
lst.append(1)
g = Fiber(h)
g.switch()
self.assertEqual(len(lst), 3)
def test_exc_state(self):
def f():
try:
raise ValueError('fun')
except:
exc_info = sys.exc_info()
t = Fiber(h)
t.switch()
self.assertEqual(exc_info, sys.exc_info())
del t
def h():
self.assertEqual(sys.exc_info(), (None, None, None))
g = Fiber(f)
g.switch()
def test_exc_state(self):
def f():
try:
raise ValueError('fun')
except:
exc_info = sys.exc_info()
t = Fiber(h)
t.switch()
self.assertEqual(exc_info, sys.exc_info())
del t
def h():
self.assertEqual(sys.exc_info(), (None, None, None))
g = Fiber(f)
g.switch()
def test_instance_dict(self):
def f():
current().test = 42
def deldict(g):
del g.__dict__
def setdict(g, value):
g.__dict__ = value
g = Fiber(f)
self.assertEqual(g.__dict__, {})
g.switch()
self.assertEqual(g.test, 42)
self.assertEqual(g.__dict__, {'test': 42})
g.__dict__ = g.__dict__
self.assertEqual(g.__dict__, {'test': 42})
self.assertRaises(AttributeError, deldict, g)
self.assertRaises(TypeError, setdict, g, 42)
def test_instance_dict(self):
if is_pypy:
return
def f():
current().test = 42
def deldict(g):
del g.__dict__
def setdict(g, value):
g.__dict__ = value
g = Fiber(f)
self.assertEqual(g.__dict__, {})
g.switch()
self.assertEqual(g.test, 42)
self.assertEqual(g.__dict__, {'test': 42})
g.__dict__ = g.__dict__
self.assertEqual(g.__dict__, {'test': 42})
self.assertRaises(AttributeError, deldict, g)
self.assertRaises(TypeError, setdict, g, 42)
def test_throw_goes_to_original_parent(self):
main = fibers.current()
def f1():
try:
main.switch("f1 ready to catch")
except IndexError:
return "caught"
else:
return "normal exit"
def f2():
main.switch("from f2")
g1 = Fiber(f1)
g2 = Fiber(target=f2, parent=g1)
self.assertRaises(IndexError, g2.throw, IndexError)
self.assertFalse(g2.is_alive())
self.assertTrue(g1.is_alive()) # g1 is skipped because it was not started
g1 = Fiber(f1)
g2 = Fiber(target=f2, parent=g1)
res = g1.switch()
self.assertEqual(res, "f1 ready to catch")
res = g2.throw(IndexError)
self.assertEqual(res, "caught")
self.assertFalse(g2.is_alive())
self.assertFalse(g1.is_alive())
g1 = Fiber(f1)
g2 = Fiber(target=f2, parent=g1)
res = g1.switch()
self.assertEqual(res, "f1 ready to catch")
res = g2.switch()
self.assertEqual(res, "from f2")
res = g2.throw(IndexError)
self.assertEqual(res, "caught")
self.assertFalse(g2.is_alive())
self.assertFalse(g1.is_alive())
def test_val(self):
def f():
try:
switch("ok")
except RuntimeError:
val = sys.exc_info()[1]
if str(val) == "ciao":
switch("ok")
return
switch("fail")
g = Fiber(f)
res = g.switch()
self.assertEqual(res, "ok")
res = g.throw(RuntimeError("ciao"))
self.assertEqual(res, "ok")
g = Fiber(f)
res = g.switch()
self.assertEqual(res, "ok")
res = g.throw(RuntimeError, "ciao")
self.assertEqual(res, "ok")
def test_kill(self):
def f():
try:
switch("ok")
switch("fail")
except Exception as e:
return e
g = Fiber(f)
res = g.switch()
self.assertEqual(res, "ok")
res = g.throw(ValueError)
self.assertTrue(isinstance(res, ValueError))
self.assertFalse(g.is_alive())
def test_class(self):
def f():
try:
switch("ok")
except RuntimeError:
switch("ok")
return
switch("fail")
g = Fiber(f)
res = g.switch()
self.assertEqual(res, "ok")
res = g.throw(RuntimeError)
self.assertEqual(res, "ok")
def test_finalizer_crash(self):
# This test is designed to crash when active greenlets
# are made garbage collectable, until the underlying
# problem is resolved. How does it work:
# - order of object creation is important
# - array is created first, so it is moved to unreachable first
# - we create a cycle between a greenlet and this array
# - we create an object that participates in gc, is only
# referenced by a greenlet, and would corrupt gc lists
# on destruction, the easiest is to use an object with
# a finalizer
# - because array is the first object in unreachable it is
# cleared first, which causes all references to greenlet
# to disappear and causes greenlet to be destroyed, but since
# it is still live it causes a switch during gc, which causes
# an object with finalizer to be destroyed, which causes stack
# corruption and then a crash
class object_with_finalizer(object):
def __del__(self):
pass
array = []
parent = current()
def greenlet_body():
current().object = object_with_finalizer()
try:
parent.switch()
finally:
del current().object
g = Fiber(greenlet_body)
g.array = array
array.append(g)
g.switch()
del array
del g
current()
gc.collect()
def test_val(self):
def f():
try:
switch("ok")
except RuntimeError:
val = sys.exc_info()[1]
if str(val) == "ciao":
switch("ok")
return
switch("fail")
g = Fiber(f)
res = g.switch()
self.assertEqual(res, "ok")
res = g.throw(RuntimeError("ciao"))
self.assertEqual(res, "ok")
g = Fiber(f)
res = g.switch()
self.assertEqual(res, "ok")
res = g.throw(RuntimeError, "ciao")
self.assertEqual(res, "ok")
def test_class(self):
def f():
try:
switch("ok")
except RuntimeError:
switch("ok")
return
switch("fail")
g = Fiber(f)
res = g.switch()
self.assertEqual(res, "ok")
res = g.throw(RuntimeError)
self.assertEqual(res, "ok")
def test_kill(self):
def f():
try:
switch("ok")
switch("fail")
except Exception as e:
return e
g = Fiber(f)
res = g.switch()
self.assertEqual(res, "ok")
res = g.throw(ValueError)
self.assertTrue(isinstance(res, ValueError))
self.assertFalse(g.is_alive())
def test_threaded_reparent(self):
data = {}
created_event = threading.Event()
done_event = threading.Event()
def foo():
data['g'] = Fiber(lambda: None)
created_event.set()
done_event.wait()
def blank():
current().parent.switch()
def setparent(g, value):
g.parent = value
thread = threading.Thread(target=foo)
thread.start()
created_event.wait()
g = Fiber(blank)
g.switch()
self.assertRaises(ValueError, setparent, g, data['g'])
done_event.set()
thread.join()
def test_throw_goes_to_original_parent3(self):
main = fibers.current()
def f1():
try:
main.switch("f1 ready to catch")
except IndexError:
return "caught"
else:
return "normal exit"
def f2():
main.switch("from f2")
g1 = Fiber(f1)
g2 = Fiber(target=f2, parent=g1)
res = g1.switch()
self.assertEqual(res, "f1 ready to catch")
res = g2.switch()
self.assertEqual(res, "from f2")
res = g2.throw(IndexError)
self.assertEqual(res, "caught")
self.assertFalse(g2.is_alive())
self.assertFalse(g1.is_alive())
def test_threaded_reparent(self):
data = {}
created_event = threading.Event()
done_event = threading.Event()
def foo():
data['g'] = Fiber(lambda: None)
created_event.set()
done_event.wait()
def blank():
current().parent.switch()
def setparent(g, value):
g.parent = value
thread = threading.Thread(target=foo)
thread.start()
created_event.wait()
g = Fiber(blank)
g.switch()
self.assertRaises(ValueError, setparent, g, data['g'])
done_event.set()
thread.join()
def test_finalizer_crash(self):
# This test is designed to crash when active greenlets
# are made garbage collectable, until the underlying
# problem is resolved. How does it work:
# - order of object creation is important
# - array is created first, so it is moved to unreachable first
# - we create a cycle between a greenlet and this array
# - we create an object that participates in gc, is only
# referenced by a greenlet, and would corrupt gc lists
# on destruction, the easiest is to use an object with
# a finalizer
# - because array is the first object in unreachable it is
# cleared first, which causes all references to greenlet
# to disappear and causes greenlet to be destroyed, but since
# it is still live it causes a switch during gc, which causes
# an object with finalizer to be destroyed, which causes stack
# corruption and then a crash
class object_with_finalizer(object):
def __del__(self):
pass
array = []
parent = current()
def greenlet_body():
current().object = object_with_finalizer()
try:
parent.switch()
finally:
del current().object
g = Fiber(greenlet_body)
g.array = array
array.append(g)
g.switch()
del array
del g
current()
gc.collect()
def test_two_children(self):
lst = []
def f():
lst.append(1)
current().parent.switch()
lst.extend([1, 1])
g = Fiber(f)
h = Fiber(f)
g.switch()
self.assertEqual(len(lst), 1)
h.switch()
self.assertEqual(len(lst), 2)
h.switch()
self.assertEqual(len(lst), 4)
self.assertEqual(h.is_alive(), False)
g.switch()
self.assertEqual(len(lst), 6)
self.assertEqual(g.is_alive(), False)
def test_two_children(self):
lst = []
def f():
lst.append(1)
current().parent.switch()
lst.extend([1, 1])
g = Fiber(f)
h = Fiber(f)
g.switch()
self.assertEqual(len(lst), 1)
h.switch()
self.assertEqual(len(lst), 2)
h.switch()
self.assertEqual(len(lst), 4)
self.assertEqual(h.is_alive(), False)
g.switch()
self.assertEqual(len(lst), 6)
self.assertEqual(g.is_alive(), False)
class Server(object):
_instance = None
@classmethod
def _get(cls):
if cls._instance is None:
raise Exception("use Server.start() method")
return cls._instance
def __init__(self, logger=None):
"""
Note
---
Do not call the constructor directory.
Instead, use `Server.start` to launch a server.
"""
self.observed_ps = defaultdict(list)
self.observed_all_ps = defaultdict(list)
self.observed_task = defaultdict(list)
self.observed_all_tasks = defaultdict(
list) # (task_id) => list of (task_ids, callback)
self.max_submitted_task_id = 0
self._logger = logger or self._default_logger()
self._fibers = []
self._comm = None
@classmethod
def start(cls, logger=None):
"""
start a scheduling of tasks.
Examples
---
>>> with Server.start():
....
"""
cls._instance = cls(logger)
return cls._instance
def __enter__(self):
self._loop_fiber = Fiber(target=self._loop)
self._comm = MPI.Comm.Get_parent()
self._logger.debug("accepted")
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
return False # re-raise exception
if self._loop_fiber.is_alive():
self._loop_fiber.switch()
self._comm.Disconnect()
@classmethod
def watch_ps(cls, ps, callback):
"""
add a callback function when ps gets completed.
Parameters
---
ps : ParameterSet
callback : callable
"""
cls._get().observed_ps[ps.id()].append(callback)
@classmethod
def watch_all_ps(cls, ps_set, callback):
"""
add a callback function when a set of ParameterSets is completed.
Parameters
---
ps_set : list of ParameterSet
callback : callable
"""
ids = [ps.id() for ps in ps_set]
key = tuple(ids)
cls._get().observed_all_ps[key].append(callback)
@classmethod
def watch_task(cls, task, callback):
"""
add a callback function when a Task is completed.
Parameters
---
task : Task
callback : callable
"""
cls._get().observed_task[task.id()].append(callback)
@classmethod
def watch_all_tasks(cls, tasks, callback):
"""
add a callback function when a set of Tasks is completed.
Parameters
---
tasks : list of Task
callback : callable
"""
key = tuple([t.id() for t in tasks])
for t in tasks:
pair = (key, callback)
cls._get().observed_all_tasks[t.id()].append(pair)
@classmethod
def do_async(cls, func, *args, **kwargs):
"""
do coroutine asynchronously.
Examples
---
Server.do_async( lambda : Task.create(...) )
Parameters
---
tasks : list of Task
callback : callable
"""
self = cls._get()
def _f():
func(*args, **kwargs)
self._loop_fiber.switch()
fb = Fiber(target=_f)
self._fibers.append(fb)
@classmethod
def await_ps(cls, ps):
"""
wait until parameterSet is complete.
During waiting, other co-routines are concurrently executed.
Parameters
---
ps : ParameterSet
"""
self = cls._get()
fb = Fiber.current()
def _callback():
self._fibers.append(fb)
cls.watch_ps(ps, _callback)
self._loop_fiber.switch()
@classmethod
def await_all_ps(cls, ps_set):
"""
wait until a set of ParameterSets is complete.
While waiting, other co-routines are concurrently executed.
Parameters
---
ps_set : list of ParameterSet
"""
self = cls._get()
fb = Fiber.current()
def _callback():
self._fibers.append(fb)
cls.watch_all_ps(ps_set, _callback)
self._loop_fiber.switch()
@classmethod
def await_task(cls, task):
"""
wait until a Task is complete.
Parameters
---
task : Task
"""
self = cls._get()
fb = Fiber.current()
def _callback():
self._fibers.append(fb)
cls.watch_task(task, _callback)
self._loop_fiber.switch()
@classmethod
def await_all_tasks(cls, tasks):
"""
wait until a set of Tasks is complete.
Parameters
---
tasks : list of Task
"""
self = cls._get()
fb = Fiber.current()
def _callback():
self._fibers.append(fb)
cls.watch_all_tasks(tasks, _callback)
self._loop_fiber.switch()
def _loop(self):
self._launch_all_fibers()
self._submit_all()
self._logger.debug("start polling")
t = self._receive_result()
while t:
self._exec_callback_for_task(t)
self._exec_callback_for_all_task(t)
if isinstance(t, Run):
ps = t.parameter_set()
if ps.is_finished():
self._exec_callback()
self._submit_all()
t = self._receive_result()
def _default_logger(self):
logger = logging.getLogger(__name__)
log_level = logging.INFO
if 'CARAVAN_SEARCH_ENGINE_LOGLEVEL' in os.environ:
s = os.environ['CARAVAN_SEARCH_ENGINE_LOGLEVEL']
levels = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL
}
log_level = levels[s]
logger.setLevel(log_level)
logger.propagate = False
if not logger.handlers:
ch = logging.StreamHandler()
ch.setLevel(log_level)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
return logger
def _has_callbacks(self):
return (len(self.observed_ps) + len(self.observed_all_ps)) > 0
def _has_unfinished_tasks(self):
for r in Task.all()[:self.max_submitted_task_id]:
if not r.is_finished():
return True
return False
def _submit_all(self):
tasks_to_be_submitted = [
t for t in Task.all()[self.max_submitted_task_id:]
if not t.is_finished()
]
self._logger.debug("submitting %d Tasks" % len(tasks_to_be_submitted))
self._print_tasks(tasks_to_be_submitted)
self.max_submitted_task_id = len(Task.all())
def _print_tasks(self, tasks):
b_tasks = [{
"id": t.id(),
"cmd": t.command(),
"input": t.input()
} for t in tasks]
packed = msgpack.packb(b_tasks)
self._comm.Send([bytearray(packed), MPI.CHAR], dest=0, tag=0)
self._logger.debug(f"sent packed")
def _launch_all_fibers(self):
while self._fibers:
f = self._fibers.pop(0)
self._logger.debug("starting fiber")
f.switch()
def _exec_callback(self):
while self._check_completed_ps() or self._check_completed_ps_all():
pass
def _check_completed_ps(self):
executed = False
for psid in list(self.observed_ps.keys()):
callbacks = self.observed_ps[psid]
ps = ParameterSet.find(psid)
while ps.is_finished() and len(callbacks) > 0:
self._logger.debug("executing callback for ParameterSet %d" %
ps.id())
f = callbacks.pop(0)
f()
self._launch_all_fibers()
executed = True
empty_keys = [k for k, v in self.observed_ps.items() if len(v) == 0]
for k in empty_keys:
self.observed_ps.pop(k)
return executed
def _check_completed_ps_all(self):
executed = False
for psids in list(self.observed_all_ps.keys()):
pss = [ParameterSet.find(psid) for psid in psids]
callbacks = self.observed_all_ps[psids]
while len(callbacks) > 0 and all([ps.is_finished() for ps in pss]):
self._logger.debug("executing callback for ParameterSet %s" %
repr(psids))
f = callbacks.pop(0)
f()
self._launch_all_fibers()
executed = True
empty_keys = [
k for k, v in self.observed_all_ps.items() if len(v) == 0
]
for k in empty_keys:
self.observed_all_ps.pop(k)
return executed
def _exec_callback_for_task(self, task):
executed = False
callbacks = self.observed_task[task.id()]
while len(callbacks) > 0:
self._logger.debug("executing callback for Task %d" % task.id())
f = callbacks.pop(0)
f()
self._launch_all_fibers()
executed = True
self.observed_task.pop(task.id())
return executed
def _exec_callback_for_all_task(self, task):
executed = False
callback_pairs = self.observed_all_tasks[task.id()]
to_be_removed = []
for (idx, pair) in enumerate(callback_pairs):
task_ids = pair[0]
if all([Task.find(t).is_finished() for t in task_ids]):
self._logger.debug("executing callback for Tasks %s" %
str(task_ids))
f = pair[1]
f()
to_be_removed.append(idx)
self._launch_all_fibers()
executed = True
for idx in to_be_removed:
callback_pairs.pop(idx)
if len(callback_pairs) == 0:
self.observed_all_tasks.pop(task.id())
return executed
def _receive_bytes(self):
s = MPI.Status()
# instead of `comm.Probe`, `Iprobe` is used to avoid a busy wait
while not self._comm.Iprobe(status=s):
time.sleep(0.01)
if s.count == 0: assert s.tag == 1
recvbuf = bytearray(s.count)
self._comm.Recv([recvbuf, s.count, MPI.CHAR],
source=s.source,
tag=s.tag)
return recvbuf
def _receive_result(self):
data_b = self._receive_bytes()
if len(data_b) == 0: return None
self._logger.debug("received: %s bytes" % len(data_b))
unpacked = msgpack.unpackb(data_b)
self._logger.debug("received: %s" % str(unpacked))
tid = unpacked["id"]
rc = unpacked["rc"]
rank = unpacked["rank"]
start_at = unpacked["start_at"]
finish_at = unpacked["finish_at"]
output = unpacked["output"]
t = Task.find(tid)
t._store_result(output, rc, rank, start_at, finish_at)
self._logger.debug("stored result of Task %d" % tid)
return t
def _debug(self):
sys.stderr.write(str(self.observed_ps) + "\n")
sys.stderr.write(str(self.observed_all_ps) + "\n")
def _dead_fiber():
g = Fiber(lambda: None)
g.switch()
return g
def h():
lst.append(1)
i = Fiber(f)
i.switch()
lst.append(1)
def test_send_exception(self):
seen = []
g1 = Fiber(target=fmain, args=(seen, ))
g1.switch()
self.assertRaises(KeyError, send_exception, g1, KeyError)
self.assertEqual(seen, [KeyError])
def runner(x):
g = Fiber(lambda: time.sleep(x))
g.switch()
def creator():
g = Fiber(worker)
g.switch()
result.append(g)
def runner(x):
g = Fiber(lambda: time.sleep(x))
g.switch()
def test_send_exception(self):
seen = []
g1 = Fiber(target=fmain, args=(seen, ))
g1.switch()
self.assertRaises(KeyError, send_exception, g1, KeyError)
self.assertEqual(seen, [KeyError])
class StubServer(Server):
def __init__(self, stub_simulator, num_proc, logger, dump_path):
"""
Note
---
Do not call the constructor directory.
Instead, use `StubServer.start` to launch a server.
"""
super().__init__(logger)
self._stub_simulator = stub_simulator
self._num_proc = num_proc
self._dump_path = dump_path
self._queue = EventQueue(self._num_proc)
@classmethod
def start(cls,
stub_simulator,
num_proc,
logger=None,
dump_path='tasks.msgpack'):
"""
Start a scheduling of tasks with stub simulator.
This is provided for testing search engines without actually running the simulations.
Instead of actually conducting simulations, it mimics the event using a dummy simulator `stub_simulator`.
`stub_simulator` is a function which receives a Task as its argument and returns a tuple of (output, dt).
Here, `output` is a json-like object and `dt` is a duration of simulation in seconds.
It simulates the task scheduling assuming that the tasks are parallely executed by `num_proc` processes.
Examples
---
>>> def stub_sim(task):
return ({"res1":1.0, "res2":2.0}, 100)
>>> wtih StubServer.start(stub_sim, 8):
....
"""
Server._instance = cls(stub_simulator, num_proc, logger, dump_path)
return Server._instance
# override the methods
def _print_tasks(self, tasks):
for t in tasks:
res, dt = self._stub_simulator(t)
t._output = res
t._dt = int(1000 * dt)
self._queue.push_all(tasks)
def _receive_result(self):
t = self._queue.pop()
if t is None:
return None
t._rc = 0
return t
def __enter__(self):
self._loop_fiber = Fiber(target=self._loop)
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
return False # re-raise exception
if self._loop_fiber.is_alive():
self._loop_fiber.switch()
Task._dump_binary(self._dump_path)
def h():
lst.append(1)
i = Fiber(f)
i.switch()
lst.append(1)
def creator():
g = Fiber(worker)
g.switch()
result.append(g)
def _dead_fiber():
g = Fiber(lambda: None)
g.switch()
return g
class EventLoop(object):
def __init__(self):
if getattr(_tls, 'loop', None) is not None:
raise RuntimeError(
'cannot instantiate more than one event loop per thread')
_tls.loop = self
self._loop = pyuv.Loop()
self._loop.excepthook = self._handle_error
self._loop.event_loop = self
self._threadpool = ThreadPool(self)
self.task = Fiber(self._run_loop)
self._destroyed = False
self._started = False
self._running = False
self._fd_map = dict()
self._signals = dict()
self._timers = set()
self._ready = deque()
self._ready_processor = pyuv.Idle(self._loop)
self._waker = pyuv.Async(self._loop, self._async_cb)
self._waker.unref()
self._install_signal_checker()
@classmethod
def current(cls):
"""Get the current event loop singleton object.
"""
try:
return _tls.loop
except AttributeError:
# create loop only for main thread
if threading.current_thread().name == 'MainThread':
_tls.loop = cls()
return _tls.loop
raise RuntimeError(
'there is no event loop created in the current thread')
@property
def running(self):
return self._running
def call_soon(self, callback, *args, **kw):
handler = Handler(callback, *args, **kw)
self._add_callback(handler)
return handler
def call_from_thread(self, callback, *args, **kw):
handler = Handler(callback, *args, **kw)
# Here we don't call self._add_callback on purpose, because it's not thread
# safe to start pyuv handles. We just append the callback to the queue and
# wakeup the loop. This is thread safe because the queue is only processed
# in a single place and in a thread safe manner.
self._ready.append(handler)
self._waker.send()
return handler
def call_later(self, delay, callback, *args, **kw):
if delay <= 0:
return self.call_soon(callback, *args, **kw)
timer_h = pyuv.Timer(self._loop)
handler = Timer(timer_h, callback, *args, **kw)
timer_h.handler = handler
timer_h.start(self._timer_cb, delay, 0)
self._timers.add(timer_h)
return handler
def call_at(self, when, callback, *args, **kw):
return self.call_later(when - self.time(), callback, *args, **kw)
def time(self):
return _time()
def add_reader(self, fd, callback, *args, **kw):
handler = Handler(callback, *args, **kw)
try:
poll_h = self._fd_map[fd]
except KeyError:
poll_h = self._create_poll_handle(fd)
self._fd_map[fd] = poll_h
else:
if poll_h.read_handler:
raise RuntimeError(
'another reader is already registered for fd {}'.format(
fd))
poll_h.pevents |= pyuv.UV_READABLE
poll_h.read_handler = handler
poll_h.start(poll_h.pevents, self._poll_cb)
def remove_reader(self, fd):
try:
poll_h = self._fd_map[fd]
except KeyError:
return False
else:
handler = poll_h.read_handler
poll_h.pevents &= ~pyuv.UV_READABLE
poll_h.read_handler = None
if poll_h.pevents == 0:
poll_h.close()
del self._fd_map[fd]
else:
poll_h.start(poll_h.pevents, self._poll_cb)
if handler:
handler.cancel()
return True
return False
def add_writer(self, fd, callback, *args, **kw):
handler = Handler(callback, *args, **kw)
try:
poll_h = self._fd_map[fd]
except KeyError:
poll_h = self._create_poll_handle(fd)
self._fd_map[fd] = poll_h
else:
if poll_h.write_handler:
raise RuntimeError(
'another writer is already registered for fd {}'.format(
fd))
poll_h.pevents |= pyuv.UV_WRITABLE
poll_h.write_handler = handler
poll_h.start(poll_h.pevents, self._poll_cb)
def remove_writer(self, fd):
try:
poll_h = self._fd_map[fd]
except KeyError:
return False
else:
handler = poll_h.write_handler
poll_h.pevents &= ~pyuv.UV_WRITABLE
poll_h.write_handler = None
if poll_h.pevents == 0:
poll_h.close()
del self._fd_map[fd]
else:
poll_h.start(poll_h.pevents, self._poll_cb)
if handler:
handler.cancel()
return True
return False
def add_signal_handler(self, sig, callback, *args, **kwargs):
self._validate_signal(sig)
signal_h = pyuv.Signal(self._loop)
handler = SignalHandler(signal_h, callback, *args, **kwargs)
signal_h.handler = handler
signal_h.signum = sig
try:
signal_h.start(self._signal_cb, sig)
signal_h.unref()
except Exception as e:
signal_h.close()
raise RuntimeError(str(e))
else:
self._signals.setdefault(sig, set()).add(signal_h)
return handler
def remove_signal_handler(self, sig):
self._validate_signal(sig)
try:
handles = self._signals.pop(sig)
except KeyError:
return False
for signal_h in handles:
del signal_h.handler
signal_h.close()
return True
def switch(self):
if not self._started:
self._run(forever=False)
return
current = Fiber.current()
assert current is not self.task, 'Cannot switch to MAIN from MAIN'
try:
if self.task.parent is not current:
current.parent = self.task
except ValueError:
pass # gets raised if there is a Fiber parent cycle
return self.task.switch()
def run(self, mode=RUN_DEFAULT):
if Fiber.current() is not self.task.parent:
raise RuntimeError('run() can only be called from MAIN fiber')
if not self.task.is_alive():
raise RuntimeError('event loop has already ended')
if self._started:
raise RuntimeError('event loop was already started')
self._started = True
self._running = True
self._run_mode = mode
try:
self.task.switch()
finally:
self._running = False
def run_forever(self):
self.run(mode=RUN_FOREVER)
def stop(self):
if not self._started:
raise RuntimeError('event loop has not been started yet')
if self._loop:
self._loop.stop()
def destroy(self):
if self._running:
raise RuntimeError(
'destroy() cannot be called while the loop is running')
if self._destroyed:
raise RuntimeError('Event loop already destroyed')
loop = getattr(_tls, 'loop', None)
if loop is not self:
raise RuntimeError(
'destroy() can only be called from the same thread were the event loop was created'
)
del _tls.loop, loop
self._destroyed = True
self._uninstall_signal_checker()
self._cleanup_loop()
self._loop.event_loop = None
self._loop.excepthook = None
self._loop = None
self._threadpool = None
self._ready_processor = None
self._waker = None
self._fd_map.clear()
self._signals.clear()
self._timers.clear()
self._ready.clear()
# internal
def _add_callback(self, cb):
self._ready.append(cb)
if not self._ready_processor.active:
self._ready_processor.start(self._process_ready)
def _handle_error(self, typ, value, tb):
if not issubclass(typ, SystemExit):
traceback.print_exception(typ, value, tb)
if issubclass(typ, (KeyboardInterrupt, SystemExit, SystemError)):
assert Fiber.current() is self.task
self.task.parent.throw(typ, value, tb)
def _run_loop(self):
if self._run_mode == RUN_FOREVER:
self._waker.ref()
self._loop.run(pyuv.UV_RUN_DEFAULT)
def _cleanup_loop(self):
def cb(handle):
if not handle.closed:
handle.close()
self._loop.walk(cb)
# All handles are now closed, run will not block
self._loop.run(pyuv.UV_RUN_NOWAIT)
def _create_poll_handle(self, fd):
poll_h = pyuv.Poll(self._loop, fd)
poll_h.pevents = 0
poll_h.read_handler = None
poll_h.write_handler = None
return poll_h
def _process_ready(self, handle):
# Run all queued callbacks
ntodo = len(self._ready)
for x in range(ntodo):
handler = self._ready.popleft()
if not handler._cancelled:
# loop.excepthook takes care of exception handling
handler()
if not self._ready:
self._ready_processor.stop()
def _async_cb(self, handle):
if not self._ready_processor.active:
self._ready_processor.start(self._process_ready)
def _timer_cb(self, timer):
assert not timer.handler._cancelled
self._add_callback(timer.handler)
if not timer.repeat:
timer.close()
self._timers.remove(timer)
del timer.handler
def _signal_cb(self, signal_h, signum):
self._add_callback(signal_h.handler)
def _poll_cb(self, poll_h, events, error):
fd = poll_h.fileno()
if error is not None:
# An error happened, signal both readability and writability and
# let the error propagate
if poll_h.read_handler is not None:
if poll_h.read_handler._cancelled:
self.remove_reader(fd)
else:
self._add_callback(poll_h.read_handler)
if poll_h.write_handler is not None:
if poll_h.write_handler._cancelled:
self.remove_writer(fd)
else:
self._add_callback(poll_h.write_handler)
return
old_events = poll_h.pevents
modified = False
if events & pyuv.UV_READABLE:
if poll_h.read_handler is not None:
if poll_h.read_handler._cancelled:
self.remove_reader(fd)
modified = True
else:
self._add_callback(poll_h.read_handler)
else:
poll_h.pevents &= ~pyuv.UV_READABLE
if events & pyuv.UV_WRITABLE:
if poll_h.write_handler is not None:
if poll_h.write_handler._cancelled:
self.remove_writer(fd)
modified = True
else:
self._add_callback(poll_h.write_handler)
else:
poll_h.pevents &= ~pyuv.UV_WRITABLE
if not modified and old_events != poll_h.pevents:
# Rearm the handle
poll_h.start(poll_h.pevents, self._poll_cb)
def _install_signal_checker(self):
self._socketpair = SocketPair()
self._signal_checker = None
if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix':
try:
old_wakeup_fd = signal.set_wakeup_fd(
self._socketpair.writer_fileno())
if old_wakeup_fd != -1:
# Already set, restore it
signal.set_wakeup_fd(old_wakeup_fd)
self._socketpair.close()
self._socketpair = None
else:
self._signal_checker = pyuv.util.SignalChecker(
self._loop, self._socketpair.reader_fileno())
self._signal_checker.start()
self._signal_checker.unref()
except ValueError:
self._socketpair.close()
self._socketpair = None
def _uninstall_signal_checker(self):
if self._signal_checker:
self._signal_checker.close()
self._signal_checker = None
if self._socketpair:
self._socketpair.close()
self._socketpair = None
def _validate_signal(self, sig):
if not isinstance(sig, int):
raise TypeError('sig must be an int, not {!r}'.format(sig))
if signal is None:
raise RuntimeError('Signals are not supported')
if not (1 <= sig < signal.NSIG):
raise ValueError('sig {} out of range(1, {})'.format(
sig, signal.NSIG))
class EventLoop(object):
def __init__(self):
if getattr(_tls, 'loop', None) is not None:
raise RuntimeError('cannot instantiate more than one event loop per thread')
_tls.loop = self
self._loop = pyuv.Loop()
self._loop.excepthook = self._handle_error
self._loop.event_loop = self
self._threadpool = ThreadPool(self)
self.task = Fiber(self._run_loop)
self._destroyed = False
self._started = False
self._running = False
self._fd_map = dict()
self._signals = dict()
self._timers = set()
self._ready = deque()
self._ready_processor = pyuv.Idle(self._loop)
self._waker = pyuv.Async(self._loop, self._async_cb)
self._waker.unref()
self._install_signal_checker()
@classmethod
def current(cls):
"""Get the current event loop singleton object.
"""
try:
return _tls.loop
except AttributeError:
# create loop only for main thread
if threading.current_thread().name == 'MainThread':
_tls.loop = cls()
return _tls.loop
raise RuntimeError('there is no event loop created in the current thread')
@property
def running(self):
return self._running
def call_soon(self, callback, *args, **kw):
handler = Handler(callback, *args, **kw)
self._add_callback(handler)
return handler
def call_from_thread(self, callback, *args, **kw):
handler = Handler(callback, *args, **kw)
# Here we don't call self._add_callback on purpose, because it's not thread
# safe to start pyuv handles. We just append the callback to the queue and
# wakeup the loop. This is thread safe because the queue is only processed
# in a single place and in a thread safe manner.
self._ready.append(handler)
self._waker.send()
return handler
def call_later(self, delay, callback, *args, **kw):
if delay <= 0:
return self.call_soon(callback, *args, **kw)
timer_h = pyuv.Timer(self._loop)
handler = Timer(timer_h, callback, *args, **kw)
timer_h.handler = handler
timer_h.start(self._timer_cb, delay, 0)
self._timers.add(timer_h)
return handler
def call_at(self, when, callback, *args, **kw):
return self.call_later(when-self.time(), callback, *args, **kw)
def time(self):
return _time()
def add_reader(self, fd, callback, *args, **kw):
handler = Handler(callback, *args, **kw)
try:
poll_h = self._fd_map[fd]
except KeyError:
poll_h = self._create_poll_handle(fd)
self._fd_map[fd] = poll_h
else:
if poll_h.read_handler:
raise RuntimeError('another reader is already registered for fd {}'.format(fd))
poll_h.pevents |= pyuv.UV_READABLE
poll_h.read_handler = handler
poll_h.start(poll_h.pevents, self._poll_cb)
def remove_reader(self, fd):
try:
poll_h = self._fd_map[fd]
except KeyError:
return False
else:
handler = poll_h.read_handler
poll_h.pevents &= ~pyuv.UV_READABLE
poll_h.read_handler = None
if poll_h.pevents == 0:
poll_h.close()
del self._fd_map[fd]
else:
poll_h.start(poll_h.pevents, self._poll_cb)
if handler:
handler.cancel()
return True
return False
def add_writer(self, fd, callback, *args, **kw):
handler = Handler(callback, *args, **kw)
try:
poll_h = self._fd_map[fd]
except KeyError:
poll_h = self._create_poll_handle(fd)
self._fd_map[fd] = poll_h
else:
if poll_h.write_handler:
raise RuntimeError('another writer is already registered for fd {}'.format(fd))
poll_h.pevents |= pyuv.UV_WRITABLE
poll_h.write_handler = handler
poll_h.start(poll_h.pevents, self._poll_cb)
def remove_writer(self, fd):
try:
poll_h = self._fd_map[fd]
except KeyError:
return False
else:
handler = poll_h.write_handler
poll_h.pevents &= ~pyuv.UV_WRITABLE
poll_h.write_handler = None
if poll_h.pevents == 0:
poll_h.close()
del self._fd_map[fd]
else:
poll_h.start(poll_h.pevents, self._poll_cb)
if handler:
handler.cancel()
return True
return False
def add_signal_handler(self, sig, callback, *args, **kwargs):
self._validate_signal(sig)
signal_h = pyuv.Signal(self._loop)
handler = SignalHandler(signal_h, callback, *args, **kwargs)
signal_h.handler = handler
signal_h.signum = sig
try:
signal_h.start(self._signal_cb, sig)
signal_h.unref()
except Exception as e:
signal_h.close()
raise RuntimeError(str(e))
else:
self._signals.setdefault(sig, set()).add(signal_h)
return handler
def remove_signal_handler(self, sig):
self._validate_signal(sig)
try:
handles = self._signals.pop(sig)
except KeyError:
return False
for signal_h in handles:
del signal_h.handler
signal_h.close()
return True
def switch(self):
if not self._started:
self.run()
return
current = Fiber.current()
assert current is not self.task, 'Cannot switch to MAIN from MAIN'
try:
if self.task.parent is not current:
current.parent = self.task
except ValueError:
pass # gets raised if there is a Fiber parent cycle
return self.task.switch()
def run(self, mode=RUN_DEFAULT):
if Fiber.current() is not self.task.parent:
raise RuntimeError('run() can only be called from MAIN fiber')
if not self.task.is_alive():
raise RuntimeError('event loop has already ended')
if self._started:
raise RuntimeError('event loop was already started')
self._started = True
self._running = True
self._run_mode = mode
try:
self.task.switch()
finally:
self._running = False
def run_forever(self):
self.run(mode=RUN_FOREVER)
def stop(self):
if not self._started:
raise RuntimeError('event loop has not been started yet')
if self._loop:
self._loop.stop()
def destroy(self):
if self._running:
raise RuntimeError('destroy() cannot be called while the loop is running')
if self._destroyed:
raise RuntimeError('Event loop already destroyed')
loop = getattr(_tls, 'loop', None)
if loop is not self:
raise RuntimeError('destroy() can only be called from the same thread were the event loop was created')
del _tls.loop, loop
self._destroyed = True
self._uninstall_signal_checker()
self._cleanup_loop()
self._loop.event_loop = None
self._loop.excepthook = None
self._loop = None
self._threadpool = None
self._ready_processor = None
self._waker = None
self._fd_map.clear()
self._signals.clear()
self._timers.clear()
self._ready.clear()
# internal
def _add_callback(self, cb):
self._ready.append(cb)
if not self._ready_processor.active:
self._ready_processor.start(self._process_ready)
def _handle_error(self, typ, value, tb):
if not issubclass(typ, SystemExit):
traceback.print_exception(typ, value, tb)
if issubclass(typ, (KeyboardInterrupt, SystemExit, SystemError)):
assert Fiber.current() is self.task
self.task.parent.throw(typ, value, tb)
def _run_loop(self):
if self._run_mode == RUN_FOREVER:
self._waker.ref()
self._loop.run(pyuv.UV_RUN_DEFAULT)
def _cleanup_loop(self):
def cb(handle):
if not handle.closed:
handle.close()
self._loop.walk(cb)
# All handles are now closed, run will not block
self._loop.run(pyuv.UV_RUN_NOWAIT)
def _create_poll_handle(self, fd):
poll_h = pyuv.Poll(self._loop, fd)
poll_h.pevents = 0
poll_h.read_handler = None
poll_h.write_handler = None
return poll_h
def _process_ready(self, handle):
# Run all queued callbacks
ntodo = len(self._ready)
for x in range(ntodo):
handler = self._ready.popleft()
if not handler._cancelled:
# loop.excepthook takes care of exception handling
handler()
if not self._ready:
self._ready_processor.stop()
def _async_cb(self, handle):
if not self._ready_processor.active:
self._ready_processor.start(self._process_ready)
def _timer_cb(self, timer):
assert not timer.handler._cancelled
self._add_callback(timer.handler)
if not timer.repeat:
timer.close()
self._timers.remove(timer)
del timer.handler
def _signal_cb(self, signal_h, signum):
self._add_callback(signal_h.handler)
def _poll_cb(self, poll_h, events, error):
fd = poll_h.fileno()
if error is not None:
# An error happened, signal both readability and writability and
# let the error propagate
if poll_h.read_handler is not None:
if poll_h.read_handler._cancelled:
self.remove_reader(fd)
else:
self._add_callback(poll_h.read_handler)
if poll_h.write_handler is not None:
if poll_h.write_handler._cancelled:
self.remove_writer(fd)
else:
self._add_callback(poll_h.write_handler)
return
old_events = poll_h.pevents
modified = False
if events & pyuv.UV_READABLE:
if poll_h.read_handler is not None:
if poll_h.read_handler._cancelled:
self.remove_reader(fd)
modified = True
else:
self._add_callback(poll_h.read_handler)
else:
poll_h.pevents &= ~pyuv.UV_READABLE
if events & pyuv.UV_WRITABLE:
if poll_h.write_handler is not None:
if poll_h.write_handler._cancelled:
self.remove_writer(fd)
modified = True
else:
self._add_callback(poll_h.write_handler)
else:
poll_h.pevents &= ~pyuv.UV_WRITABLE
if not modified and old_events != poll_h.pevents:
# Rearm the handle
poll_h.start(poll_h.pevents, self._poll_cb)
def _install_signal_checker(self):
self._socketpair = SocketPair()
self._signal_checker = None
if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix':
try:
old_wakeup_fd = signal.set_wakeup_fd(self._socketpair.writer_fileno())
if old_wakeup_fd != -1:
# Already set, restore it
signal.set_wakeup_fd(old_wakeup_fd)
self._socketpair.close()
self._socketpair = None
else:
self._signal_checker = pyuv.util.SignalChecker(self._loop, self._socketpair.reader_fileno())
self._signal_checker.start()
self._signal_checker.unref()
except ValueError:
self._socketpair.close()
self._socketpair = None
def _uninstall_signal_checker(self):
if self._signal_checker:
self._signal_checker.close()
self._signal_checker = None
if self._socketpair:
self._socketpair.close()
self._socketpair = None
def _validate_signal(self, sig):
if not isinstance(sig, int):
raise TypeError('sig must be an int, not {!r}'.format(sig))
if signal is None:
raise RuntimeError('Signals are not supported')
if not (1 <= sig < signal.NSIG):
raise ValueError('sig {} out of range(1, {})'.format(sig, signal.NSIG))
class Server(object):
_instance = None
@classmethod
def get(cls):
if cls._instance is None:
raise Exception("use Server.start() method")
return cls._instance
def __init__(self, logger=None):
self.observed_ps = defaultdict(list)
self.observed_all_ps = defaultdict(list)
self.observed_task = defaultdict(list)
self.observed_all_tasks = defaultdict(
list) # (task_id) => list of (task_ids, callback)
self.max_submitted_task_id = 0
self._logger = logger or self._default_logger()
self._fibers = []
self._out = None
@classmethod
def start(cls, logger=None, redirect_stdout=False):
cls._instance = cls(logger)
cls._instance._out = os.fdopen(sys.stdout.fileno(),
mode='w',
buffering=1)
sys.stdin = os.fdopen(sys.stdin.fileno(), mode='r', buffering=1)
if redirect_stdout:
sys.stdout = sys.stderr
return cls._instance
def __enter__(self):
self._loop_fiber = Fiber(target=self._loop)
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
return False # re-raise exception
if self._loop_fiber.is_alive():
self._loop_fiber.switch()
@classmethod
def watch_ps(cls, ps, callback):
cls.get().observed_ps[ps.id].append(callback)
@classmethod
def watch_all_ps(cls, ps_set, callback):
ids = [ps.id for ps in ps_set]
key = tuple(ids)
cls.get().observed_all_ps[key].append(callback)
@classmethod
def watch_task(cls, task, callback):
cls.get().observed_task[task.id].append(callback)
@classmethod
def watch_all_tasks(cls, tasks, callback):
key = tuple([t.id for t in tasks])
for t in tasks:
pair = (key, callback)
cls.get().observed_all_tasks[t.id].append(pair)
@classmethod
def async (cls, func, *args, **kwargs):
self = cls.get()
def _f():
func(*args, **kwargs)
self._loop_fiber.switch()
fb = Fiber(target=_f)
self._fibers.append(fb)
@classmethod
def await_ps(cls, ps):
self = cls.get()
fb = Fiber.current()
def _callback(ps):
self._fibers.append(fb)
cls.watch_ps(ps, _callback)
self._loop_fiber.switch()
@classmethod
def await_all_ps(cls, ps_set):
self = cls.get()
fb = Fiber.current()
def _callback(pss):
self._fibers.append(fb)
cls.watch_all_ps(ps_set, _callback)
self._loop_fiber.switch()
@classmethod
def await_task(cls, task):
self = cls.get()
fb = Fiber.current()
def _callback(ps):
self._fibers.append(fb)
cls.watch_task(task, _callback)
self._loop_fiber.switch()
@classmethod
def await_all_tasks(cls, tasks):
self = cls.get()
fb = Fiber.current()
def _callback(ts):
self._fibers.append(fb)
cls.watch_all_tasks(tasks, _callback)
self._loop_fiber.switch()
def _loop(self):
self._launch_all_fibers()
self._submit_all()
self._logger.debug("start polling")
t = self._receive_result()
while t:
self._exec_callback_for_task(t)
self._exec_callback_for_all_task(t)
if isinstance(t, Run):
ps = t.parameter_set()
if ps.is_finished():
self._exec_callback()
self._submit_all()
t = self._receive_result()
def _default_logger(self):
logger = logging.getLogger(__name__)
log_level = logging.INFO
if 'CARAVAN_SEARCH_ENGINE_LOGLEVEL' in os.environ:
s = os.environ['CARAVAN_SEARCH_ENGINE_LOGLEVEL']
levels = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL
}
log_level = levels[s]
logger.setLevel(log_level)
logger.propagate = False
if not logger.handlers:
ch = logging.StreamHandler()
ch.setLevel(log_level)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
return logger
def _has_callbacks(self):
return (len(self.observed_ps) + len(self.observed_all_ps)) > 0
def _has_unfinished_tasks(self):
for r in Task.all()[:self.max_submitted_task_id]:
if not r.is_finished():
return True
return False
def _submit_all(self):
tasks_to_be_submitted = [
t for t in Task.all()[self.max_submitted_task_id:]
if not t.is_finished()
]
self._logger.debug("submitting %d Tasks" % len(tasks_to_be_submitted))
self._print_tasks(tasks_to_be_submitted)
self.max_submitted_task_id = len(Task.all())
def _print_tasks(self, tasks):
for t in tasks:
line = "%d %s\n" % (t.id, t.command)
self._out.write(line)
self._out.write("\n")
def _launch_all_fibers(self):
while self._fibers:
f = self._fibers.pop(0)
self._logger.debug("starting fiber")
f.switch()
def _exec_callback(self):
while self._check_completed_ps() or self._check_completed_ps_all():
pass
def _check_completed_ps(self):
executed = False
for psid in list(self.observed_ps.keys()):
callbacks = self.observed_ps[psid]
ps = ParameterSet.find(psid)
while ps.is_finished() and len(callbacks) > 0:
self._logger.debug("executing callback for ParameterSet %d" %
ps.id)
f = callbacks.pop(0)
f(ps)
self._launch_all_fibers()
executed = True
empty_keys = [k for k, v in self.observed_ps.items() if len(v) == 0]
for k in empty_keys:
self.observed_ps.pop(k)
return executed
def _check_completed_ps_all(self):
executed = False
for psids in list(self.observed_all_ps.keys()):
pss = [ParameterSet.find(psid) for psid in psids]
callbacks = self.observed_all_ps[psids]
while len(callbacks) > 0 and all([ps.is_finished() for ps in pss]):
self._logger.debug("executing callback for ParameterSet %s" %
repr(psids))
f = callbacks.pop(0)
f(pss)
self._launch_all_fibers()
executed = True
empty_keys = [
k for k, v in self.observed_all_ps.items() if len(v) == 0
]
for k in empty_keys:
self.observed_all_ps.pop(k)
return executed
def _exec_callback_for_task(self, task):
executed = False
callbacks = self.observed_task[task.id]
while len(callbacks) > 0:
self._logger.debug("executing callback for Task %d" % task.id)
f = callbacks.pop(0)
f(task)
self._launch_all_fibers()
executed = True
self.observed_task.pop(task.id)
return executed
def _exec_callback_for_all_task(self, task):
executed = False
callback_pairs = self.observed_all_tasks[task.id]
to_be_removed = []
for (idx, pair) in enumerate(callback_pairs):
task_ids = pair[0]
if all([Task.find(t).is_finished() for t in task_ids]):
self._logger.debug("executing callback for Tasks %s" %
str(task_ids))
f = pair[1]
f(Task.find(t) for t in task_ids)
to_be_removed.append(idx)
self._launch_all_fibers()
executed = True
for idx in to_be_removed:
callback_pairs.pop(idx)
if len(callback_pairs) == 0:
self.observed_all_tasks.pop(task.id)
return executed
def _receive_result(self):
line = sys.stdin.readline()
if not line: return None
line = line.rstrip()
self._logger.debug("received: %s" % line)
if not line: return None
l = line.split(' ')
tid, rc, place_id, start_at, finish_at = [int(x) for x in l[:5]]
results = [float(x) for x in l[5:]]
t = Task.find(tid)
t.store_result(results, rc, place_id, start_at, finish_at)
self._logger.debug("stored result of Task %d" % tid)
return t
def _debug(self):
sys.stderr.write(str(self.observed_ps) + "\n")
sys.stderr.write(str(self.observed_all_ps) + "\n")
