def send_exception(g, exc): # note: send_exception(g, exc) can be now done with g.throw(exc). # the purpose of this test is to explicitely check the propagation rules. def crasher(exc): raise exc g1 = Fiber(target=crasher, args=(exc, ), parent=g) g1.switch()
def test_finished_parent(self): def f(): return 42 g = Fiber(f) g.switch() self.assertFalse(g.is_alive()) self.assertRaises(ValueError, Fiber, parent=g)
def test_arg_refs(self): args = ('a', 'b', 'c') refcount_before = sys.getrefcount(args) g = Fiber(target=lambda *x: None, args=args) self.assertEqual(sys.getrefcount(args), refcount_before+1) g.switch() self.assertEqual(sys.getrefcount(args), refcount_before) del g self.assertEqual(sys.getrefcount(args), refcount_before)
def f(): try: raise ValueError('fun') except: exc_info = sys.exc_info() t = Fiber(h) t.switch() self.assertEqual(exc_info, sys.exc_info()) del t
def test_kwarg_refs(self): kwargs = {'a': 1234} refcount_before = sys.getrefcount(kwargs) g = Fiber(lambda **x: None, kwargs=kwargs) self.assertEqual(sys.getrefcount(kwargs), refcount_before+1) g.switch() self.assertEqual(sys.getrefcount(kwargs), refcount_before) del g self.assertEqual(sys.getrefcount(kwargs), refcount_before)
def test_exception(self): seen = [] g1 = Fiber(target=fmain, args=(seen, )) g2 = Fiber(target=fmain, args=(seen, )) g1.switch() g2.switch() g2.parent = g1 self.assertEqual(seen, []) self.assertRaises(SomeError, g2.switch) self.assertEqual(seen, [SomeError])
def test_kwarg_refs(self): if not has_refcount: return kwargs = {'a': 1234} refcount_before = sys.getrefcount(kwargs) g = Fiber(lambda **x: None, kwargs=kwargs) self.assertEqual(sys.getrefcount(kwargs), refcount_before + 1) g.switch() self.assertEqual(sys.getrefcount(kwargs), refcount_before) del g self.assertEqual(sys.getrefcount(kwargs), refcount_before)
def test_arg_refs(self): if not has_refcount: return args = ('a', 'b', 'c') refcount_before = sys.getrefcount(args) g = Fiber(target=lambda *x: None, args=args) self.assertEqual(sys.getrefcount(args), refcount_before + 1) g.switch() self.assertEqual(sys.getrefcount(args), refcount_before) del g self.assertEqual(sys.getrefcount(args), refcount_before)
def test_simple2(self): lst = [] def f(): lst.append(1) current().parent.switch() lst.append(3) g = Fiber(f) lst.append(0) g.switch() lst.append(2) g.switch() lst.append(4) self.assertEqual(lst, list(range(5)))
def test_two_recursive_children(self): lst = [] def f(): lst.append(1) current().parent.switch() def h(): lst.append(1) i = Fiber(f) i.switch() lst.append(1) g = Fiber(h) g.switch() self.assertEqual(len(lst), 3)
def test_exc_state(self): def f(): try: raise ValueError('fun') except: exc_info = sys.exc_info() t = Fiber(h) t.switch() self.assertEqual(exc_info, sys.exc_info()) del t def h(): self.assertEqual(sys.exc_info(), (None, None, None)) g = Fiber(f) g.switch()
def test_instance_dict(self): def f(): current().test = 42 def deldict(g): del g.__dict__ def setdict(g, value): g.__dict__ = value g = Fiber(f) self.assertEqual(g.__dict__, {}) g.switch() self.assertEqual(g.test, 42) self.assertEqual(g.__dict__, {'test': 42}) g.__dict__ = g.__dict__ self.assertEqual(g.__dict__, {'test': 42}) self.assertRaises(AttributeError, deldict, g) self.assertRaises(TypeError, setdict, g, 42)
def test_instance_dict(self): if is_pypy: return def f(): current().test = 42 def deldict(g): del g.__dict__ def setdict(g, value): g.__dict__ = value g = Fiber(f) self.assertEqual(g.__dict__, {}) g.switch() self.assertEqual(g.test, 42) self.assertEqual(g.__dict__, {'test': 42}) g.__dict__ = g.__dict__ self.assertEqual(g.__dict__, {'test': 42}) self.assertRaises(AttributeError, deldict, g) self.assertRaises(TypeError, setdict, g, 42)
def test_throw_goes_to_original_parent(self): main = fibers.current() def f1(): try: main.switch("f1 ready to catch") except IndexError: return "caught" else: return "normal exit" def f2(): main.switch("from f2") g1 = Fiber(f1) g2 = Fiber(target=f2, parent=g1) self.assertRaises(IndexError, g2.throw, IndexError) self.assertFalse(g2.is_alive()) self.assertTrue(g1.is_alive()) # g1 is skipped because it was not started g1 = Fiber(f1) g2 = Fiber(target=f2, parent=g1) res = g1.switch() self.assertEqual(res, "f1 ready to catch") res = g2.throw(IndexError) self.assertEqual(res, "caught") self.assertFalse(g2.is_alive()) self.assertFalse(g1.is_alive()) g1 = Fiber(f1) g2 = Fiber(target=f2, parent=g1) res = g1.switch() self.assertEqual(res, "f1 ready to catch") res = g2.switch() self.assertEqual(res, "from f2") res = g2.throw(IndexError) self.assertEqual(res, "caught") self.assertFalse(g2.is_alive()) self.assertFalse(g1.is_alive())
def test_val(self): def f(): try: switch("ok") except RuntimeError: val = sys.exc_info()[1] if str(val) == "ciao": switch("ok") return switch("fail") g = Fiber(f) res = g.switch() self.assertEqual(res, "ok") res = g.throw(RuntimeError("ciao")) self.assertEqual(res, "ok") g = Fiber(f) res = g.switch() self.assertEqual(res, "ok") res = g.throw(RuntimeError, "ciao") self.assertEqual(res, "ok")
def test_kill(self): def f(): try: switch("ok") switch("fail") except Exception as e: return e g = Fiber(f) res = g.switch() self.assertEqual(res, "ok") res = g.throw(ValueError) self.assertTrue(isinstance(res, ValueError)) self.assertFalse(g.is_alive())
def test_class(self): def f(): try: switch("ok") except RuntimeError: switch("ok") return switch("fail") g = Fiber(f) res = g.switch() self.assertEqual(res, "ok") res = g.throw(RuntimeError) self.assertEqual(res, "ok")
def test_finalizer_crash(self): # This test is designed to crash when active greenlets # are made garbage collectable, until the underlying # problem is resolved. How does it work: # - order of object creation is important # - array is created first, so it is moved to unreachable first # - we create a cycle between a greenlet and this array # - we create an object that participates in gc, is only # referenced by a greenlet, and would corrupt gc lists # on destruction, the easiest is to use an object with # a finalizer # - because array is the first object in unreachable it is # cleared first, which causes all references to greenlet # to disappear and causes greenlet to be destroyed, but since # it is still live it causes a switch during gc, which causes # an object with finalizer to be destroyed, which causes stack # corruption and then a crash class object_with_finalizer(object): def __del__(self): pass array = [] parent = current() def greenlet_body(): current().object = object_with_finalizer() try: parent.switch() finally: del current().object g = Fiber(greenlet_body) g.array = array array.append(g) g.switch() del array del g current() gc.collect()
def test_threaded_reparent(self): data = {} created_event = threading.Event() done_event = threading.Event() def foo(): data['g'] = Fiber(lambda: None) created_event.set() done_event.wait() def blank(): current().parent.switch() def setparent(g, value): g.parent = value thread = threading.Thread(target=foo) thread.start() created_event.wait() g = Fiber(blank) g.switch() self.assertRaises(ValueError, setparent, g, data['g']) done_event.set() thread.join()
def test_throw_goes_to_original_parent3(self): main = fibers.current() def f1(): try: main.switch("f1 ready to catch") except IndexError: return "caught" else: return "normal exit" def f2(): main.switch("from f2") g1 = Fiber(f1) g2 = Fiber(target=f2, parent=g1) res = g1.switch() self.assertEqual(res, "f1 ready to catch") res = g2.switch() self.assertEqual(res, "from f2") res = g2.throw(IndexError) self.assertEqual(res, "caught") self.assertFalse(g2.is_alive()) self.assertFalse(g1.is_alive())
def test_two_children(self): lst = [] def f(): lst.append(1) current().parent.switch() lst.extend([1, 1]) g = Fiber(f) h = Fiber(f) g.switch() self.assertEqual(len(lst), 1) h.switch() self.assertEqual(len(lst), 2) h.switch() self.assertEqual(len(lst), 4) self.assertEqual(h.is_alive(), False) g.switch() self.assertEqual(len(lst), 6) self.assertEqual(g.is_alive(), False)
class Server(object): _instance = None @classmethod def _get(cls): if cls._instance is None: raise Exception("use Server.start() method") return cls._instance def __init__(self, logger=None): """ Note --- Do not call the constructor directory. Instead, use `Server.start` to launch a server. """ self.observed_ps = defaultdict(list) self.observed_all_ps = defaultdict(list) self.observed_task = defaultdict(list) self.observed_all_tasks = defaultdict( list) # (task_id) => list of (task_ids, callback) self.max_submitted_task_id = 0 self._logger = logger or self._default_logger() self._fibers = [] self._comm = None @classmethod def start(cls, logger=None): """ start a scheduling of tasks. Examples --- >>> with Server.start(): .... """ cls._instance = cls(logger) return cls._instance def __enter__(self): self._loop_fiber = Fiber(target=self._loop) self._comm = MPI.Comm.Get_parent() self._logger.debug("accepted") def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: return False # re-raise exception if self._loop_fiber.is_alive(): self._loop_fiber.switch() self._comm.Disconnect() @classmethod def watch_ps(cls, ps, callback): """ add a callback function when ps gets completed. Parameters --- ps : ParameterSet callback : callable """ cls._get().observed_ps[ps.id()].append(callback) @classmethod def watch_all_ps(cls, ps_set, callback): """ add a callback function when a set of ParameterSets is completed. Parameters --- ps_set : list of ParameterSet callback : callable """ ids = [ps.id() for ps in ps_set] key = tuple(ids) cls._get().observed_all_ps[key].append(callback) @classmethod def watch_task(cls, task, callback): """ add a callback function when a Task is completed. Parameters --- task : Task callback : callable """ cls._get().observed_task[task.id()].append(callback) @classmethod def watch_all_tasks(cls, tasks, callback): """ add a callback function when a set of Tasks is completed. Parameters --- tasks : list of Task callback : callable """ key = tuple([t.id() for t in tasks]) for t in tasks: pair = (key, callback) cls._get().observed_all_tasks[t.id()].append(pair) @classmethod def do_async(cls, func, *args, **kwargs): """ do coroutine asynchronously. Examples --- Server.do_async( lambda : Task.create(...) ) Parameters --- tasks : list of Task callback : callable """ self = cls._get() def _f(): func(*args, **kwargs) self._loop_fiber.switch() fb = Fiber(target=_f) self._fibers.append(fb) @classmethod def await_ps(cls, ps): """ wait until parameterSet is complete. During waiting, other co-routines are concurrently executed. Parameters --- ps : ParameterSet """ self = cls._get() fb = Fiber.current() def _callback(): self._fibers.append(fb) cls.watch_ps(ps, _callback) self._loop_fiber.switch() @classmethod def await_all_ps(cls, ps_set): """ wait until a set of ParameterSets is complete. While waiting, other co-routines are concurrently executed. Parameters --- ps_set : list of ParameterSet """ self = cls._get() fb = Fiber.current() def _callback(): self._fibers.append(fb) cls.watch_all_ps(ps_set, _callback) self._loop_fiber.switch() @classmethod def await_task(cls, task): """ wait until a Task is complete. Parameters --- task : Task """ self = cls._get() fb = Fiber.current() def _callback(): self._fibers.append(fb) cls.watch_task(task, _callback) self._loop_fiber.switch() @classmethod def await_all_tasks(cls, tasks): """ wait until a set of Tasks is complete. Parameters --- tasks : list of Task """ self = cls._get() fb = Fiber.current() def _callback(): self._fibers.append(fb) cls.watch_all_tasks(tasks, _callback) self._loop_fiber.switch() def _loop(self): self._launch_all_fibers() self._submit_all() self._logger.debug("start polling") t = self._receive_result() while t: self._exec_callback_for_task(t) self._exec_callback_for_all_task(t) if isinstance(t, Run): ps = t.parameter_set() if ps.is_finished(): self._exec_callback() self._submit_all() t = self._receive_result() def _default_logger(self): logger = logging.getLogger(__name__) log_level = logging.INFO if 'CARAVAN_SEARCH_ENGINE_LOGLEVEL' in os.environ: s = os.environ['CARAVAN_SEARCH_ENGINE_LOGLEVEL'] levels = { 'DEBUG': logging.DEBUG, 'INFO': logging.INFO, 'WARNING': logging.WARNING, 'ERROR': logging.ERROR, 'CRITICAL': logging.CRITICAL } log_level = levels[s] logger.setLevel(log_level) logger.propagate = False if not logger.handlers: ch = logging.StreamHandler() ch.setLevel(log_level) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) return logger def _has_callbacks(self): return (len(self.observed_ps) + len(self.observed_all_ps)) > 0 def _has_unfinished_tasks(self): for r in Task.all()[:self.max_submitted_task_id]: if not r.is_finished(): return True return False def _submit_all(self): tasks_to_be_submitted = [ t for t in Task.all()[self.max_submitted_task_id:] if not t.is_finished() ] self._logger.debug("submitting %d Tasks" % len(tasks_to_be_submitted)) self._print_tasks(tasks_to_be_submitted) self.max_submitted_task_id = len(Task.all()) def _print_tasks(self, tasks): b_tasks = [{ "id": t.id(), "cmd": t.command(), "input": t.input() } for t in tasks] packed = msgpack.packb(b_tasks) self._comm.Send([bytearray(packed), MPI.CHAR], dest=0, tag=0) self._logger.debug(f"sent packed") def _launch_all_fibers(self): while self._fibers: f = self._fibers.pop(0) self._logger.debug("starting fiber") f.switch() def _exec_callback(self): while self._check_completed_ps() or self._check_completed_ps_all(): pass def _check_completed_ps(self): executed = False for psid in list(self.observed_ps.keys()): callbacks = self.observed_ps[psid] ps = ParameterSet.find(psid) while ps.is_finished() and len(callbacks) > 0: self._logger.debug("executing callback for ParameterSet %d" % ps.id()) f = callbacks.pop(0) f() self._launch_all_fibers() executed = True empty_keys = [k for k, v in self.observed_ps.items() if len(v) == 0] for k in empty_keys: self.observed_ps.pop(k) return executed def _check_completed_ps_all(self): executed = False for psids in list(self.observed_all_ps.keys()): pss = [ParameterSet.find(psid) for psid in psids] callbacks = self.observed_all_ps[psids] while len(callbacks) > 0 and all([ps.is_finished() for ps in pss]): self._logger.debug("executing callback for ParameterSet %s" % repr(psids)) f = callbacks.pop(0) f() self._launch_all_fibers() executed = True empty_keys = [ k for k, v in self.observed_all_ps.items() if len(v) == 0 ] for k in empty_keys: self.observed_all_ps.pop(k) return executed def _exec_callback_for_task(self, task): executed = False callbacks = self.observed_task[task.id()] while len(callbacks) > 0: self._logger.debug("executing callback for Task %d" % task.id()) f = callbacks.pop(0) f() self._launch_all_fibers() executed = True self.observed_task.pop(task.id()) return executed def _exec_callback_for_all_task(self, task): executed = False callback_pairs = self.observed_all_tasks[task.id()] to_be_removed = [] for (idx, pair) in enumerate(callback_pairs): task_ids = pair[0] if all([Task.find(t).is_finished() for t in task_ids]): self._logger.debug("executing callback for Tasks %s" % str(task_ids)) f = pair[1] f() to_be_removed.append(idx) self._launch_all_fibers() executed = True for idx in to_be_removed: callback_pairs.pop(idx) if len(callback_pairs) == 0: self.observed_all_tasks.pop(task.id()) return executed def _receive_bytes(self): s = MPI.Status() # instead of `comm.Probe`, `Iprobe` is used to avoid a busy wait while not self._comm.Iprobe(status=s): time.sleep(0.01) if s.count == 0: assert s.tag == 1 recvbuf = bytearray(s.count) self._comm.Recv([recvbuf, s.count, MPI.CHAR], source=s.source, tag=s.tag) return recvbuf def _receive_result(self): data_b = self._receive_bytes() if len(data_b) == 0: return None self._logger.debug("received: %s bytes" % len(data_b)) unpacked = msgpack.unpackb(data_b) self._logger.debug("received: %s" % str(unpacked)) tid = unpacked["id"] rc = unpacked["rc"] rank = unpacked["rank"] start_at = unpacked["start_at"] finish_at = unpacked["finish_at"] output = unpacked["output"] t = Task.find(tid) t._store_result(output, rc, rank, start_at, finish_at) self._logger.debug("stored result of Task %d" % tid) return t def _debug(self): sys.stderr.write(str(self.observed_ps) + "\n") sys.stderr.write(str(self.observed_all_ps) + "\n")
def _dead_fiber(): g = Fiber(lambda: None) g.switch() return g
def h(): lst.append(1) i = Fiber(f) i.switch() lst.append(1)
def test_send_exception(self): seen = [] g1 = Fiber(target=fmain, args=(seen, )) g1.switch() self.assertRaises(KeyError, send_exception, g1, KeyError) self.assertEqual(seen, [KeyError])
def runner(x): g = Fiber(lambda: time.sleep(x)) g.switch()
def creator(): g = Fiber(worker) g.switch() result.append(g)
class StubServer(Server): def __init__(self, stub_simulator, num_proc, logger, dump_path): """ Note --- Do not call the constructor directory. Instead, use `StubServer.start` to launch a server. """ super().__init__(logger) self._stub_simulator = stub_simulator self._num_proc = num_proc self._dump_path = dump_path self._queue = EventQueue(self._num_proc) @classmethod def start(cls, stub_simulator, num_proc, logger=None, dump_path='tasks.msgpack'): """ Start a scheduling of tasks with stub simulator. This is provided for testing search engines without actually running the simulations. Instead of actually conducting simulations, it mimics the event using a dummy simulator `stub_simulator`. `stub_simulator` is a function which receives a Task as its argument and returns a tuple of (output, dt). Here, `output` is a json-like object and `dt` is a duration of simulation in seconds. It simulates the task scheduling assuming that the tasks are parallely executed by `num_proc` processes. Examples --- >>> def stub_sim(task): return ({"res1":1.0, "res2":2.0}, 100) >>> wtih StubServer.start(stub_sim, 8): .... """ Server._instance = cls(stub_simulator, num_proc, logger, dump_path) return Server._instance # override the methods def _print_tasks(self, tasks): for t in tasks: res, dt = self._stub_simulator(t) t._output = res t._dt = int(1000 * dt) self._queue.push_all(tasks) def _receive_result(self): t = self._queue.pop() if t is None: return None t._rc = 0 return t def __enter__(self): self._loop_fiber = Fiber(target=self._loop) def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: return False # re-raise exception if self._loop_fiber.is_alive(): self._loop_fiber.switch() Task._dump_binary(self._dump_path)
class EventLoop(object): def __init__(self): if getattr(_tls, 'loop', None) is not None: raise RuntimeError( 'cannot instantiate more than one event loop per thread') _tls.loop = self self._loop = pyuv.Loop() self._loop.excepthook = self._handle_error self._loop.event_loop = self self._threadpool = ThreadPool(self) self.task = Fiber(self._run_loop) self._destroyed = False self._started = False self._running = False self._fd_map = dict() self._signals = dict() self._timers = set() self._ready = deque() self._ready_processor = pyuv.Idle(self._loop) self._waker = pyuv.Async(self._loop, self._async_cb) self._waker.unref() self._install_signal_checker() @classmethod def current(cls): """Get the current event loop singleton object. """ try: return _tls.loop except AttributeError: # create loop only for main thread if threading.current_thread().name == 'MainThread': _tls.loop = cls() return _tls.loop raise RuntimeError( 'there is no event loop created in the current thread') @property def running(self): return self._running def call_soon(self, callback, *args, **kw): handler = Handler(callback, *args, **kw) self._add_callback(handler) return handler def call_from_thread(self, callback, *args, **kw): handler = Handler(callback, *args, **kw) # Here we don't call self._add_callback on purpose, because it's not thread # safe to start pyuv handles. We just append the callback to the queue and # wakeup the loop. This is thread safe because the queue is only processed # in a single place and in a thread safe manner. self._ready.append(handler) self._waker.send() return handler def call_later(self, delay, callback, *args, **kw): if delay <= 0: return self.call_soon(callback, *args, **kw) timer_h = pyuv.Timer(self._loop) handler = Timer(timer_h, callback, *args, **kw) timer_h.handler = handler timer_h.start(self._timer_cb, delay, 0) self._timers.add(timer_h) return handler def call_at(self, when, callback, *args, **kw): return self.call_later(when - self.time(), callback, *args, **kw) def time(self): return _time() def add_reader(self, fd, callback, *args, **kw): handler = Handler(callback, *args, **kw) try: poll_h = self._fd_map[fd] except KeyError: poll_h = self._create_poll_handle(fd) self._fd_map[fd] = poll_h else: if poll_h.read_handler: raise RuntimeError( 'another reader is already registered for fd {}'.format( fd)) poll_h.pevents |= pyuv.UV_READABLE poll_h.read_handler = handler poll_h.start(poll_h.pevents, self._poll_cb) def remove_reader(self, fd): try: poll_h = self._fd_map[fd] except KeyError: return False else: handler = poll_h.read_handler poll_h.pevents &= ~pyuv.UV_READABLE poll_h.read_handler = None if poll_h.pevents == 0: poll_h.close() del self._fd_map[fd] else: poll_h.start(poll_h.pevents, self._poll_cb) if handler: handler.cancel() return True return False def add_writer(self, fd, callback, *args, **kw): handler = Handler(callback, *args, **kw) try: poll_h = self._fd_map[fd] except KeyError: poll_h = self._create_poll_handle(fd) self._fd_map[fd] = poll_h else: if poll_h.write_handler: raise RuntimeError( 'another writer is already registered for fd {}'.format( fd)) poll_h.pevents |= pyuv.UV_WRITABLE poll_h.write_handler = handler poll_h.start(poll_h.pevents, self._poll_cb) def remove_writer(self, fd): try: poll_h = self._fd_map[fd] except KeyError: return False else: handler = poll_h.write_handler poll_h.pevents &= ~pyuv.UV_WRITABLE poll_h.write_handler = None if poll_h.pevents == 0: poll_h.close() del self._fd_map[fd] else: poll_h.start(poll_h.pevents, self._poll_cb) if handler: handler.cancel() return True return False def add_signal_handler(self, sig, callback, *args, **kwargs): self._validate_signal(sig) signal_h = pyuv.Signal(self._loop) handler = SignalHandler(signal_h, callback, *args, **kwargs) signal_h.handler = handler signal_h.signum = sig try: signal_h.start(self._signal_cb, sig) signal_h.unref() except Exception as e: signal_h.close() raise RuntimeError(str(e)) else: self._signals.setdefault(sig, set()).add(signal_h) return handler def remove_signal_handler(self, sig): self._validate_signal(sig) try: handles = self._signals.pop(sig) except KeyError: return False for signal_h in handles: del signal_h.handler signal_h.close() return True def switch(self): if not self._started: self._run(forever=False) return current = Fiber.current() assert current is not self.task, 'Cannot switch to MAIN from MAIN' try: if self.task.parent is not current: current.parent = self.task except ValueError: pass # gets raised if there is a Fiber parent cycle return self.task.switch() def run(self, mode=RUN_DEFAULT): if Fiber.current() is not self.task.parent: raise RuntimeError('run() can only be called from MAIN fiber') if not self.task.is_alive(): raise RuntimeError('event loop has already ended') if self._started: raise RuntimeError('event loop was already started') self._started = True self._running = True self._run_mode = mode try: self.task.switch() finally: self._running = False def run_forever(self): self.run(mode=RUN_FOREVER) def stop(self): if not self._started: raise RuntimeError('event loop has not been started yet') if self._loop: self._loop.stop() def destroy(self): if self._running: raise RuntimeError( 'destroy() cannot be called while the loop is running') if self._destroyed: raise RuntimeError('Event loop already destroyed') loop = getattr(_tls, 'loop', None) if loop is not self: raise RuntimeError( 'destroy() can only be called from the same thread were the event loop was created' ) del _tls.loop, loop self._destroyed = True self._uninstall_signal_checker() self._cleanup_loop() self._loop.event_loop = None self._loop.excepthook = None self._loop = None self._threadpool = None self._ready_processor = None self._waker = None self._fd_map.clear() self._signals.clear() self._timers.clear() self._ready.clear() # internal def _add_callback(self, cb): self._ready.append(cb) if not self._ready_processor.active: self._ready_processor.start(self._process_ready) def _handle_error(self, typ, value, tb): if not issubclass(typ, SystemExit): traceback.print_exception(typ, value, tb) if issubclass(typ, (KeyboardInterrupt, SystemExit, SystemError)): assert Fiber.current() is self.task self.task.parent.throw(typ, value, tb) def _run_loop(self): if self._run_mode == RUN_FOREVER: self._waker.ref() self._loop.run(pyuv.UV_RUN_DEFAULT) def _cleanup_loop(self): def cb(handle): if not handle.closed: handle.close() self._loop.walk(cb) # All handles are now closed, run will not block self._loop.run(pyuv.UV_RUN_NOWAIT) def _create_poll_handle(self, fd): poll_h = pyuv.Poll(self._loop, fd) poll_h.pevents = 0 poll_h.read_handler = None poll_h.write_handler = None return poll_h def _process_ready(self, handle): # Run all queued callbacks ntodo = len(self._ready) for x in range(ntodo): handler = self._ready.popleft() if not handler._cancelled: # loop.excepthook takes care of exception handling handler() if not self._ready: self._ready_processor.stop() def _async_cb(self, handle): if not self._ready_processor.active: self._ready_processor.start(self._process_ready) def _timer_cb(self, timer): assert not timer.handler._cancelled self._add_callback(timer.handler) if not timer.repeat: timer.close() self._timers.remove(timer) del timer.handler def _signal_cb(self, signal_h, signum): self._add_callback(signal_h.handler) def _poll_cb(self, poll_h, events, error): fd = poll_h.fileno() if error is not None: # An error happened, signal both readability and writability and # let the error propagate if poll_h.read_handler is not None: if poll_h.read_handler._cancelled: self.remove_reader(fd) else: self._add_callback(poll_h.read_handler) if poll_h.write_handler is not None: if poll_h.write_handler._cancelled: self.remove_writer(fd) else: self._add_callback(poll_h.write_handler) return old_events = poll_h.pevents modified = False if events & pyuv.UV_READABLE: if poll_h.read_handler is not None: if poll_h.read_handler._cancelled: self.remove_reader(fd) modified = True else: self._add_callback(poll_h.read_handler) else: poll_h.pevents &= ~pyuv.UV_READABLE if events & pyuv.UV_WRITABLE: if poll_h.write_handler is not None: if poll_h.write_handler._cancelled: self.remove_writer(fd) modified = True else: self._add_callback(poll_h.write_handler) else: poll_h.pevents &= ~pyuv.UV_WRITABLE if not modified and old_events != poll_h.pevents: # Rearm the handle poll_h.start(poll_h.pevents, self._poll_cb) def _install_signal_checker(self): self._socketpair = SocketPair() self._signal_checker = None if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix': try: old_wakeup_fd = signal.set_wakeup_fd( self._socketpair.writer_fileno()) if old_wakeup_fd != -1: # Already set, restore it signal.set_wakeup_fd(old_wakeup_fd) self._socketpair.close() self._socketpair = None else: self._signal_checker = pyuv.util.SignalChecker( self._loop, self._socketpair.reader_fileno()) self._signal_checker.start() self._signal_checker.unref() except ValueError: self._socketpair.close() self._socketpair = None def _uninstall_signal_checker(self): if self._signal_checker: self._signal_checker.close() self._signal_checker = None if self._socketpair: self._socketpair.close() self._socketpair = None def _validate_signal(self, sig): if not isinstance(sig, int): raise TypeError('sig must be an int, not {!r}'.format(sig)) if signal is None: raise RuntimeError('Signals are not supported') if not (1 <= sig < signal.NSIG): raise ValueError('sig {} out of range(1, {})'.format( sig, signal.NSIG))
class EventLoop(object): def __init__(self): if getattr(_tls, 'loop', None) is not None: raise RuntimeError('cannot instantiate more than one event loop per thread') _tls.loop = self self._loop = pyuv.Loop() self._loop.excepthook = self._handle_error self._loop.event_loop = self self._threadpool = ThreadPool(self) self.task = Fiber(self._run_loop) self._destroyed = False self._started = False self._running = False self._fd_map = dict() self._signals = dict() self._timers = set() self._ready = deque() self._ready_processor = pyuv.Idle(self._loop) self._waker = pyuv.Async(self._loop, self._async_cb) self._waker.unref() self._install_signal_checker() @classmethod def current(cls): """Get the current event loop singleton object. """ try: return _tls.loop except AttributeError: # create loop only for main thread if threading.current_thread().name == 'MainThread': _tls.loop = cls() return _tls.loop raise RuntimeError('there is no event loop created in the current thread') @property def running(self): return self._running def call_soon(self, callback, *args, **kw): handler = Handler(callback, *args, **kw) self._add_callback(handler) return handler def call_from_thread(self, callback, *args, **kw): handler = Handler(callback, *args, **kw) # Here we don't call self._add_callback on purpose, because it's not thread # safe to start pyuv handles. We just append the callback to the queue and # wakeup the loop. This is thread safe because the queue is only processed # in a single place and in a thread safe manner. self._ready.append(handler) self._waker.send() return handler def call_later(self, delay, callback, *args, **kw): if delay <= 0: return self.call_soon(callback, *args, **kw) timer_h = pyuv.Timer(self._loop) handler = Timer(timer_h, callback, *args, **kw) timer_h.handler = handler timer_h.start(self._timer_cb, delay, 0) self._timers.add(timer_h) return handler def call_at(self, when, callback, *args, **kw): return self.call_later(when-self.time(), callback, *args, **kw) def time(self): return _time() def add_reader(self, fd, callback, *args, **kw): handler = Handler(callback, *args, **kw) try: poll_h = self._fd_map[fd] except KeyError: poll_h = self._create_poll_handle(fd) self._fd_map[fd] = poll_h else: if poll_h.read_handler: raise RuntimeError('another reader is already registered for fd {}'.format(fd)) poll_h.pevents |= pyuv.UV_READABLE poll_h.read_handler = handler poll_h.start(poll_h.pevents, self._poll_cb) def remove_reader(self, fd): try: poll_h = self._fd_map[fd] except KeyError: return False else: handler = poll_h.read_handler poll_h.pevents &= ~pyuv.UV_READABLE poll_h.read_handler = None if poll_h.pevents == 0: poll_h.close() del self._fd_map[fd] else: poll_h.start(poll_h.pevents, self._poll_cb) if handler: handler.cancel() return True return False def add_writer(self, fd, callback, *args, **kw): handler = Handler(callback, *args, **kw) try: poll_h = self._fd_map[fd] except KeyError: poll_h = self._create_poll_handle(fd) self._fd_map[fd] = poll_h else: if poll_h.write_handler: raise RuntimeError('another writer is already registered for fd {}'.format(fd)) poll_h.pevents |= pyuv.UV_WRITABLE poll_h.write_handler = handler poll_h.start(poll_h.pevents, self._poll_cb) def remove_writer(self, fd): try: poll_h = self._fd_map[fd] except KeyError: return False else: handler = poll_h.write_handler poll_h.pevents &= ~pyuv.UV_WRITABLE poll_h.write_handler = None if poll_h.pevents == 0: poll_h.close() del self._fd_map[fd] else: poll_h.start(poll_h.pevents, self._poll_cb) if handler: handler.cancel() return True return False def add_signal_handler(self, sig, callback, *args, **kwargs): self._validate_signal(sig) signal_h = pyuv.Signal(self._loop) handler = SignalHandler(signal_h, callback, *args, **kwargs) signal_h.handler = handler signal_h.signum = sig try: signal_h.start(self._signal_cb, sig) signal_h.unref() except Exception as e: signal_h.close() raise RuntimeError(str(e)) else: self._signals.setdefault(sig, set()).add(signal_h) return handler def remove_signal_handler(self, sig): self._validate_signal(sig) try: handles = self._signals.pop(sig) except KeyError: return False for signal_h in handles: del signal_h.handler signal_h.close() return True def switch(self): if not self._started: self.run() return current = Fiber.current() assert current is not self.task, 'Cannot switch to MAIN from MAIN' try: if self.task.parent is not current: current.parent = self.task except ValueError: pass # gets raised if there is a Fiber parent cycle return self.task.switch() def run(self, mode=RUN_DEFAULT): if Fiber.current() is not self.task.parent: raise RuntimeError('run() can only be called from MAIN fiber') if not self.task.is_alive(): raise RuntimeError('event loop has already ended') if self._started: raise RuntimeError('event loop was already started') self._started = True self._running = True self._run_mode = mode try: self.task.switch() finally: self._running = False def run_forever(self): self.run(mode=RUN_FOREVER) def stop(self): if not self._started: raise RuntimeError('event loop has not been started yet') if self._loop: self._loop.stop() def destroy(self): if self._running: raise RuntimeError('destroy() cannot be called while the loop is running') if self._destroyed: raise RuntimeError('Event loop already destroyed') loop = getattr(_tls, 'loop', None) if loop is not self: raise RuntimeError('destroy() can only be called from the same thread were the event loop was created') del _tls.loop, loop self._destroyed = True self._uninstall_signal_checker() self._cleanup_loop() self._loop.event_loop = None self._loop.excepthook = None self._loop = None self._threadpool = None self._ready_processor = None self._waker = None self._fd_map.clear() self._signals.clear() self._timers.clear() self._ready.clear() # internal def _add_callback(self, cb): self._ready.append(cb) if not self._ready_processor.active: self._ready_processor.start(self._process_ready) def _handle_error(self, typ, value, tb): if not issubclass(typ, SystemExit): traceback.print_exception(typ, value, tb) if issubclass(typ, (KeyboardInterrupt, SystemExit, SystemError)): assert Fiber.current() is self.task self.task.parent.throw(typ, value, tb) def _run_loop(self): if self._run_mode == RUN_FOREVER: self._waker.ref() self._loop.run(pyuv.UV_RUN_DEFAULT) def _cleanup_loop(self): def cb(handle): if not handle.closed: handle.close() self._loop.walk(cb) # All handles are now closed, run will not block self._loop.run(pyuv.UV_RUN_NOWAIT) def _create_poll_handle(self, fd): poll_h = pyuv.Poll(self._loop, fd) poll_h.pevents = 0 poll_h.read_handler = None poll_h.write_handler = None return poll_h def _process_ready(self, handle): # Run all queued callbacks ntodo = len(self._ready) for x in range(ntodo): handler = self._ready.popleft() if not handler._cancelled: # loop.excepthook takes care of exception handling handler() if not self._ready: self._ready_processor.stop() def _async_cb(self, handle): if not self._ready_processor.active: self._ready_processor.start(self._process_ready) def _timer_cb(self, timer): assert not timer.handler._cancelled self._add_callback(timer.handler) if not timer.repeat: timer.close() self._timers.remove(timer) del timer.handler def _signal_cb(self, signal_h, signum): self._add_callback(signal_h.handler) def _poll_cb(self, poll_h, events, error): fd = poll_h.fileno() if error is not None: # An error happened, signal both readability and writability and # let the error propagate if poll_h.read_handler is not None: if poll_h.read_handler._cancelled: self.remove_reader(fd) else: self._add_callback(poll_h.read_handler) if poll_h.write_handler is not None: if poll_h.write_handler._cancelled: self.remove_writer(fd) else: self._add_callback(poll_h.write_handler) return old_events = poll_h.pevents modified = False if events & pyuv.UV_READABLE: if poll_h.read_handler is not None: if poll_h.read_handler._cancelled: self.remove_reader(fd) modified = True else: self._add_callback(poll_h.read_handler) else: poll_h.pevents &= ~pyuv.UV_READABLE if events & pyuv.UV_WRITABLE: if poll_h.write_handler is not None: if poll_h.write_handler._cancelled: self.remove_writer(fd) modified = True else: self._add_callback(poll_h.write_handler) else: poll_h.pevents &= ~pyuv.UV_WRITABLE if not modified and old_events != poll_h.pevents: # Rearm the handle poll_h.start(poll_h.pevents, self._poll_cb) def _install_signal_checker(self): self._socketpair = SocketPair() self._signal_checker = None if hasattr(signal, 'set_wakeup_fd') and os.name == 'posix': try: old_wakeup_fd = signal.set_wakeup_fd(self._socketpair.writer_fileno()) if old_wakeup_fd != -1: # Already set, restore it signal.set_wakeup_fd(old_wakeup_fd) self._socketpair.close() self._socketpair = None else: self._signal_checker = pyuv.util.SignalChecker(self._loop, self._socketpair.reader_fileno()) self._signal_checker.start() self._signal_checker.unref() except ValueError: self._socketpair.close() self._socketpair = None def _uninstall_signal_checker(self): if self._signal_checker: self._signal_checker.close() self._signal_checker = None if self._socketpair: self._socketpair.close() self._socketpair = None def _validate_signal(self, sig): if not isinstance(sig, int): raise TypeError('sig must be an int, not {!r}'.format(sig)) if signal is None: raise RuntimeError('Signals are not supported') if not (1 <= sig < signal.NSIG): raise ValueError('sig {} out of range(1, {})'.format(sig, signal.NSIG))
class Server(object): _instance = None @classmethod def get(cls): if cls._instance is None: raise Exception("use Server.start() method") return cls._instance def __init__(self, logger=None): self.observed_ps = defaultdict(list) self.observed_all_ps = defaultdict(list) self.observed_task = defaultdict(list) self.observed_all_tasks = defaultdict( list) # (task_id) => list of (task_ids, callback) self.max_submitted_task_id = 0 self._logger = logger or self._default_logger() self._fibers = [] self._out = None @classmethod def start(cls, logger=None, redirect_stdout=False): cls._instance = cls(logger) cls._instance._out = os.fdopen(sys.stdout.fileno(), mode='w', buffering=1) sys.stdin = os.fdopen(sys.stdin.fileno(), mode='r', buffering=1) if redirect_stdout: sys.stdout = sys.stderr return cls._instance def __enter__(self): self._loop_fiber = Fiber(target=self._loop) def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: return False # re-raise exception if self._loop_fiber.is_alive(): self._loop_fiber.switch() @classmethod def watch_ps(cls, ps, callback): cls.get().observed_ps[ps.id].append(callback) @classmethod def watch_all_ps(cls, ps_set, callback): ids = [ps.id for ps in ps_set] key = tuple(ids) cls.get().observed_all_ps[key].append(callback) @classmethod def watch_task(cls, task, callback): cls.get().observed_task[task.id].append(callback) @classmethod def watch_all_tasks(cls, tasks, callback): key = tuple([t.id for t in tasks]) for t in tasks: pair = (key, callback) cls.get().observed_all_tasks[t.id].append(pair) @classmethod def async (cls, func, *args, **kwargs): self = cls.get() def _f(): func(*args, **kwargs) self._loop_fiber.switch() fb = Fiber(target=_f) self._fibers.append(fb) @classmethod def await_ps(cls, ps): self = cls.get() fb = Fiber.current() def _callback(ps): self._fibers.append(fb) cls.watch_ps(ps, _callback) self._loop_fiber.switch() @classmethod def await_all_ps(cls, ps_set): self = cls.get() fb = Fiber.current() def _callback(pss): self._fibers.append(fb) cls.watch_all_ps(ps_set, _callback) self._loop_fiber.switch() @classmethod def await_task(cls, task): self = cls.get() fb = Fiber.current() def _callback(ps): self._fibers.append(fb) cls.watch_task(task, _callback) self._loop_fiber.switch() @classmethod def await_all_tasks(cls, tasks): self = cls.get() fb = Fiber.current() def _callback(ts): self._fibers.append(fb) cls.watch_all_tasks(tasks, _callback) self._loop_fiber.switch() def _loop(self): self._launch_all_fibers() self._submit_all() self._logger.debug("start polling") t = self._receive_result() while t: self._exec_callback_for_task(t) self._exec_callback_for_all_task(t) if isinstance(t, Run): ps = t.parameter_set() if ps.is_finished(): self._exec_callback() self._submit_all() t = self._receive_result() def _default_logger(self): logger = logging.getLogger(__name__) log_level = logging.INFO if 'CARAVAN_SEARCH_ENGINE_LOGLEVEL' in os.environ: s = os.environ['CARAVAN_SEARCH_ENGINE_LOGLEVEL'] levels = { 'DEBUG': logging.DEBUG, 'INFO': logging.INFO, 'WARNING': logging.WARNING, 'ERROR': logging.ERROR, 'CRITICAL': logging.CRITICAL } log_level = levels[s] logger.setLevel(log_level) logger.propagate = False if not logger.handlers: ch = logging.StreamHandler() ch.setLevel(log_level) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) return logger def _has_callbacks(self): return (len(self.observed_ps) + len(self.observed_all_ps)) > 0 def _has_unfinished_tasks(self): for r in Task.all()[:self.max_submitted_task_id]: if not r.is_finished(): return True return False def _submit_all(self): tasks_to_be_submitted = [ t for t in Task.all()[self.max_submitted_task_id:] if not t.is_finished() ] self._logger.debug("submitting %d Tasks" % len(tasks_to_be_submitted)) self._print_tasks(tasks_to_be_submitted) self.max_submitted_task_id = len(Task.all()) def _print_tasks(self, tasks): for t in tasks: line = "%d %s\n" % (t.id, t.command) self._out.write(line) self._out.write("\n") def _launch_all_fibers(self): while self._fibers: f = self._fibers.pop(0) self._logger.debug("starting fiber") f.switch() def _exec_callback(self): while self._check_completed_ps() or self._check_completed_ps_all(): pass def _check_completed_ps(self): executed = False for psid in list(self.observed_ps.keys()): callbacks = self.observed_ps[psid] ps = ParameterSet.find(psid) while ps.is_finished() and len(callbacks) > 0: self._logger.debug("executing callback for ParameterSet %d" % ps.id) f = callbacks.pop(0) f(ps) self._launch_all_fibers() executed = True empty_keys = [k for k, v in self.observed_ps.items() if len(v) == 0] for k in empty_keys: self.observed_ps.pop(k) return executed def _check_completed_ps_all(self): executed = False for psids in list(self.observed_all_ps.keys()): pss = [ParameterSet.find(psid) for psid in psids] callbacks = self.observed_all_ps[psids] while len(callbacks) > 0 and all([ps.is_finished() for ps in pss]): self._logger.debug("executing callback for ParameterSet %s" % repr(psids)) f = callbacks.pop(0) f(pss) self._launch_all_fibers() executed = True empty_keys = [ k for k, v in self.observed_all_ps.items() if len(v) == 0 ] for k in empty_keys: self.observed_all_ps.pop(k) return executed def _exec_callback_for_task(self, task): executed = False callbacks = self.observed_task[task.id] while len(callbacks) > 0: self._logger.debug("executing callback for Task %d" % task.id) f = callbacks.pop(0) f(task) self._launch_all_fibers() executed = True self.observed_task.pop(task.id) return executed def _exec_callback_for_all_task(self, task): executed = False callback_pairs = self.observed_all_tasks[task.id] to_be_removed = [] for (idx, pair) in enumerate(callback_pairs): task_ids = pair[0] if all([Task.find(t).is_finished() for t in task_ids]): self._logger.debug("executing callback for Tasks %s" % str(task_ids)) f = pair[1] f(Task.find(t) for t in task_ids) to_be_removed.append(idx) self._launch_all_fibers() executed = True for idx in to_be_removed: callback_pairs.pop(idx) if len(callback_pairs) == 0: self.observed_all_tasks.pop(task.id) return executed def _receive_result(self): line = sys.stdin.readline() if not line: return None line = line.rstrip() self._logger.debug("received: %s" % line) if not line: return None l = line.split(' ') tid, rc, place_id, start_at, finish_at = [int(x) for x in l[:5]] results = [float(x) for x in l[5:]] t = Task.find(tid) t.store_result(results, rc, place_id, start_at, finish_at) self._logger.debug("stored result of Task %d" % tid) return t def _debug(self): sys.stderr.write(str(self.observed_ps) + "\n") sys.stderr.write(str(self.observed_all_ps) + "\n")