class DataMonitor(object): _STOP_REQUEST = object() def __init__(self, client, path, callback, args, kwargs): self.client = client self.path = path self.callback = callback self.args = args self.kwargs = kwargs self.started = AsyncResult() self.queue = Queue() self._delay = 1.343 self.max_delay = 180 def _monitor(self): """Run the monitoring loop.""" def watcher(event): self.queue.put(event) while True: try: data, stat = self.client.get(self.path, watcher) except zookeeper.NoNodeException: if not self.started.ready(): self.started.set(None) gevent.sleep(1) continue except (zookeeper.ConnectionLossException, zookeeper.SessionExpiredException, zookeeper.InvalidStateException), err: if not self.started.ready(): self.started.set_exception(err) break logging.error("got %r while monitoring %s", str(err), self.path) gevent.sleep(self._delay) self._delay += self._delay * random.random() self._delay = min(self._delay, self.max_delay) continue except Exception, err: if not self.started.ready(): self.started.set_exception(err) break raise self.callback(data, *self.args, **self.kwargs) if not self.started.ready(): self.started.set(None) event = self.queue.get() if event is self._STOP_REQUEST: break
class DataMonitor(object): _STOP_REQUEST = object() def __init__(self, client, path, callback, args, kwargs): self.client = client self.path = path self.callback = callback self.args = args self.kwargs = kwargs self.started = AsyncResult() self.queue = Queue() self._delay = 1.343 self.max_delay = 180 def _monitor(self): """Run the monitoring loop.""" def watcher(event): self.queue.put(event) while True: try: data, stat = self.client.get(self.path, watcher) except zookeeper.NoNodeException: if not self.started.ready(): self.started.set(None) gevent.sleep(1) continue except (zookeeper.ConnectionLossException, zookeeper.SessionExpiredException, zookeeper.InvalidStateException), err: if not self.started.ready(): self.started.set_exception(err) break logging.error("got %r while monitoring %s", str(err), self.path) gevent.sleep(self._delay) self._delay += self._delay * random.random() self._delay = min(self._delay, self.max_delay) continue except Exception, err: if not self.started.ready(): self.started.set_exception(err) break raise self.callback(data, *self.args, **self.kwargs) if not self.started.ready(): self.started.set(None) event = self.queue.get() if event is self._STOP_REQUEST: break
def send_raw_with_result(self, data, receiver_address): """ Sends data to receiver_address and returns an AsyncResult that will be set once the message is acknowledged. Always returns same AsyncResult instance for equal input. """ host_port = self.get_host_port(receiver_address) echohash = sha3(data + receiver_address) if echohash not in self.senthashes_to_states: async_result = AsyncResult() self.senthashes_to_states[echohash] = SentMessageState( async_result, receiver_address, ) else: async_result = self.senthashes_to_states[echohash].async_result if not async_result.ready(): self.transport.send( self.raiden, host_port, data, ) return async_result
def test__control_flow_expired_call(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) self.addCleanup(p.stop) def make_call(call, ctx, val): ar = p._routing_call(call, ctx, val) return ar.get(timeout=10) ctx = { 'reply-by' : 0 } # no need for real time, as it compares by CURRENT >= this value futurear = AsyncResult() with patch('pyon.ion.process.greenlet') as gcm: waitar = AsyncResult() gcm.getcurrent().kill.side_effect = lambda *a, **k: waitar.set() ar = p._routing_call(futurear.set, ctx, sentinel.val) waitar.get(timeout=10) # futurear is not set self.assertFalse(futurear.ready()) # neither is the ar we got back from routing_call self.assertFalse(ar.ready()) # we should've been killed, though self.assertEquals(gcm.getcurrent().kill.call_count, 1) self.assertIsInstance(gcm.getcurrent().kill.call_args[1]['exception'], IonTimeout) # put a new call through (to show unblocked) futurear2 = AsyncResult() ar2 = p._routing_call(futurear2.set, MagicMock(), sentinel.val2) ar2.get(timeout=2)
def test_zero_max_size(self): q = queue.Channel() def sender(evt, q): q.put('hi') evt.set('done') def receiver(evt, q): x = q.get() evt.set(x) e1 = AsyncResult() e2 = AsyncResult() p1 = gevent.spawn(sender, e1, q) gevent.sleep(0.001) self.assert_(not e1.ready()) p2 = gevent.spawn(receiver, e2, q) self.assertEquals(e2.get(), 'hi') self.assertEquals(e1.get(), 'done') timeout = gevent.Timeout.start_new(0) try: gevent.joinall([p1, p2]) finally: timeout.cancel()
def test__interrupt_control_thread(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) self.addCleanup(p.stop) # put a call in that will never finish waitar = AsyncResult() # test specific, wait for this to indicate we're being processed/hung callar = AsyncResult() # test specific, an ar that is just waited on by the spin call def spin(inar, outar): outar.set(True) inar.wait() ar = p._routing_call(spin, MagicMock(), callar, waitar) # wait until we get notice we're being processed waitar.get(timeout=2) # interrupt it p._interrupt_control_thread() # the ar we got back from routing_call will not be set, it never finished the call self.assertFalse(ar.ready()) # to prove we're unblocked, run another call through the control thread ar2 = p._routing_call(callar.set, MagicMock(), sentinel.val) ar2.get(timeout=2) self.assertTrue(callar.ready()) self.assertEquals(callar.get(), sentinel.val)
def test_zero_max_size(self): q = queue.Queue(0) def sender(evt, q): q.put('hi') evt.set('done') def receiver(evt, q): x = q.get() evt.set(x) e1 = AsyncResult() e2 = AsyncResult() p1 = gevent.spawn(sender, e1, q) gevent.sleep(0.001) self.assert_(not e1.ready()) p2 = gevent.spawn(receiver, e2, q) self.assertEquals(e2.get(), 'hi') self.assertEquals(e1.get(), 'done') timeout = gevent.Timeout.start_new(0) try: gevent.joinall([p1, p2]) finally: timeout.cancel()
def test__interrupt_control_thread(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) self.addCleanup(p.stop) # put a call in that will never finish waitar = AsyncResult( ) # test specific, wait for this to indicate we're being processed/hung callar = AsyncResult( ) # test specific, an ar that is just waited on by the spin call def spin(inar, outar): outar.set(True) inar.wait() ar = p._routing_call(spin, MagicMock(), callar, waitar) # wait until we get notice we're being processed waitar.get(timeout=2) # interrupt it p._interrupt_control_thread() # the ar we got back from routing_call will not be set, it never finished the call self.assertFalse(ar.ready()) # to prove we're unblocked, run another call through the control thread ar2 = p._routing_call(callar.set, MagicMock(), sentinel.val) ar2.get(timeout=2) self.assertTrue(callar.ready()) self.assertEquals(callar.get(), sentinel.val)
def send_raw_with_result(self, data, receiver_address): """ Sends data to receiver_address and returns an AsyncResult that will be set once the message is acknowledged. Always returns same AsyncResult instance for equal input. """ host_port = self.get_host_port(receiver_address) echohash = sha3(data + receiver_address) if echohash not in self.senthashes_to_states: async_result = AsyncResult() self.senthashes_to_states[echohash] = SentMessageState( async_result, receiver_address, ) else: async_result = self.senthashes_to_states[echohash].async_result if not async_result.ready(): self.transport.send( self.raiden, host_port, data, ) return async_result
class G(object): def __init__(self, pool_size=300, timeout=1): self.pool_size = pool_size self.timeout = timeout logging.basicConfig(format='%(message)s') self.logger = logging.getLogger(name='g') self.logger.setLevel(logging.INFO) self.pool = Pool(self.pool_size) self.async_result = AsyncResult() def url(self, ip, params=None): if params: suffix = '/search?q=%s' % urllib.quote_plus(' '.join(params)) else: suffix = '' return 'http://%s%s' % (ip, suffix) def worker(self, ip): if not self.async_result.ready(): try: resp = requests.head(self.url(ip), timeout=self.timeout) except requests.exceptions.RequestException: pass # ignore timeouts, connection errors else: if resp.status_code == requests.codes.ok: if not self.async_result.ready(): self.async_result.set(ip) def run(self): # read and parse ip json file gdir = os.path.dirname(__file__) path = os.path.join(gdir, 'ips.json') ips = json.load(open(path)) # start a thread for pool gevent.spawn(self.pool.map, self.worker, ips) # block main thread for the 1st available IP try: ip = self.async_result.get(timeout=5) except gevent.timeout.Timeout: self.logger.info('Timeout (5s).') else: self.pool.kill() self.logger.info(ip) webbrowser.open(self.url(ip, params=sys.argv[1:]))
class DataMonitor(object): _STOP_REQUEST = object() def __init__(self, client, path, callback, args, kwargs): self.client = client self.path = path self.callback = callback self.args = args self.kwargs = kwargs self.started = AsyncResult() self.queue = Queue() def _monitor(self): """Run the monitoring loop.""" def watcher(event): self.queue.put(event) while True: try: data, stat = self.client.get(self.path, watcher) except zookeeper.NoNodeException: if not self.started.ready(): self.started.set(None) gevent.sleep(1) continue except Exception, err: if not self.started.ready(): self.started.set_exception(err) break self.callback(data, *self.args, **self.kwargs) if not self.started.ready(): self.started.set(None) event = self.queue.get() if event is self._STOP_REQUEST: break
class Future: def __init__(self): self.result = AsyncResult() def set(self, value): self.result.set(value) def get(self): return self.result.get() def on_ready(self, func): while self.result.ready() != False: gevent.sleep(0) func(self.result.get())
def test_termination_message_is_ignored_when_sender_is_not_watched(defer): node = DummyNode() defer(node.stop) received = AsyncResult() class Watcher(Actor): def receive(self, msg): received.set(msg) dummy = node.spawn(Actor) w = node.spawn(Watcher) w << ('terminated', dummy) sleep(.01) ok_(not received.ready())
def test_termination_message_is_ignored_when_sender_is_not_watched(defer): node = DummyNode() defer(node.stop) received = AsyncResult() class Watcher(Actor): def receive(self, msg): received.set(msg) dummy = node.spawn(Actor) w = node.spawn(Watcher) w << ('terminated', dummy) sleep(.01) ok_(not received.ready())
def test_zero_max_size(self): q = queue.Channel() def sender(evt, q): q.put("hi") evt.set("done") def receiver(evt, q): x = q.get() evt.set(x) e1 = AsyncResult() e2 = AsyncResult() p1 = gevent.spawn(sender, e1, q) gevent.sleep(0.001) self.assertTrue(not e1.ready()) p2 = gevent.spawn(receiver, e2, q) self.assertEqual(e2.get(), "hi") self.assertEqual(e1.get(), "done") with gevent.Timeout(0): gevent.joinall([p1, p2])
def test_zero_max_size(self): q = queue.Channel() def sender(evt, q): q.put('hi') evt.set('done') def receiver(evt, q): x = q.get() evt.set(x) e1 = AsyncResult() e2 = AsyncResult() p1 = gevent.spawn(sender, e1, q) gevent.sleep(0.001) self.assertTrue(not e1.ready()) p2 = gevent.spawn(receiver, e2, q) self.assertEqual(e2.get(), 'hi') self.assertEqual(e1.get(), 'done') with gevent.Timeout(0): gevent.joinall([p1, p2])
def test__control_flow_expired_call(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) self.addCleanup(p.stop) def make_call(call, ctx, val): ar = p._routing_call(call, ctx, val) return ar.get(timeout=10) ctx = { 'reply-by': 0 } # no need for real time, as it compares by CURRENT >= this value futurear = AsyncResult() with patch('pyon.ion.process.greenlet') as gcm: waitar = AsyncResult() gcm.getcurrent().kill.side_effect = lambda *a, **k: waitar.set() ar = p._routing_call(futurear.set, ctx, sentinel.val) waitar.get(timeout=10) # futurear is not set self.assertFalse(futurear.ready()) # neither is the ar we got back from routing_call self.assertFalse(ar.ready()) # we should've been killed, though self.assertEquals(gcm.getcurrent().kill.call_count, 1) self.assertIsInstance( gcm.getcurrent().kill.call_args[1]['exception'], IonTimeout) # put a new call through (to show unblocked) futurear2 = AsyncResult() ar2 = p._routing_call(futurear2.set, MagicMock(), sentinel.val2) ar2.get(timeout=2)
class MsgBase(MsgSender,MsgReceiver): """A message which expects a reply.""" timeout = None blocking = False # True if the message needs a reply before sending more _timer = None _last_channel = None _send_err = None _recv_err = None def __init__(self,*a,**k): super(MsgBase,self).__init__(*a,**k) self.result = AsyncResult() def abort(self): if not self.result.ready(): self.result.set(RuntimeError("aborted")) super(MsgBase,self).abort() def list(self): s = super(MsgBase,self) if hasattr(s,'list'): yield s if self.timeout: yield("timeout",self.timeout) if self.result.ready(): try: yield("result",self.result.get()) except Exception as ex: yield("error",repr(ex)) else: yield("status","pending") def send(self,channel): """write myself to the channel. Return None|SEND_AGAIN|RECV_AGAIN.""" self._set_timeout() self._last_channel = channel if self._send_err is None: self._send_err = MSG_ERROR("You need to override %s.send"%self.__class__.__name__) return self._send_err def recv(self,data): """A message has been received. Return NOT_MINE|MINE|RECV_AGAIN|SEND_AGAIN.""" self._clear_timeout() if self._recv_err is None: self._recv_err = MSG_ERROR("You need to override %s.recv"%self.__class__.__name__) return self._recv_err def retry(self): """Check whether to retry this message""" self._clear_timeout() return SEND_AGAIN def done(self): """Processing is finished.""" self._clear_timeout() if self.result is not None and not self.result.successful(): raise RuntimeError("Did not trigger the result in %s.dataReceived()"%(self.__class__.__name__,)) def do_timeout(self): if self._last_channel is not None: self._last_channel.close() self._last_channel = None def _set_timeout(self): if self.timeout is not None: self._timer = callLater(True,self.timeout,self._timeout) def _clear_timeout(self): if self._timer is not None: self._timer.cancel() self._timer = None def _timeout(self): self._timer = None self.do_timeout()
class Popen(SubprocessProtocol): def __init__(self, args, bufsize=-1, stdin=None, stdout=None, stderr=None, shell=False, universal_newlines=False, **kwargs): """Create new Popen instance.""" assert not universal_newlines, "universal_newlines must be False" hub = get_hub() self.pid = None self.returncode = None self.universal_newlines = universal_newlines self.result = AsyncResult() self.stdin = None self.stdout = None self.stderr = None self._transport = None if shell: hub.wait_async(hub.loop.subprocess_shell( lambda: _DelegateProtocol(self), *args, stdin=stdin, stdout=stdout, stderr=stderr, bufsize=bufsize, **kwargs)) else: hub.wait_async(hub.loop.subprocess_exec( lambda: _DelegateProtocol(self), *args, stdin=stdin, stdout=stdout, stderr=stderr, bufsize=bufsize, **kwargs)) def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode) def communicate(self, input=None, timeout=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional input argument should be a string to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr).""" #TODO: timeout greenlets = [] if self.stdin: greenlets.append(spawn(write_and_close, self.stdin, input)) if self.stdout: stdout = spawn(self.stdout.read) greenlets.append(stdout) else: stdout = None if self.stderr: stderr = spawn(self.stderr.read) greenlets.append(stderr) else: stderr = None joinall(greenlets) if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() self.wait() return (None if stdout is None else stdout.value or '', None if stderr is None else stderr.value or '') def poll(self): return self._internal_poll() def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) # XXX unlink def send_signal(self, sig): self._transport.send_signal(sig) def terminate(self): self._transport.terminate() def kill(self): #noinspection PyProtectedMember if self._transport._proc is not None: self._transport.kill() if mswindows: # # Windows methods # def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: self.returncode = GetExitCodeProcess(self._handle) self.result.set(self.returncode) return self.returncode def rawlink(self, callback): if not self.result.ready() and not self._waiting: self._waiting = True Greenlet.spawn(self._wait) self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _blocking_wait(self): WaitForSingleObject(self._handle, INFINITE) self.returncode = GetExitCodeProcess(self._handle) return self.returncode def _wait(self): self.threadpool.spawn(self._blocking_wait).rawlink(self.result) def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: if not self._waiting: self._waiting = True self._wait() return self.result.wait(timeout=timeout) else: # # POSIX methods # def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" return self.result.wait(timeout=timeout)
class OutTimer(Collected): """Timer for timed outputs""" storage = OutTimers.storage q = None _timer = None _timer = None def __init__(self,parent,timer,nextval): global tseq tseq += 1 self.name = parent.name+(str(tseq),) super(OutTimer,self).__init__() self.parent = parent self.end = timer self.val = nextval self.q = AsyncResult() self._start() def info(self): return "%s:%s" % (self.name, self.val) def list(self): n = now() for r in super(OutTimer,self).list(): yield r yield ("output",self.parent.name) yield ("start", self.started) yield ("end", self.end) yield ("next value",self.val) def _start(self): if self._timer: self._timer.cancel() self.started = now() self._timer = callLater(False,self.end,self._timeout) def _timeout(self): self._timer = None try: self.parent.write(self.val) except Exception as ex: fix_exception(ex) self.q.set(ex) else: self.q.set(None) def done(self): """called externally via _tmwrite() when the external timer writes""" if self._timer: self._timer.cancel() self._timer = None if not self.q.ready(): self.q.set(None) def cancel(self): self.delete() def delete(self, ctx=None): if self._timer is not None: self._timer.cancel() self._timer = None self.q.set(DelayCancelled(self)) super(OutTimer,self).delete(ctx)
class ChildrenMonitor(object): """Simple monitor that monitors the children of a node and their content. """ _STOP_REQUEST = object() def __init__(self, client, path, into, factory, args, listener): self.client = client self.path = path self.into = into if into is not None else {} self.factory = factory if factory is not None else str self.args = args self.listener = listener or MonitorListener() self.started = AsyncResult() self.queue = Queue() self.stats = {} self._delay = 1.343 self.max_delay = 180 def _monitor(self): """Run the monitoring loop.""" def watcher(event): self.queue.put(event) while True: try: children = self.client.get_children(self.path, watcher) except zookeeper.NoNodeException: if not self.started.ready(): self.started.set(None) gevent.sleep(1) continue except (zookeeper.ConnectionLossException, zookeeper.SessionExpiredException, zookeeper.InvalidStateException) as err: if not self.started.ready(): self.started.set_exception(err) break logging.error("got %r while monitoring %s", str(err), self.path) gevent.sleep(self._delay) self._delay += self._delay * random.random() self._delay = min(self._delay, self.max_delay) continue except Exception, err: if not self.started.ready(): self.started.set_exception(err) break raise for child in children: if not child in self.stats: try: data, stat = self.client.get(os.path.join(self.path, child)) except zookeeper.NoNodeException: print "race condition while getting", os.path.join( self.path, child) else: self.into[child] = self.factory(data, *self.args) self.listener.created(child, self.into[child]) self.stats[child] = stat else: try: data, stat = self.client.get(os.path.join(self.path, child)) except zookeeper.NoNodeException: print "race condition while getting", os.path.join( self.path, child) # should we remove it here? else: if stat['version'] != self.stats[child]['version']: self.into[child] = self.factory(data, *self.args) self.listener.modified(child, self.into[child]) self.stats[child] = stat for child in self.into.keys(): if child not in children: del self.into[child] del self.stats[child] self.listener.deleted(child) if not self.started.ready(): self.started.set(None) self.listener.commit() event = self.queue.get() if event is self._STOP_REQUEST: break
class Popen(object): def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, threadpool=None): """Create new Popen instance.""" if not isinstance(bufsize, (int, long)): raise TypeError("bufsize must be an integer") hub = get_hub() if mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") if close_fds and (stdin is not None or stdout is not None or stderr is not None): raise ValueError( "close_fds is not supported on Windows " "platforms if you redirect stdin/stdout/stderr") if threadpool is None: threadpool = hub.threadpool self.threadpool = threadpool self._waiting = False else: # POSIX if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") if creationflags != 0: raise ValueError("creationflags is only supported on Windows " "platforms") assert threadpool is None self._loop = hub.loop self.stdin = None self.stdout = None self.stderr = None self.pid = None self.returncode = None self.universal_newlines = universal_newlines self.result = AsyncResult() # Input and output objects. The general principle is like # this: # # Parent Child # ------ ----- # p2cwrite ---stdin---> p2cread # c2pread <--stdout--- c2pwrite # errread <--stderr--- errwrite # # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects # are None when not using PIPEs. The child objects are None # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) self._execute_child(args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) if mswindows: if p2cwrite is not None: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) if c2pread is not None: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) if errread is not None: errread = msvcrt.open_osfhandle(errread.Detach(), 0) if p2cwrite is not None: self.stdin = FileObject(p2cwrite, 'wb') if c2pread is not None: if universal_newlines: self.stdout = FileObject(c2pread, 'rU') else: self.stdout = FileObject(c2pread, 'rb') if errread is not None: if universal_newlines: self.stderr = FileObject(errread, 'rU') else: self.stderr = FileObject(errread, 'rb') def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % ( self.__class__.__name__, id(self), self.pid, self.returncode) def _on_child(self, watcher): watcher.stop() status = watcher.rstatus if os.WIFSIGNALED(status): self.returncode = -os.WTERMSIG(status) else: self.returncode = os.WEXITSTATUS(status) self.result.set(self.returncode) def communicate(self, input=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional input argument should be a string to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr).""" greenlets = [] if self.stdin: greenlets.append(spawn(write_and_close, self.stdin, input)) if self.stdout: stdout = spawn(self.stdout.read) greenlets.append(stdout) else: stdout = None if self.stderr: stderr = spawn(self.stderr.read) greenlets.append(stderr) else: stderr = None joinall(greenlets) if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() self.wait() return (None if stdout is None else stdout.value or '', None if stderr is None else stderr.value or '') def poll(self): return self._internal_poll() def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) # XXX unlink if mswindows: # # Windows methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: return (None, None, None, None, None, None) p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None if stdin is None: p2cread = GetStdHandle(STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = CreatePipe(None, 0) elif stdin == PIPE: p2cread, p2cwrite = CreatePipe(None, 0) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) if stdout is None: c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = CreatePipe(None, 0) elif stdout == PIPE: c2pread, c2pwrite = CreatePipe(None, 0) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) if stderr is None: errwrite = GetStdHandle(STD_ERROR_HANDLE) if errwrite is None: _, errwrite = CreatePipe(None, 0) elif stderr == PIPE: errread, errwrite = CreatePipe(None, 0) elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" return DuplicateHandle(GetCurrentProcess(), handle, GetCurrentProcess(), 0, 1, DUPLICATE_SAME_ACCESS) def _find_w9xpopen(self): """Find and return absolut path to w9xpopen.exe""" w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), "w9xpopen.exe") if not os.path.exists(w9xpopen): # Eeek - file-not-found - possibly an embedding # situation - see if we can locate it in sys.exec_prefix w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), "w9xpopen.exe") if not os.path.exists(w9xpopen): raise RuntimeError("Cannot locate w9xpopen.exe, which is " "needed for Popen to work with your " "shell or platform.") return w9xpopen def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (MS Windows version)""" if not isinstance(args, types.StringTypes): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() if None not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite if shell: startupinfo.dwFlags |= STARTF_USESHOWWINDOW startupinfo.wShowWindow = SW_HIDE comspec = os.environ.get("COMSPEC", "cmd.exe") args = '{} /c "{}"'.format(comspec, args) if GetVersion() >= 0x80000000 or os.path.basename( comspec).lower() == "command.com": # Win9x, or using command.com on NT. We need to # use the w9xpopen intermediate program. For more # information, see KB Q150956 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) w9xpopen = self._find_w9xpopen() args = '"%s" %s' % (w9xpopen, args) # Not passing CREATE_NEW_CONSOLE has been known to # cause random failures on win9x. Specifically a # dialog: "Your program accessed mem currently in # use at xxx" and a hopeful warning about the # stability of your system. Cost is Ctrl+C wont # kill children. creationflags |= CREATE_NEW_CONSOLE # Start the process try: hp, ht, pid, tid = CreateProcess( executable, args, # no special security None, None, int(not close_fds), creationflags, env, cwd, startupinfo) except pywintypes.error as e: # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really # translate errno using _sys_errlist (or similar), but # how can this be done from Python? raise WindowsError(*e.args) finally: # Child is launched. Close the parent's copy of those pipe # handles that only the child should have open. You need # to make sure that no handles to the write end of the # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. if p2cread is not None: p2cread.Close() if c2pwrite is not None: c2pwrite.Close() if errwrite is not None: errwrite.Close() # Retain the process handle, but close the thread handle self._handle = hp self.pid = pid ht.Close() def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: self.returncode = GetExitCodeProcess(self._handle) self.result.set(self.returncode) return self.returncode def rawlink(self, callback): if not self.result.ready() and not self._waiting: self._waiting = True Greenlet.spawn(self._wait) self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _blocking_wait(self): WaitForSingleObject(self._handle, INFINITE) self.returncode = GetExitCodeProcess(self._handle) return self.returncode def _wait(self): self.threadpool.spawn(self._blocking_wait).rawlink(self.result) def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: if not self._waiting: self._waiting = True self._wait() return self.result.wait(timeout=timeout) def send_signal(self, sig): """Send a signal to the process """ if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: os.kill(self.pid, signal.CTRL_C_EVENT) elif sig == signal.CTRL_BREAK_EVENT: os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process """ TerminateProcess(self._handle, 1) kill = terminate else: # # POSIX methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None if stdin is None: pass elif stdin == PIPE: p2cread, p2cwrite = self.pipe_cloexec() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == PIPE: c2pread, c2pwrite = self.pipe_cloexec() elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == PIPE: errread, errwrite = self.pipe_cloexec() elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) if cloexec: fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) else: fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _remove_nonblock_flag(self, fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK) fcntl.fcntl(fd, fcntl.F_SETFL, flags) def pipe_cloexec(self): """Create a pipe with FDs set CLOEXEC.""" # Pipes' FDs are set CLOEXEC by default because we don't want them # to be inherited by other subprocesses: the CLOEXEC flag is removed # from the child's FDs by _dup2(), between fork() and exec(). # This is not atomic: we would need the pipe2() syscall for that. r, w = os.pipe() self._set_cloexec_flag(r) self._set_cloexec_flag(w) return r, w def _close_fds(self, but): if hasattr(os, 'closerange'): os.closerange(3, but) os.closerange(but + 1, MAXFD) else: for i in xrange(3, MAXFD): if i == but: continue try: os.close(i) except: pass def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (POSIX version)""" if isinstance(args, types.StringTypes): args = [args] else: args = list(args) if shell: args = ["/bin/sh", "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] self._loop.install_sigchld() # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = self.pipe_cloexec() try: try: gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = fork() except: if gc_was_enabled: gc.enable() raise if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # When duping fds, if there arises a situation # where one of the fds is either 0, 1 or 2, it # is possible that it is overwritten (#12607). if c2pwrite == 0: c2pwrite = os.dup(c2pwrite) if errwrite == 0 or errwrite == 1: errwrite = os.dup(errwrite) # Dup fds for child def _dup2(a, b): # dup2() removes the CLOEXEC flag but # we must do it ourselves if dup2() # would be a no-op (issue #10806). if a == b: self._set_cloexec_flag(a, False) elif a is not None: os.dup2(a, b) self._remove_nonblock_flag(b) _dup2(p2cread, 0) _dup2(c2pwrite, 1) _dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the # same fd more than once, or standard fds. closed = set([None]) for fd in [p2cread, c2pwrite, errwrite]: if fd not in closed and fd > 2: os.close(fd) closed.add(fd) # Close all other fds, if asked for if close_fds: self._close_fds(but=errpipe_write) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception( exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) finally: # Make sure that the process exits no matter what. # The return code does not matter much as it won't be # reported to the application os._exit(1) # Parent self._watcher = self._loop.child(self.pid) self._watcher.start(self._on_child, self._watcher) if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) if p2cread is not None and p2cwrite is not None: os.close(p2cread) if c2pwrite is not None and c2pread is not None: os.close(c2pwrite) if errwrite is not None and errread is not None: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising exception errpipe_read = FileObject(errpipe_read, 'rb') data = errpipe_read.read() finally: if hasattr(errpipe_read, 'close'): errpipe_read.close() else: os.close(errpipe_read) if data != "": self.wait() child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception def _handle_exitstatus(self, sts): if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) elif os.WIFEXITED(sts): self.returncode = os.WEXITSTATUS(sts) else: # Should never happen raise RuntimeError("Unknown child exit status!") def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" return self.result.wait(timeout=timeout) def send_signal(self, sig): """Send a signal to the process """ os.kill(self.pid, sig) def terminate(self): """Terminate the process with SIGTERM """ self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ self.send_signal(signal.SIGKILL)
class ProxyBot(Bot): def __init__(self, name, client, game, bot): self.name = name self.client = client self.bot = bot if bot: self.TIMEOUT = 60.0 else: self.TIMEOUT = None self.expecting = None self._vote = None self._select = None self._sabotage = None self._join = None self._part = None self.game = game def __call__(self, game, index, spy): """This function pretends to be a Builder, but in fact just configures this object in place as it's easier to setup and maintain.""" Player.__init__(self, self.name, index) self.state = game self.spy = spy self._join = Event() self.channel = '%s-player-%i' % (self.game, index) self.client.send_message(message.Join(self.channel)) self.client.send_message(message.Join(self.game)) # Use elegant /INVITE command for humans that have better clients. # FIXME: Invitation fails when there's already someone else in the channel. self.client.send_message(message.Command([self.name, self.channel], 'INVITE')) return self def bakeTeam(self, team): return ', '.join([str(p) for p in team]) def makeTeam(self, msg): for s in '\t,.!;?': msg = msg.replace(s, ' ') names = [n for n in msg.split(' ') if n] players = [] for n in names: players.append(self.makePlayer(n)) return players def makePlayer(self, name): for p in self.state.players: if str(p.index) in name: return p if name in p.name: return p assert False, "Can't find player for input name '%s'." % (name) def makeAnnouncement(self, msg): return {self.makePlayer(m.group(1)): float(m.group(2).rstrip('.')) for m in RE_MAPPING.finditer(msg)} def send(self, msg): self.client.msg(self.channel, msg) def onGameRevealed(self, players, spies): roles = {True: "Spy", False: "Resistance"} s = "" if self.spy: s = "; SPIES " + self.bakeTeam(spies) w = self._join.wait() # timeout=self.Timeout assert w is not None, "Problem with bot %r joining." % self self._join = None self.send('REVEAL %s; ROLE %s; PLAYERS %s%s.' % (self.game, roles[self.spy], self.bakeTeam(players), s)) def onMissionAttempt(self, mission, tries, leader): self.send('MISSION %i.%i; LEADER %s.' % (mission, tries, Player.__repr__(leader))) def select(self, players, count): self._select = AsyncResult() self.state.count = count self.expecting = self.process_SELECTED self.send('SELECT %i!' % (count)) if not self.bot: self.send('/me ' + self.expecting.__doc__) selection = self._select.get(timeout=self.TIMEOUT) self._select = None return selection def process_SELECTED(self, msg): """Type a list of players to select for the team, e.g. `select 1, 2.`""" if 'select' in msg[1].lower(): msg = ' '.join(msg[2:]) else: msg = ' '.join(msg[1:]) team = self.makeTeam(msg) if len(team) != self.state.count: self.send('SELECT %i?' % (self.state.count)) else: assert self._select is not None self._select.set(team) def onTeamSelected(self, leader, team): self._vote = AsyncResult() self.expecting = self.process_VOTED self.state.team = team[:] self.send("VOTE %s?" % (self.bakeTeam(team))) if not self.bot: self.send('/me ' + self.expecting.__doc__) def vote(self, team): v = self._vote.get(timeout=self.TIMEOUT) self._vote = None return v def process_VOTED(self, msg): """Enter your vote, for example as `YES` or `NO`.""" result = parseYesOrNo(' '.join(msg[1:])) if result is not None: assert self._vote is not None self._vote.set(result) def onVoteComplete(self, votes): self.send("VOTES %s." % (', '.join([showYesOrNo(v) for v in votes]))) v = [b for b in votes if b] if self in self.state.team and len(v) > 2: self._sabotage = AsyncResult() self.expecting = self.process_SABOTAGED self.send("SABOTAGE?") if not self.bot: self.send('/me ' + self.expecting.__doc__) else: self._sabotage = None def sabotage(self): assert self._sabotage is not None s = self._sabotage.get(timeout=self.TIMEOUT) self._sabotage = None return s def process_SABOTAGED(self, msg): """Decide whether to sabotage, for typing in `YES` or `NO`.""" result = parseYesOrNo(' '.join(msg[1:])) if result is not None: assert self._sabotage is not None if result and not self.spy: self.send("Can't sabotage mission: you are resistance!") result = False self._sabotage.set(result) def onMissionComplete(self, sabotaged): # Force synchronization in case sabotage() is not called due to the bot # being resistance. This helps hide human identity by having the same # input delay in Spy or Resistance cases. if self._sabotage and not self._sabotage.ready(): s = self._sabotage.get(timeout=self.TIMEOUT) assert not s, "Expecting sabotage() to be False if it was handled automatically." self.send("SABOTAGES %i." % (sabotaged)) self.expecting = None self.do_announce() def onMissionFailed(self, leader, team): self.do_announce() def do_announce(self): self._announce = AsyncResult() self.expecting = self.process_ANNOUNCED self.send('ANNOUNCE!') if not self.bot: self.send('/me ' + self.expecting.__doc__) def announce(self): timeout = self.TIMEOUT if self.bot else 10.0 try: ann = self._announce.get(timeout=timeout) except Timeout as t: if self.bot: raise t ann = {} self._announce = None return ann def process_ANNOUNCED(self, msg): """Input a list of players and their spy probabilities, e.g. 3: 0.0, 4: 1.0.""" if 'announce' in msg[1].lower(): msg = ' '.join(msg[2:]) else: msg = ' '.join(msg[1:]) ann = self.makeAnnouncement(msg) self._announce.set(ann) def onAnnouncement(self, source, announcement): self.send("ANNOUNCES %s: %r" % (source, announcement)) def onGameComplete(self, win, spies): if not self.spy: self.send("RESULT %s; SPIES %s." % ("Win" if win else "Loss", self.bakeTeam(spies))) else: self.send("RESULT %s." % ("Loss" if win else "Win",)) self.client.send_message(message.Command(self.game, 'PART')) # Bots wait for the host to leave the channel for synchronization # purposes, but for humans we can display the results anyway. self._part = Event() if self.bot: self._part.wait(timeout=self.TIMEOUT) self.client.send_message(message.Command(self.channel, 'PART'))
class IRCClientManager(HasLogger, GClient): """An abstraction around an irc connection that provides automatic reconnects while preserving channel membership and re-enqueuing any messages we know were never sent (so messages may still be lost, but it is less likely). Takes a generic callback with args (irc_client_manager, msg) for all incoming PrivMsgs. """ def __init__(self, host, nick, callback, channels=None, logger=None, **irc_kwargs): irc_kwargs.update(hostname=host, nick=nick) self.recv_callback = callback self.irc_kwargs = irc_kwargs self.channels = set() if channels is None else channels self.channel_pending = defaultdict(lambda: 0) # {channel: num of pending messages on queue} self._client = AsyncResult() self._recv_queue = Queue() super(IRCClientManager, self).__init__(logger=logger) @property def all_open_channels(self): """self.channels is desired channels, self.channel_pending tracks any with unsent messages, actual open channels is a union of the two.""" return self.channels | set(self.channel_pending.keys()) def _start(self): self._client_loop_worker = self.group.spawn(self._client_loop) def _client_loop(self): try: backoff = Backoff(start=0.1, limit=10, rate=5) while True: self.logger.info("Starting new irc connection") client = girc.Client(**self.irc_kwargs) self.logger.debug("Joining channels: {}".format(self.all_open_channels)) for channel in self.all_open_channels: client.channel(channel).join() client.handler(self._client_recv, command=girc.message.Privmsg) self._client.set(client) try: client.start() self.logger.debug("Started new irc connection") backoff.reset() client.wait_for_stop() except Exception as ex: self.logger.warning("irc connection died, retrying in {}".format(backoff.peek()), exc_info=True) # clear _client if no-one else has if self._client.ready(): assert self._client.get() is client self._client = AsyncResult() gevent.sleep(backoff.get()) else: self.logger.info("irc connection exited gracefully, stopping") self.stop() # graceful exit return except Exception as ex: self.stop(ex) def _client_recv(self, client, msg): self._recv_queue.put(msg) @property def client(self): while True: waiter = self._client client = waiter.get() if not client._stopping: return client # clear _client if no-one else has if self._client is waiter: self._client = AsyncResult() def send(self, channel, text): self.channel_pending[channel] += 1 self.logger.debug("Enqueuing message for channel {} ({} now pending): {!r}".format( channel, self.channel_pending[channel], text )) super(IRCClientManager, self).send((channel, text)) def update_channels(self, new_channels): old_channels = self.all_open_channels self.channels = new_channels if not self._client.ready() or self._client.get()._stopping: self.logger.debug("Ignoring channel resync, client not running") return # no active connection, we're done client = self._client.get() self.logger.debug("Updating channels: {} to {}".format(old_channels, self.all_open_channels)) for channel in old_channels - self.all_open_channels: client.channel(channel).part() for channel in new_channels - self.all_open_channels: client.channel(channel).join() def _send(self, msg): if msg == 'stop': self.logger.debug("Calling graceful stop due to stop message on queue") self.stop() # does not return assert False channel, text = msg try: girc.message.Privmsg(self.client, channel, text).send(block=True) except Exception: # we can't be certain the message wasn't sent, so discard it # but at least we know all remaining items in the queue have not been. pass finally: self.channel_pending[channel] -= 1 self.logger.debug("Sent message for channel {} ({} now pending): {!r}".format( channel, self.channel_pending[channel], text )) if self.channel_pending[channel] <= 0: del self.channel_pending[channel] if self._client.ready() and channel not in self.all_open_channels: self._client.get().channel(channel).part() def _receive(self): for msg in self._recv_queue: if msg.target not in self.channels: self.logger.debug("Ignoring message {}, not a channel we care about".format(msg)) # ignore PMs and messages from channels we're only holding open while we finish sending return try: self.recv_callback(self, msg) except Exception: self.logger.exception("Failed to handle message {}".format(msg)) pass def wait_and_stop(self): """Graceful stop. Waits to send all remaining messages.""" self.logger.debug("Setting stop message for flushing send queue then stop") super(IRCClientManager, self).send('stop') self.wait_for_stop() def _stop(self, ex): if self._client.ready(): self._client.get().quit()
class MessageChannel(Channel): """A channel that adds useful semantics for publishing and consuming messages. Semantics that are added: * Support for registering consumers to receive basic.deliver events. Consumers also receive errors and are automatically deregistered when basic_cancel is received. * Support for the RabbitMQ extension confirm_select, which makes basic_publish block * Can check for messages returned with basic_return """ def __init__(self, connection, id): super(MessageChannel, self).__init__(connection, id) self.consumer_id = 1 self.consumers = {} self.returned = AsyncResult() self.listeners.set_handler('basic.deliver', self.on_deliver) self.listeners.set_handler('basic.return', self.on_basic_return) self.listeners.set_handler('basic.cancel-ok', self.on_cancel_ok) self.listeners.set_handler('basic.cancel', self.on_cancel_ok) def on_deliver(self, message): """Called when a message is received. Dispatches the message to the registered consumer. """ self.consumers[message.consumer_tag](message) def on_basic_return(self, msg): """When we receive a basic.return message, store it. The value can later be checked using .check_returned(). """ self.returned.set(msg) def check_returned(self): """Raise an error if a message has been returned. This also clears the returned frame, with the intention that each basic.return message may cause at most one MessageReturned error. """ if self._method and self._method.name == 'basic.return': self.must_now_block() returned = self.returned.get() else: try: returned = self.returned.get_nowait() except gevent.Timeout: return self.clear_returned() if returned: raise exceptions.return_exception_from_frame(returned) def clear_returned(self): """Discard any returned message.""" if self.returned.ready(): # we can only replace returned if it is ready - otherwise anything # that was blocked waiting would wait forever. self.returned = AsyncResult() def on_error(self, exc): """Override on_error, to pass error to all consumers.""" for consumer in self.consumers.values(): self.queue.put((consumer, (exc, ))) super(MessageChannel, self).on_error(exc) def on_cancel_ok(self, frame): """The server has cancelled a consumer. We can remove its consumer tag from the registered consumers.""" del (self.consumers[frame.consumer_tag]) def basic_consume(self, queue='', no_local=False, no_ack=False, exclusive=False, arguments={}, callback=None): """Begin consuming messages from a queue. Consumers last as long as the channel they were declared on, or until the client cancels them. :param queue: Specifies the name of the queue to consume from. :param no_local: Do not deliver own messages. If this flag is set the server will not send messages to the connection that published them. :param no_ack: Don't require acknowledgements. If this flag is set the server does not expect acknowledgements for messages. That is, when a message is delivered to the client the server assumes the delivery will succeed and immediately dequeues it. This functionality may increase performance but at the cost of reliability. Messages can get lost if a client dies before they are delivered to the application. :param exclusive: Request exclusive consumer access, meaning only this consumer can access the queue. :param arguments: A set of arguments for the consume. The syntax and semantics of these arguments depends on the server implementation. :param callback: A callback to be called for each message received. """ tag = 'ct-%d' % self.consumer_id self.consumer_id += 1 kwargs = dict(queue=queue, no_local=no_local, no_ack=no_ack, exclusive=exclusive, arguments=arguments, consumer_tag=tag) if callback is not None: self.consumers[tag] = callback return super(MessageChannel, self).basic_consume(**kwargs) else: queue = MessageQueue(self, tag) self.consumers[tag] = queue.put super(MessageChannel, self).basic_consume(**kwargs) return queue def basic_get(self, *args, **kwargs): """Wrap basic_get to return None if the response is basic.get-empty. This will be easier for users to check than testing whether a response is get-empty. """ r = super(MessageChannel, self).basic_get(*args, **kwargs) return r if isinstance(r, Message) else None def confirm_select(self, nowait=False): """Turn on RabbitMQ's publisher acknowledgements. See http://www.rabbitmq.com/confirms.html There are two things that need to be done: * Swap basic_publish to a version that blocks waiting for the corresponding ack. * Support nowait (because this method blocks or not depending on that argument) """ self.basic_publish = self.basic_publish_with_confirm if nowait: super(MessageChannel, self).confirm_select(nowait=nowait) else: # Send frame directly, as no callback will be received self._send(spec.FrameConfirmSelect(1)) def basic_publish_with_confirm(self, exchange='', routing_key='', mandatory=False, immediate=False, headers={}, body=''): """Version of basic publish that blocks waiting for confirm.""" method = super(MessageChannel, self).basic_publish self.clear_returned() ret = self._call_sync(method, ('basic.ack', 'basic.nack'), exchange, routing_key, mandatory, immediate, headers, body) if ret.name == 'basic.nack': raise exceptions.PublishFailed(ret) if mandatory or immediate: self.check_returned() return ret
class InputTable(scheme.Table): _table_name = 'input' _table_collection = input_tables id = scheme.Column('api') type = scheme.Column('api') parent = scheme.Column( 'api', lambda self, value: value and [value._table_name, value.id] or None) timeout = scheme.Column( 'api', lambda self, timeout: timeout and int(timeout.eta * 1000) or None) elements = scheme.Column('api') result = scheme.Column('api') close_aborts = scheme.Column('api') ignore_api = False def __init__(self, type, parent, timeout, elements, close_aborts, ignore_api=False): self.type = type self.parent = parent self.timeout = None self.elements = [isinstance(e, list) and e or [e] for e in elements] self.close_aborts = close_aborts self.ignore_api = ignore_api if parent: parent.input = self self._result = AsyncResult() self.reset_timeout(timeout) def set_result(self, value): if self._result.ready(): #raise RuntimeError('result of input already set') return with scheme.transaction: self.result = value self.reset_timeout(None) self._result.set(value) event.fire("input:result", self) def set_error(self, value): if self._result.ready(): #raise RuntimeError('result of input already set') return with scheme.transaction: self.result = str(value) self.reset_timeout(None) self._result.set_exception(value) event.fire("input:error", self) def reset_timeout(self, timeout): with scheme.transaction: if self.timeout: self.timeout.kill() if timeout: self.timeout = gevent.spawn_later(timeout, self._timed_out) self.timeout.eta = time.time() + timeout elif self.timeout: self.timeout = None def _timed_out(self): with scheme.transaction: self.timeout = None self.set_error(InputTimeout())
class ChildrenMonitor(object): """Simple monitor that monitors the children of a node and their content. """ _STOP_REQUEST = object() def __init__(self, client, path, into, factory, args, listener): self.client = client self.path = path self.into = into if into is not None else {} self.factory = factory if factory is not None else str self.args = args self.listener = listener or MonitorListener() self.started = AsyncResult() self.queue = Queue() self.stats = {} self._delay = 1.343 self.max_delay = 180 def _monitor(self): """Run the monitoring loop.""" def watcher(event): self.queue.put(event) while True: try: children = self.client.get_children(self.path, watcher) except zookeeper.NoNodeException: if not self.started.ready(): self.started.set(None) gevent.sleep(1) continue except (zookeeper.ConnectionLossException, zookeeper.SessionExpiredException, zookeeper.InvalidStateException) as err: if not self.started.ready(): self.started.set_exception(err) break logging.error("got %r while monitoring %s", str(err), self.path) gevent.sleep(self._delay) self._delay += self._delay * random.random() self._delay = min(self._delay, self.max_delay) continue except Exception, err: if not self.started.ready(): self.started.set_exception(err) break raise for child in children: if not child in self.stats: try: data, stat = self.client.get( os.path.join(self.path, child)) except zookeeper.NoNodeException: print "race condition while getting", os.path.join( self.path, child) else: self.into[child] = self.factory(data, *self.args) self.listener.created(child, self.into[child]) self.stats[child] = stat else: try: data, stat = self.client.get( os.path.join(self.path, child)) except zookeeper.NoNodeException: print "race condition while getting", os.path.join( self.path, child) # should we remove it here? else: if stat['version'] != self.stats[child]['version']: self.into[child] = self.factory(data, *self.args) self.listener.modified(child, self.into[child]) self.stats[child] = stat for child in self.into.keys(): if child not in children: del self.into[child] del self.stats[child] self.listener.deleted(child) if not self.started.ready(): self.started.set(None) self.listener.commit() event = self.queue.get() if event is self._STOP_REQUEST: break
class ProxyBot(Bot): def __init__(self, name, client, game, bot): self.name = name self.client = client self.bot = bot if bot: self.TIMEOUT = 1.0 else: self.TIMEOUT = None self.expecting = None self._vote = None self._select = None self._sabotage = None self._join = None self._part = None self.game = game def __call__(self, game, index, spy): """This function pretends to be a Builder, but in fact just configures this object in place as it's easier to setup and maintain.""" Player.__init__(self, self.name, index) self.state = game self.spy = spy self.channel = '%s-player-%i' % (self.game, index) self.client.send_message(message.Join(self.channel)) self.client.send_message(message.Join(self.game)) self._join = Event() # Use elegant /INVITE command for humans that have better clients. self.client.send_message( message.Command([self.name, self.channel], 'INVITE')) return self def bakeTeam(self, team): return ', '.join([str(p) for p in team]) def makeTeam(self, msg): for s in '\t,.!;?': msg = msg.replace(s, ' ') names = [n for n in msg.split(' ') if n] players = [] for n in names: players.append(self.makePlayer(n)) return players def makePlayer(self, name): for p in self.state.players: if str(p.index) in name: return p if name in p.name: return p assert False, "Can't find player for input name '%s'." % (name) def send(self, msg): self.client.msg(self.channel, msg) def onGameRevealed(self, players, spies): roles = {True: "Spy", False: "Resistance"} s = "" if self.spy: s = "; SPIES " + self.bakeTeam(spies) self._join.wait() self._join = None self.send('REVEAL %s; ROLE %s; PLAYERS %s%s.' % (self.game, roles[self.spy], self.bakeTeam(players), s)) def onMissionAttempt(self, mission, tries, leader): self.send('MISSION %i.%i; LEADER %s.' % (mission, tries, Player.__repr__(leader))) def select(self, players, count): self.send('SELECT %i!' % (count)) self._select = AsyncResult() self.state.count = count self.expecting = self.process_SELECTED return self._select.get(timeout=self.TIMEOUT) def process_SELECTED(self, msg): if 'select' in msg[1].lower(): msg = ' '.join(msg[2:]) else: msg = ' '.join(msg[1:]) team = self.makeTeam(msg) if len(team) != self.state.count: self.send('SELECT %i?' % (self.state.count)) else: self._select.set(team) def onTeamSelected(self, leader, team): self.state.team = team[:] self.send("VOTE %s?" % (self.bakeTeam(team))) self._vote = AsyncResult() self.expecting = self.process_VOTED def vote(self, team): return self._vote.get(timeout=self.TIMEOUT) def process_VOTED(self, msg): result = parseYesOrNo(' '.join(msg[1:])) if result is not None: self._vote.set(result) def onVoteComplete(self, votes): self.send("VOTES %s." % (', '.join([showYesOrNo(v) for v in votes]))) v = [b for b in votes if b] if self in self.state.team and len(v) > 2: self.send("SABOTAGE?") self._sabotage = AsyncResult() self.expecting = self.process_SABOTAGED else: self._sabotage = None def sabotage(self): assert self._sabotage is not None return self._sabotage.get(timeout=self.TIMEOUT) def process_SABOTAGED(self, msg): result = parseYesOrNo(' '.join(msg[1:])) if result is not None: self._sabotage.set(result) def onMissionComplete(self, sabotaged): # Force synchronization in case sabotage() is not called due to the bot # being resistance. This helps hide human identity by having the same # input delay in Spy or Resistance cases. if self._sabotage and not self._sabotage.ready(): s = self._sabotage.get(timeout=self.TIMEOUT) assert not s, "Expecting sabotage() to be False if it was handled automatically." self.send("SABOTAGES %i." % (sabotaged)) self.expecting = None def onGameComplete(self, win, spies): self.send("RESULT %s; SPIES %s." % (showYesOrNo(win), self.bakeTeam(spies))) self.client.send_message(message.Command(self.game, 'PART')) self._part = Event() self._part.wait() self.client.send_message(message.Command(self.channel, 'PART'))
class Popen(object): def __init__(self, args, bufsize=None, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, threadpool=None, **kwargs): """Create new Popen instance.""" if not PY3 and kwargs: raise TypeError("Got unexpected keyword arguments", kwargs) pass_fds = kwargs.pop('pass_fds', ()) start_new_session = kwargs.pop('start_new_session', False) restore_signals = kwargs.pop('restore_signals', True) hub = get_hub() if bufsize is None: # bufsize has different defaults on Py3 and Py2 if PY3: bufsize = -1 else: bufsize = 0 if not isinstance(bufsize, integer_types): raise TypeError("bufsize must be an integer") if mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") any_stdio_set = (stdin is not None or stdout is not None or stderr is not None) if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: if any_stdio_set: close_fds = False else: close_fds = True elif close_fds and any_stdio_set: raise ValueError( "close_fds is not supported on Windows " "platforms if you redirect stdin/stdout/stderr") if threadpool is None: threadpool = hub.threadpool self.threadpool = threadpool self._waiting = False else: # POSIX if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: # close_fds has different defaults on Py3/Py2 if PY3: close_fds = True else: close_fds = False if pass_fds and not close_fds: import warnings warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) close_fds = True if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") if creationflags != 0: raise ValueError("creationflags is only supported on Windows " "platforms") assert threadpool is None self._loop = hub.loop if PY3: self.args = args self.stdin = None self.stdout = None self.stderr = None self.pid = None self.returncode = None self.universal_newlines = universal_newlines self.result = AsyncResult() # Input and output objects. The general principle is like # this: # # Parent Child # ------ ----- # p2cwrite ---stdin---> p2cread # c2pread <--stdout--- c2pwrite # errread <--stderr--- errwrite # # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects # are None when not using PIPEs. The child objects are None # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) # We wrap OS handles *before* launching the child, otherwise a # quickly terminating child could make our fds unwrappable # (see #8458). if mswindows: if p2cwrite is not None: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) if c2pread is not None: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) if errread is not None: errread = msvcrt.open_osfhandle(errread.Detach(), 0) if p2cwrite is not None: if PY3 and universal_newlines: # Under Python 3, if we left on the 'b' we'd get different results # depending on whether we used FileObjectPosix or FileObjectThread self.stdin = FileObject(p2cwrite, 'wb', bufsize) self.stdin._translate = True self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True, line_buffering=(bufsize == 1)) else: self.stdin = FileObject(p2cwrite, 'wb', bufsize) if c2pread is not None: if universal_newlines: if PY3: # FileObjectThread doesn't support the 'U' qualifier # with a bufsize of 0 self.stdout = FileObject(c2pread, 'rb', bufsize) # NOTE: Universal Newlines are broken on Windows/Py3, at least # in some cases. This is true in the stdlib subprocess module # as well; the following line would fix the test cases in # test__subprocess.py that depend on python_universal_newlines, # but would be inconsistent with the stdlib: #msvcrt.setmode(self.stdout.fileno(), os.O_TEXT) self.stdout.io = io.TextIOWrapper(self.stdout.io) self.stdout.io.mode = 'r' self.stdout._translate = True else: self.stdout = FileObject(c2pread, 'rU', bufsize) else: self.stdout = FileObject(c2pread, 'rb', bufsize) if errread is not None: if universal_newlines: if PY3: self.stderr = FileObject(errread, 'rb', bufsize) self.stderr.io = io.TextIOWrapper(self.stderr.io) self.stderr._translate = True else: self.stderr = FileObject(errread, 'rU', bufsize) else: self.stderr = FileObject(errread, 'rb', bufsize) self._closed_child_pipe_fds = False try: self._execute_child(args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session) except: # Cleanup if the child failed starting. # (gevent: New in python3, but reported as gevent bug in #347. # Note that under Py2, any error raised below will replace the # original error so we have to use reraise) if not PY3: exc_info = sys.exc_info() for f in filter(None, (self.stdin, self.stdout, self.stderr)): try: f.close() except (OSError, IOError): pass # Ignore EBADF or other errors. if not self._closed_child_pipe_fds: to_close = [] if stdin == PIPE: to_close.append(p2cread) if stdout == PIPE: to_close.append(c2pwrite) if stderr == PIPE: to_close.append(errwrite) if hasattr(self, '_devnull'): to_close.append(self._devnull) for fd in to_close: try: os.close(fd) except (OSError, IOError): pass if not PY3: try: reraise(*exc_info) finally: del exc_info raise def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % ( self.__class__.__name__, id(self), self.pid, self.returncode) def _on_child(self, watcher): watcher.stop() status = watcher.rstatus if os.WIFSIGNALED(status): self.returncode = -os.WTERMSIG(status) else: self.returncode = os.WEXITSTATUS(status) self.result.set(self.returncode) def _get_devnull(self): if not hasattr(self, '_devnull'): self._devnull = os.open(os.devnull, os.O_RDWR) return self._devnull _stdout_buffer = None _stderr_buffer = None def communicate(self, input=None, timeout=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional input argument should be a string to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr). :keyword timeout: Under Python 2, this is a gevent extension; if given and it expires, we will raise :class:`gevent.timeout.Timeout`. Under Python 3, this raises the standard :exc:`TimeoutExpired` exception. """ greenlets = [] if self.stdin: greenlets.append(spawn(write_and_close, self.stdin, input)) # If the timeout parameter is used, and the caller calls back after # getting a TimeoutExpired exception, we can wind up with multiple # greenlets trying to run and read from and close stdout/stderr. # That's bad because it can lead to 'RuntimeError: reentrant call in io.BufferedReader'. # We can't just kill the previous greenlets when a timeout happens, # though, because we risk losing the output collected by that greenlet # (and Python 3, where timeout is an official parameter, explicitly says # that no output should be lost in the event of a timeout.) Instead, we're # watching for the exception and ignoring it. It's not elegant, # but it works if self.stdout: def _read_out(): try: data = self.stdout.read() except RuntimeError: return if self._stdout_buffer is not None: self._stdout_buffer += data else: self._stdout_buffer = data stdout = spawn(_read_out) greenlets.append(stdout) else: stdout = None if self.stderr: def _read_err(): try: data = self.stderr.read() except RuntimeError: return if self._stderr_buffer is not None: self._stderr_buffer += data else: self._stderr_buffer = data stderr = spawn(_read_err) greenlets.append(stderr) else: stderr = None # If we were given stdin=stdout=stderr=None, we have no way to # communicate with the child, and thus no greenlets to wait # on. This is a nonsense case, but it comes up in the test # case for Python 3.5 (test_subprocess.py # RunFuncTestCase.test_timeout). Instead, we go directly to # self.wait if not greenlets and timeout is not None: result = self.wait(timeout=timeout) # Python 3 would have already raised, but Python 2 would not # so we need to do that manually if result is None: from gevent.timeout import Timeout raise Timeout(timeout) done = joinall(greenlets, timeout=timeout) if timeout is not None and len(done) != len(greenlets): if PY3: raise TimeoutExpired(self.args, timeout) from gevent.timeout import Timeout raise Timeout(timeout) if self.stdout: try: self.stdout.close() except RuntimeError: pass if self.stderr: try: self.stderr.close() except RuntimeError: pass self.wait() stdout_value = self._stdout_buffer self._stdout_buffer = None stderr_value = self._stderr_buffer self._stderr_buffer = None # XXX: Under python 3 in universal newlines mode we should be # returning str, not bytes return (None if stdout is None else stdout_value or b'', None if stderr is None else stderr_value or b'') def poll(self): return self._internal_poll() if PY3: def __enter__(self): return self def __exit__(self, type, value, traceback): if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() try: # Flushing a BufferedWriter may raise an error if self.stdin: self.stdin.close() finally: # Wait for the process to terminate, to avoid zombies. # JAM: gevent: If the process never terminates, this # blocks forever. self.wait() if mswindows: # # Windows methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: return (None, None, None, None, None, None) p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None try: DEVNULL except NameError: _devnull = object() else: _devnull = DEVNULL if stdin is None: p2cread = GetStdHandle(STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = CreatePipe(None, 0) if PY3: p2cread = Handle(p2cread) _winapi.CloseHandle(_) elif stdin == PIPE: p2cread, p2cwrite = CreatePipe(None, 0) if PY3: p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) elif stdin == _devnull: p2cread = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) if stdout is None: c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = CreatePipe(None, 0) if PY3: c2pwrite = Handle(c2pwrite) _winapi.CloseHandle(_) elif stdout == PIPE: c2pread, c2pwrite = CreatePipe(None, 0) if PY3: c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) elif stdout == _devnull: c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) if stderr is None: errwrite = GetStdHandle(STD_ERROR_HANDLE) if errwrite is None: _, errwrite = CreatePipe(None, 0) if PY3: errwrite = Handle(errwrite) _winapi.CloseHandle(_) elif stderr == PIPE: errread, errwrite = CreatePipe(None, 0) if PY3: errread, errwrite = Handle(errread), Handle(errwrite) elif stderr == STDOUT: errwrite = c2pwrite elif stderr == _devnull: errwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" return DuplicateHandle(GetCurrentProcess(), handle, GetCurrentProcess(), 0, 1, DUPLICATE_SAME_ACCESS) def _find_w9xpopen(self): """Find and return absolut path to w9xpopen.exe""" w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), "w9xpopen.exe") if not os.path.exists(w9xpopen): # Eeek - file-not-found - possibly an embedding # situation - see if we can locate it in sys.exec_prefix w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), "w9xpopen.exe") if not os.path.exists(w9xpopen): raise RuntimeError("Cannot locate w9xpopen.exe, which is " "needed for Popen to work with your " "shell or platform.") return w9xpopen def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session): """Execute program (MS Windows version)""" assert not pass_fds, "pass_fds not supported on Windows." if not isinstance(args, string_types): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() if None not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite if shell: startupinfo.dwFlags |= STARTF_USESHOWWINDOW startupinfo.wShowWindow = SW_HIDE comspec = os.environ.get("COMSPEC", "cmd.exe") args = '{} /c "{}"'.format(comspec, args) if GetVersion() >= 0x80000000 or os.path.basename( comspec).lower() == "command.com": # Win9x, or using command.com on NT. We need to # use the w9xpopen intermediate program. For more # information, see KB Q150956 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) w9xpopen = self._find_w9xpopen() args = '"%s" %s' % (w9xpopen, args) # Not passing CREATE_NEW_CONSOLE has been known to # cause random failures on win9x. Specifically a # dialog: "Your program accessed mem currently in # use at xxx" and a hopeful warning about the # stability of your system. Cost is Ctrl+C wont # kill children. creationflags |= CREATE_NEW_CONSOLE # Start the process try: hp, ht, pid, tid = CreateProcess( executable, args, # no special security None, None, int(not close_fds), creationflags, env, cwd, startupinfo) except IOError as e: # From 2.6 on, pywintypes.error was defined as IOError # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really # translate errno using _sys_errlist (or similar), but # how can this be done from Python? if PY3: raise # don't remap here raise WindowsError(*e.args) finally: # Child is launched. Close the parent's copy of those pipe # handles that only the child should have open. You need # to make sure that no handles to the write end of the # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. def _close(x): if x is not None and x != -1: if hasattr(x, 'Close'): x.Close() else: _winapi.CloseHandle(x) _close(p2cread) _close(c2pwrite) _close(errwrite) if hasattr(self, '_devnull'): os.close(self._devnull) # Retain the process handle, but close the thread handle self._child_created = True self._handle = Handle(hp) if not hasattr(hp, 'Close') else hp self.pid = pid _winapi.CloseHandle(ht) if not hasattr(ht, 'Close') else ht.Close() def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: self.returncode = GetExitCodeProcess(self._handle) self.result.set(self.returncode) return self.returncode def rawlink(self, callback): if not self.result.ready() and not self._waiting: self._waiting = True Greenlet.spawn(self._wait) self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _blocking_wait(self): WaitForSingleObject(self._handle, INFINITE) self.returncode = GetExitCodeProcess(self._handle) return self.returncode def _wait(self): self.threadpool.spawn(self._blocking_wait).rawlink(self.result) def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: if not self._waiting: self._waiting = True self._wait() result = self.result.wait(timeout=timeout) if PY3 and timeout is not None and not self.result.ready(): raise TimeoutExpired(self.args, timeout) return result def send_signal(self, sig): """Send a signal to the process """ if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: os.kill(self.pid, signal.CTRL_C_EVENT) elif sig == signal.CTRL_BREAK_EVENT: os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process """ TerminateProcess(self._handle, 1) kill = terminate else: # # POSIX methods # def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None try: DEVNULL except NameError: _devnull = object() else: _devnull = DEVNULL if stdin is None: pass elif stdin == PIPE: p2cread, p2cwrite = self.pipe_cloexec() elif stdin == _devnull: p2cread = self._get_devnull() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == PIPE: c2pread, c2pwrite = self.pipe_cloexec() elif stdout == _devnull: c2pwrite = self._get_devnull() elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == PIPE: errread, errwrite = self.pipe_cloexec() elif stderr == STDOUT: errwrite = c2pwrite elif stderr == _devnull: errwrite = self._get_devnull() elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) if cloexec: fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) else: fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _remove_nonblock_flag(self, fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK) fcntl.fcntl(fd, fcntl.F_SETFL, flags) def pipe_cloexec(self): """Create a pipe with FDs set CLOEXEC.""" # Pipes' FDs are set CLOEXEC by default because we don't want them # to be inherited by other subprocesses: the CLOEXEC flag is removed # from the child's FDs by _dup2(), between fork() and exec(). # This is not atomic: we would need the pipe2() syscall for that. r, w = os.pipe() self._set_cloexec_flag(r) self._set_cloexec_flag(w) return r, w def _close_fds(self, keep): # `keep` is a set of fds, so we # use os.closerange from 3 to min(keep) # and then from max(keep + 1) to MAXFD and # loop through filling in the gaps. # Under new python versions, we need to explicitly set # passed fds to be inheritable or they will go away on exec if hasattr(os, 'set_inheritable'): set_inheritable = os.set_inheritable else: set_inheritable = lambda i, v: True if hasattr(os, 'closerange'): keep = sorted(keep) min_keep = min(keep) max_keep = max(keep) os.closerange(3, min_keep) os.closerange(max_keep + 1, MAXFD) for i in xrange(min_keep, max_keep): if i in keep: set_inheritable(i, True) continue try: os.close(i) except: pass else: for i in xrange(3, MAXFD): if i in keep: set_inheritable(i, True) continue try: os.close(i) except: pass def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session): """Execute program (POSIX version)""" if PY3 and isinstance(args, (str, bytes)): args = [args] elif not PY3 and isinstance(args, string_types): args = [args] else: args = list(args) if shell: args = ["/bin/sh", "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] self._loop.install_sigchld() # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = self.pipe_cloexec() # errpipe_write must not be in the standard io 0, 1, or 2 fd range. low_fds_to_close = [] while errpipe_write < 3: low_fds_to_close.append(errpipe_write) errpipe_write = os.dup(errpipe_write) for low_fd in low_fds_to_close: os.close(low_fd) try: try: gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = fork_and_watch(self._on_child, self._loop, True, fork) except: if gc_was_enabled: gc.enable() raise if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # When duping fds, if there arises a situation # where one of the fds is either 0, 1 or 2, it # is possible that it is overwritten (#12607). if c2pwrite == 0: c2pwrite = os.dup(c2pwrite) if errwrite == 0 or errwrite == 1: errwrite = os.dup(errwrite) # Dup fds for child def _dup2(a, b): # dup2() removes the CLOEXEC flag but # we must do it ourselves if dup2() # would be a no-op (issue #10806). if a == b: self._set_cloexec_flag(a, False) elif a is not None: os.dup2(a, b) self._remove_nonblock_flag(b) _dup2(p2cread, 0) _dup2(c2pwrite, 1) _dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the # same fd more than once, or standard fds. closed = set([None]) for fd in [p2cread, c2pwrite, errwrite]: if fd not in closed and fd > 2: os.close(fd) closed.add(fd) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() # Close all other fds, if asked for. This must be done # after preexec_fn runs. if close_fds: fds_to_keep = set(pass_fds) fds_to_keep.add(errpipe_write) self._close_fds(fds_to_keep) elif hasattr(os, 'get_inheritable'): # close_fds was false, and we're on # Python 3.4 or newer, so "all file # descriptors except standard streams # are closed, and inheritable handles # are only inherited if the close_fds # parameter is False." for i in xrange(3, MAXFD): try: if i == errpipe_write or os.get_inheritable( i): continue os.close(i) except: pass if restore_signals: # restore the documented signals back to sig_dfl; # not all will be defined on every platform for sig in 'SIGPIPE', 'SIGXFZ', 'SIGXFSZ': sig = getattr(signal, sig, None) if sig is not None: signal.signal(sig, signal.SIG_DFL) if start_new_session: os.setsid() if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception( exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) finally: # Make sure that the process exits no matter what. # The return code does not matter much as it won't be # reported to the application os._exit(1) # Parent self._child_created = True if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) # self._devnull is not always defined. devnull_fd = getattr(self, '_devnull', None) if p2cread is not None and p2cwrite is not None and p2cread != devnull_fd: os.close(p2cread) if c2pwrite is not None and c2pread is not None and c2pwrite != devnull_fd: os.close(c2pwrite) if errwrite is not None and errread is not None and errwrite != devnull_fd: os.close(errwrite) if devnull_fd is not None: os.close(devnull_fd) # Prevent a double close of these fds from __init__ on error. self._closed_child_pipe_fds = True # Wait for exec to fail or succeed; possibly raising exception errpipe_read = FileObject(errpipe_read, 'rb') data = errpipe_read.read() finally: if hasattr(errpipe_read, 'close'): errpipe_read.close() else: os.close(errpipe_read) if data != b"": self.wait() child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception def _handle_exitstatus(self, sts): if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) elif os.WIFEXITED(sts): self.returncode = os.WEXITSTATUS(sts) else: # Should never happen raise RuntimeError("Unknown child exit status!") def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute. :keyword timeout: The floating point number of seconds to wait. Under Python 2, this is a gevent extension, and we simply return if it expires. Under Python 3, if this time elapses without finishing the process, :exc:`TimeoutExpired` is raised.""" result = self.result.wait(timeout=timeout) if PY3 and timeout is not None and not self.result.ready(): raise TimeoutExpired(self.args, timeout) return result def send_signal(self, sig): """Send a signal to the process """ os.kill(self.pid, sig) def terminate(self): """Terminate the process with SIGTERM """ self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ self.send_signal(signal.SIGKILL)
class Popen(object): def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, threadpool=None): """Create new Popen instance.""" if not isinstance(bufsize, integer_types): raise TypeError("bufsize must be an integer") hub = get_hub() if mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") if close_fds and (stdin is not None or stdout is not None or stderr is not None): raise ValueError("close_fds is not supported on Windows " "platforms if you redirect stdin/stdout/stderr") if threadpool is None: threadpool = hub.threadpool self.threadpool = threadpool self._waiting = False else: # POSIX if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") if creationflags != 0: raise ValueError("creationflags is only supported on Windows " "platforms") assert threadpool is None self._loop = hub.loop self.stdin = None self.stdout = None self.stderr = None self.pid = None self.returncode = None self.universal_newlines = universal_newlines self.result = AsyncResult() # Input and output objects. The general principle is like # this: # # Parent Child # ------ ----- # p2cwrite ---stdin---> p2cread # c2pread <--stdout--- c2pwrite # errread <--stderr--- errwrite # # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects # are None when not using PIPEs. The child objects are None # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) self._execute_child(args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) if mswindows: if p2cwrite is not None: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) if c2pread is not None: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) if errread is not None: errread = msvcrt.open_osfhandle(errread.Detach(), 0) if p2cwrite is not None: self.stdin = FileObject(p2cwrite, 'wb') if c2pread is not None: if universal_newlines: self.stdout = FileObject(c2pread, 'rU') else: self.stdout = FileObject(c2pread, 'rb') if errread is not None: if universal_newlines: self.stderr = FileObject(errread, 'rU') else: self.stderr = FileObject(errread, 'rb') def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode) def _on_child(self, watcher): watcher.stop() status = watcher.rstatus if os.WIFSIGNALED(status): self.returncode = -os.WTERMSIG(status) else: self.returncode = os.WEXITSTATUS(status) self.result.set(self.returncode) def communicate(self, input=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional input argument should be a string to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr).""" greenlets = [] if self.stdin: greenlets.append(spawn(write_and_close, self.stdin, input)) if self.stdout: stdout = spawn(self.stdout.read) greenlets.append(stdout) else: stdout = None if self.stderr: stderr = spawn(self.stderr.read) greenlets.append(stderr) else: stderr = None joinall(greenlets) if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() self.wait() return (None if stdout is None else stdout.value or '', None if stderr is None else stderr.value or '') def poll(self): return self._internal_poll() if mswindows: # # Windows methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: return (None, None, None, None, None, None) p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None if stdin is None: p2cread = GetStdHandle(STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = CreatePipe(None, 0) elif stdin == PIPE: p2cread, p2cwrite = CreatePipe(None, 0) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) if stdout is None: c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = CreatePipe(None, 0) elif stdout == PIPE: c2pread, c2pwrite = CreatePipe(None, 0) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) if stderr is None: errwrite = GetStdHandle(STD_ERROR_HANDLE) if errwrite is None: _, errwrite = CreatePipe(None, 0) elif stderr == PIPE: errread, errwrite = CreatePipe(None, 0) elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" return DuplicateHandle(GetCurrentProcess(), handle, GetCurrentProcess(), 0, 1, DUPLICATE_SAME_ACCESS) def _find_w9xpopen(self): """Find and return absolut path to w9xpopen.exe""" w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), "w9xpopen.exe") if not os.path.exists(w9xpopen): # Eeek - file-not-found - possibly an embedding # situation - see if we can locate it in sys.exec_prefix w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), "w9xpopen.exe") if not os.path.exists(w9xpopen): raise RuntimeError("Cannot locate w9xpopen.exe, which is " "needed for Popen to work with your " "shell or platform.") return w9xpopen def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (MS Windows version)""" if not isinstance(args, string_types): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() if None not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite if shell: startupinfo.dwFlags |= STARTF_USESHOWWINDOW startupinfo.wShowWindow = SW_HIDE comspec = os.environ.get("COMSPEC", "cmd.exe") args = '{} /c "{}"'.format(comspec, args) if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com": # Win9x, or using command.com on NT. We need to # use the w9xpopen intermediate program. For more # information, see KB Q150956 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) w9xpopen = self._find_w9xpopen() args = '"%s" %s' % (w9xpopen, args) # Not passing CREATE_NEW_CONSOLE has been known to # cause random failures on win9x. Specifically a # dialog: "Your program accessed mem currently in # use at xxx" and a hopeful warning about the # stability of your system. Cost is Ctrl+C wont # kill children. creationflags |= CREATE_NEW_CONSOLE # Start the process try: hp, ht, pid, tid = CreateProcess(executable, args, # no special security None, None, int(not close_fds), creationflags, env, cwd, startupinfo) except pywintypes.error as e: # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really # translate errno using _sys_errlist (or similar), but # how can this be done from Python? raise WindowsError(*e.args) finally: # Child is launched. Close the parent's copy of those pipe # handles that only the child should have open. You need # to make sure that no handles to the write end of the # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. if p2cread is not None: p2cread.Close() if c2pwrite is not None: c2pwrite.Close() if errwrite is not None: errwrite.Close() # Retain the process handle, but close the thread handle self._handle = hp self.pid = pid ht.Close() def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: self.returncode = GetExitCodeProcess(self._handle) self.result.set(self.returncode) return self.returncode def rawlink(self, callback): if not self.result.ready() and not self._waiting: self._waiting = True Greenlet.spawn(self._wait) self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _blocking_wait(self): WaitForSingleObject(self._handle, INFINITE) self.returncode = GetExitCodeProcess(self._handle) return self.returncode def _wait(self): self.threadpool.spawn(self._blocking_wait).rawlink(self.result) def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: if not self._waiting: self._waiting = True self._wait() return self.result.wait(timeout=timeout) def send_signal(self, sig): """Send a signal to the process """ if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: os.kill(self.pid, signal.CTRL_C_EVENT) elif sig == signal.CTRL_BREAK_EVENT: os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process """ TerminateProcess(self._handle, 1) kill = terminate else: # # POSIX methods # def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None if stdin is None: pass elif stdin == PIPE: p2cread, p2cwrite = self.pipe_cloexec() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == PIPE: c2pread, c2pwrite = self.pipe_cloexec() elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == PIPE: errread, errwrite = self.pipe_cloexec() elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) if cloexec: fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) else: fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _remove_nonblock_flag(self, fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK) fcntl.fcntl(fd, fcntl.F_SETFL, flags) def pipe_cloexec(self): """Create a pipe with FDs set CLOEXEC.""" # Pipes' FDs are set CLOEXEC by default because we don't want them # to be inherited by other subprocesses: the CLOEXEC flag is removed # from the child's FDs by _dup2(), between fork() and exec(). # This is not atomic: we would need the pipe2() syscall for that. r, w = os.pipe() self._set_cloexec_flag(r) self._set_cloexec_flag(w) return r, w def _close_fds(self, but): if hasattr(os, 'closerange'): os.closerange(3, but) os.closerange(but + 1, MAXFD) else: for i in xrange(3, MAXFD): if i == but: continue try: os.close(i) except: pass def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (POSIX version)""" if isinstance(args, string_types): args = [args] else: args = list(args) if shell: args = ["/bin/sh", "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] self._loop.install_sigchld() # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = self.pipe_cloexec() try: try: gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = fork() except: if gc_was_enabled: gc.enable() raise if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # When duping fds, if there arises a situation # where one of the fds is either 0, 1 or 2, it # is possible that it is overwritten (#12607). if c2pwrite == 0: c2pwrite = os.dup(c2pwrite) if errwrite == 0 or errwrite == 1: errwrite = os.dup(errwrite) # Dup fds for child def _dup2(a, b): # dup2() removes the CLOEXEC flag but # we must do it ourselves if dup2() # would be a no-op (issue #10806). if a == b: self._set_cloexec_flag(a, False) elif a is not None: os.dup2(a, b) self._remove_nonblock_flag(b) _dup2(p2cread, 0) _dup2(c2pwrite, 1) _dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the # same fd more than once, or standard fds. closed = set([None]) for fd in [p2cread, c2pwrite, errwrite]: if fd not in closed and fd > 2: os.close(fd) closed.add(fd) # Close all other fds, if asked for if close_fds: self._close_fds(but=errpipe_write) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception(exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) finally: # Make sure that the process exits no matter what. # The return code does not matter much as it won't be # reported to the application os._exit(1) # Parent self._watcher = self._loop.child(self.pid) self._watcher.start(self._on_child, self._watcher) if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) if p2cread is not None and p2cwrite is not None: os.close(p2cread) if c2pwrite is not None and c2pread is not None: os.close(c2pwrite) if errwrite is not None and errread is not None: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising exception errpipe_read = FileObject(errpipe_read, 'rb') data = errpipe_read.read() finally: if hasattr(errpipe_read, 'close'): errpipe_read.close() else: os.close(errpipe_read) if data != b"": self.wait() child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception def _handle_exitstatus(self, sts): if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) elif os.WIFEXITED(sts): self.returncode = os.WEXITSTATUS(sts) else: # Should never happen raise RuntimeError("Unknown child exit status!") def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" return self.result.wait(timeout=timeout) def send_signal(self, sig): """Send a signal to the process """ os.kill(self.pid, sig) def terminate(self): """Terminate the process with SIGTERM """ self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ self.send_signal(signal.SIGKILL)
class Popen(object): def __init__(self, args, bufsize=None, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, threadpool=None, **kwargs): """Create new Popen instance.""" if not PY3 and kwargs: raise TypeError("Got unexpected keyword arguments", kwargs) pass_fds = kwargs.pop('pass_fds', ()) start_new_session = kwargs.pop('start_new_session', False) restore_signals = kwargs.pop('restore_signals', True) hub = get_hub() if bufsize is None: # bufsize has different defaults on Py3 and Py2 if PY3: bufsize = -1 else: bufsize = 0 if not isinstance(bufsize, integer_types): raise TypeError("bufsize must be an integer") if mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") any_stdio_set = (stdin is not None or stdout is not None or stderr is not None) if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: if any_stdio_set: close_fds = False else: close_fds = True elif close_fds and any_stdio_set: raise ValueError("close_fds is not supported on Windows " "platforms if you redirect stdin/stdout/stderr") if threadpool is None: threadpool = hub.threadpool self.threadpool = threadpool self._waiting = False else: # POSIX if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: # close_fds has different defaults on Py3/Py2 if PY3: close_fds = True else: close_fds = False if pass_fds and not close_fds: import warnings warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) close_fds = True if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") if creationflags != 0: raise ValueError("creationflags is only supported on Windows " "platforms") assert threadpool is None self._loop = hub.loop if PY3: self.args = args self.stdin = None self.stdout = None self.stderr = None self.pid = None self.returncode = None self.universal_newlines = universal_newlines self.result = AsyncResult() # Input and output objects. The general principle is like # this: # # Parent Child # ------ ----- # p2cwrite ---stdin---> p2cread # c2pread <--stdout--- c2pwrite # errread <--stderr--- errwrite # # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects # are None when not using PIPEs. The child objects are None # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) # We wrap OS handles *before* launching the child, otherwise a # quickly terminating child could make our fds unwrappable # (see #8458). if mswindows: if p2cwrite is not None: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) if c2pread is not None: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) if errread is not None: errread = msvcrt.open_osfhandle(errread.Detach(), 0) if p2cwrite is not None: if PY3 and universal_newlines: # Under Python 3, if we left on the 'b' we'd get different results # depending on whether we used FileObjectPosix or FileObjectThread self.stdin = FileObject(p2cwrite, 'wb', bufsize) self.stdin._translate = True self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True, line_buffering=(bufsize == 1)) else: self.stdin = FileObject(p2cwrite, 'wb', bufsize) if c2pread is not None: if universal_newlines: if PY3: # FileObjectThread doesn't support the 'U' qualifier # with a bufsize of 0 self.stdout = FileObject(c2pread, 'rb', bufsize) # NOTE: Universal Newlines are broken on Windows/Py3, at least # in some cases. This is true in the stdlib subprocess module # as well; the following line would fix the test cases in # test__subprocess.py that depend on python_universal_newlines, # but would be inconsistent with the stdlib: #msvcrt.setmode(self.stdout.fileno(), os.O_TEXT) self.stdout.io = io.TextIOWrapper(self.stdout.io) self.stdout.io.mode = 'r' self.stdout._translate = True else: self.stdout = FileObject(c2pread, 'rU', bufsize) else: self.stdout = FileObject(c2pread, 'rb', bufsize) if errread is not None: if universal_newlines: if PY3: self.stderr = FileObject(errread, 'rb', bufsize) self.stderr.io = io.TextIOWrapper(self.stderr.io) self.stderr._translate = True else: self.stderr = FileObject(errread, 'rU', bufsize) else: self.stderr = FileObject(errread, 'rb', bufsize) self._closed_child_pipe_fds = False try: self._execute_child(args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session) except: # Cleanup if the child failed starting. # (gevent: New in python3, but reported as gevent bug in #347. # Note that under Py2, any error raised below will replace the # original error so we have to use reraise) if not PY3: exc_info = sys.exc_info() for f in filter(None, (self.stdin, self.stdout, self.stderr)): try: f.close() except (OSError, IOError): pass # Ignore EBADF or other errors. if not self._closed_child_pipe_fds: to_close = [] if stdin == PIPE: to_close.append(p2cread) if stdout == PIPE: to_close.append(c2pwrite) if stderr == PIPE: to_close.append(errwrite) if hasattr(self, '_devnull'): to_close.append(self._devnull) for fd in to_close: try: os.close(fd) except (OSError, IOError): pass if not PY3: try: reraise(*exc_info) finally: del exc_info raise def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode) def _on_child(self, watcher): watcher.stop() status = watcher.rstatus if os.WIFSIGNALED(status): self.returncode = -os.WTERMSIG(status) else: self.returncode = os.WEXITSTATUS(status) self.result.set(self.returncode) def _get_devnull(self): if not hasattr(self, '_devnull'): self._devnull = os.open(os.devnull, os.O_RDWR) return self._devnull _stdout_buffer = None _stderr_buffer = None def communicate(self, input=None, timeout=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional input argument should be a string to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr). :keyword timeout: Under Python 2, this is a gevent extension; if given and it expires, we will raise :class:`gevent.timeout.Timeout`. Under Python 3, this raises the standard :exc:`TimeoutExpired` exception. """ greenlets = [] if self.stdin: greenlets.append(spawn(write_and_close, self.stdin, input)) # If the timeout parameter is used, and the caller calls back after # getting a TimeoutExpired exception, we can wind up with multiple # greenlets trying to run and read from and close stdout/stderr. # That's bad because it can lead to 'RuntimeError: reentrant call in io.BufferedReader'. # We can't just kill the previous greenlets when a timeout happens, # though, because we risk losing the output collected by that greenlet # (and Python 3, where timeout is an official parameter, explicitly says # that no output should be lost in the event of a timeout.) Instead, we're # watching for the exception and ignoring it. It's not elegant, # but it works if self.stdout: def _read_out(): try: data = self.stdout.read() except RuntimeError: return if self._stdout_buffer is not None: self._stdout_buffer += data else: self._stdout_buffer = data stdout = spawn(_read_out) greenlets.append(stdout) else: stdout = None if self.stderr: def _read_err(): try: data = self.stderr.read() except RuntimeError: return if self._stderr_buffer is not None: self._stderr_buffer += data else: self._stderr_buffer = data stderr = spawn(_read_err) greenlets.append(stderr) else: stderr = None # If we were given stdin=stdout=stderr=None, we have no way to # communicate with the child, and thus no greenlets to wait # on. This is a nonsense case, but it comes up in the test # case for Python 3.5 (test_subprocess.py # RunFuncTestCase.test_timeout). Instead, we go directly to # self.wait if not greenlets and timeout is not None: result = self.wait(timeout=timeout) # Python 3 would have already raised, but Python 2 would not # so we need to do that manually if result is None: from gevent.timeout import Timeout raise Timeout(timeout) done = joinall(greenlets, timeout=timeout) if timeout is not None and len(done) != len(greenlets): if PY3: raise TimeoutExpired(self.args, timeout) from gevent.timeout import Timeout raise Timeout(timeout) if self.stdout: try: self.stdout.close() except RuntimeError: pass if self.stderr: try: self.stderr.close() except RuntimeError: pass self.wait() stdout_value = self._stdout_buffer self._stdout_buffer = None stderr_value = self._stderr_buffer self._stderr_buffer = None # XXX: Under python 3 in universal newlines mode we should be # returning str, not bytes return (None if stdout is None else stdout_value or b'', None if stderr is None else stderr_value or b'') def poll(self): return self._internal_poll() if PY3: def __enter__(self): return self def __exit__(self, type, value, traceback): if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() try: # Flushing a BufferedWriter may raise an error if self.stdin: self.stdin.close() finally: # Wait for the process to terminate, to avoid zombies. # JAM: gevent: If the process never terminates, this # blocks forever. self.wait() if mswindows: # # Windows methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: return (None, None, None, None, None, None) p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None try: DEVNULL except NameError: _devnull = object() else: _devnull = DEVNULL if stdin is None: p2cread = GetStdHandle(STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = CreatePipe(None, 0) if PY3: p2cread = Handle(p2cread) _winapi.CloseHandle(_) elif stdin == PIPE: p2cread, p2cwrite = CreatePipe(None, 0) if PY3: p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) elif stdin == _devnull: p2cread = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) if stdout is None: c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = CreatePipe(None, 0) if PY3: c2pwrite = Handle(c2pwrite) _winapi.CloseHandle(_) elif stdout == PIPE: c2pread, c2pwrite = CreatePipe(None, 0) if PY3: c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) elif stdout == _devnull: c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) if stderr is None: errwrite = GetStdHandle(STD_ERROR_HANDLE) if errwrite is None: _, errwrite = CreatePipe(None, 0) if PY3: errwrite = Handle(errwrite) _winapi.CloseHandle(_) elif stderr == PIPE: errread, errwrite = CreatePipe(None, 0) if PY3: errread, errwrite = Handle(errread), Handle(errwrite) elif stderr == STDOUT: errwrite = c2pwrite elif stderr == _devnull: errwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" return DuplicateHandle(GetCurrentProcess(), handle, GetCurrentProcess(), 0, 1, DUPLICATE_SAME_ACCESS) def _find_w9xpopen(self): """Find and return absolut path to w9xpopen.exe""" w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), "w9xpopen.exe") if not os.path.exists(w9xpopen): # Eeek - file-not-found - possibly an embedding # situation - see if we can locate it in sys.exec_prefix w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), "w9xpopen.exe") if not os.path.exists(w9xpopen): raise RuntimeError("Cannot locate w9xpopen.exe, which is " "needed for Popen to work with your " "shell or platform.") return w9xpopen def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session): """Execute program (MS Windows version)""" assert not pass_fds, "pass_fds not supported on Windows." if not isinstance(args, string_types): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() if None not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite if shell: startupinfo.dwFlags |= STARTF_USESHOWWINDOW startupinfo.wShowWindow = SW_HIDE comspec = os.environ.get("COMSPEC", "cmd.exe") args = '{} /c "{}"'.format(comspec, args) if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com": # Win9x, or using command.com on NT. We need to # use the w9xpopen intermediate program. For more # information, see KB Q150956 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) w9xpopen = self._find_w9xpopen() args = '"%s" %s' % (w9xpopen, args) # Not passing CREATE_NEW_CONSOLE has been known to # cause random failures on win9x. Specifically a # dialog: "Your program accessed mem currently in # use at xxx" and a hopeful warning about the # stability of your system. Cost is Ctrl+C wont # kill children. creationflags |= CREATE_NEW_CONSOLE # Start the process try: hp, ht, pid, tid = CreateProcess(executable, args, # no special security None, None, int(not close_fds), creationflags, env, cwd, startupinfo) except IOError as e: # From 2.6 on, pywintypes.error was defined as IOError # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really # translate errno using _sys_errlist (or similar), but # how can this be done from Python? if PY3: raise # don't remap here raise WindowsError(*e.args) finally: # Child is launched. Close the parent's copy of those pipe # handles that only the child should have open. You need # to make sure that no handles to the write end of the # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. def _close(x): if x is not None and x != -1: if hasattr(x, 'Close'): x.Close() else: _winapi.CloseHandle(x) _close(p2cread) _close(c2pwrite) _close(errwrite) if hasattr(self, '_devnull'): os.close(self._devnull) # Retain the process handle, but close the thread handle self._child_created = True self._handle = Handle(hp) if not hasattr(hp, 'Close') else hp self.pid = pid _winapi.CloseHandle(ht) if not hasattr(ht, 'Close') else ht.Close() def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: self.returncode = GetExitCodeProcess(self._handle) self.result.set(self.returncode) return self.returncode def rawlink(self, callback): if not self.result.ready() and not self._waiting: self._waiting = True Greenlet.spawn(self._wait) self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _blocking_wait(self): WaitForSingleObject(self._handle, INFINITE) self.returncode = GetExitCodeProcess(self._handle) return self.returncode def _wait(self): self.threadpool.spawn(self._blocking_wait).rawlink(self.result) def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: if not self._waiting: self._waiting = True self._wait() result = self.result.wait(timeout=timeout) if PY3 and timeout is not None and not self.result.ready(): raise TimeoutExpired(self.args, timeout) return result def send_signal(self, sig): """Send a signal to the process """ if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: os.kill(self.pid, signal.CTRL_C_EVENT) elif sig == signal.CTRL_BREAK_EVENT: os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process """ TerminateProcess(self._handle, 1) kill = terminate else: # # POSIX methods # def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None try: DEVNULL except NameError: _devnull = object() else: _devnull = DEVNULL if stdin is None: pass elif stdin == PIPE: p2cread, p2cwrite = self.pipe_cloexec() elif stdin == _devnull: p2cread = self._get_devnull() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == PIPE: c2pread, c2pwrite = self.pipe_cloexec() elif stdout == _devnull: c2pwrite = self._get_devnull() elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == PIPE: errread, errwrite = self.pipe_cloexec() elif stderr == STDOUT: errwrite = c2pwrite elif stderr == _devnull: errwrite = self._get_devnull() elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) if cloexec: fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) else: fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _remove_nonblock_flag(self, fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK) fcntl.fcntl(fd, fcntl.F_SETFL, flags) def pipe_cloexec(self): """Create a pipe with FDs set CLOEXEC.""" # Pipes' FDs are set CLOEXEC by default because we don't want them # to be inherited by other subprocesses: the CLOEXEC flag is removed # from the child's FDs by _dup2(), between fork() and exec(). # This is not atomic: we would need the pipe2() syscall for that. r, w = os.pipe() self._set_cloexec_flag(r) self._set_cloexec_flag(w) return r, w def _close_fds(self, keep): # `keep` is a set of fds, so we # use os.closerange from 3 to min(keep) # and then from max(keep + 1) to MAXFD and # loop through filling in the gaps. # Under new python versions, we need to explicitly set # passed fds to be inheritable or they will go away on exec if hasattr(os, 'set_inheritable'): set_inheritable = os.set_inheritable else: set_inheritable = lambda i, v: True if hasattr(os, 'closerange'): keep = sorted(keep) min_keep = min(keep) max_keep = max(keep) os.closerange(3, min_keep) os.closerange(max_keep + 1, MAXFD) for i in xrange(min_keep, max_keep): if i in keep: set_inheritable(i, True) continue try: os.close(i) except: pass else: for i in xrange(3, MAXFD): if i in keep: set_inheritable(i, True) continue try: os.close(i) except: pass def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session): """Execute program (POSIX version)""" if PY3 and isinstance(args, (str, bytes)): args = [args] elif not PY3 and isinstance(args, string_types): args = [args] else: args = list(args) if shell: args = ["/bin/sh", "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] self._loop.install_sigchld() # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = self.pipe_cloexec() # errpipe_write must not be in the standard io 0, 1, or 2 fd range. low_fds_to_close = [] while errpipe_write < 3: low_fds_to_close.append(errpipe_write) errpipe_write = os.dup(errpipe_write) for low_fd in low_fds_to_close: os.close(low_fd) try: try: gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = fork_and_watch(self._on_child, self._loop, True, fork) except: if gc_was_enabled: gc.enable() raise if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # When duping fds, if there arises a situation # where one of the fds is either 0, 1 or 2, it # is possible that it is overwritten (#12607). if c2pwrite == 0: c2pwrite = os.dup(c2pwrite) if errwrite == 0 or errwrite == 1: errwrite = os.dup(errwrite) # Dup fds for child def _dup2(a, b): # dup2() removes the CLOEXEC flag but # we must do it ourselves if dup2() # would be a no-op (issue #10806). if a == b: self._set_cloexec_flag(a, False) elif a is not None: os.dup2(a, b) self._remove_nonblock_flag(b) _dup2(p2cread, 0) _dup2(c2pwrite, 1) _dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the # same fd more than once, or standard fds. closed = set([None]) for fd in [p2cread, c2pwrite, errwrite]: if fd not in closed and fd > 2: os.close(fd) closed.add(fd) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() # Close all other fds, if asked for. This must be done # after preexec_fn runs. if close_fds: fds_to_keep = set(pass_fds) fds_to_keep.add(errpipe_write) self._close_fds(fds_to_keep) elif hasattr(os, 'get_inheritable'): # close_fds was false, and we're on # Python 3.4 or newer, so "all file # descriptors except standard streams # are closed, and inheritable handles # are only inherited if the close_fds # parameter is False." for i in xrange(3, MAXFD): try: if i == errpipe_write or os.get_inheritable(i): continue os.close(i) except: pass if restore_signals: # restore the documented signals back to sig_dfl; # not all will be defined on every platform for sig in 'SIGPIPE', 'SIGXFZ', 'SIGXFSZ': sig = getattr(signal, sig, None) if sig is not None: signal.signal(sig, signal.SIG_DFL) if start_new_session: os.setsid() if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception(exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) finally: # Make sure that the process exits no matter what. # The return code does not matter much as it won't be # reported to the application os._exit(1) # Parent self._child_created = True if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) # self._devnull is not always defined. devnull_fd = getattr(self, '_devnull', None) if p2cread is not None and p2cwrite is not None and p2cread != devnull_fd: os.close(p2cread) if c2pwrite is not None and c2pread is not None and c2pwrite != devnull_fd: os.close(c2pwrite) if errwrite is not None and errread is not None and errwrite != devnull_fd: os.close(errwrite) if devnull_fd is not None: os.close(devnull_fd) # Prevent a double close of these fds from __init__ on error. self._closed_child_pipe_fds = True # Wait for exec to fail or succeed; possibly raising exception errpipe_read = FileObject(errpipe_read, 'rb') data = errpipe_read.read() finally: if hasattr(errpipe_read, 'close'): errpipe_read.close() else: os.close(errpipe_read) if data != b"": self.wait() child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception def _handle_exitstatus(self, sts): if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) elif os.WIFEXITED(sts): self.returncode = os.WEXITSTATUS(sts) else: # Should never happen raise RuntimeError("Unknown child exit status!") def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute. :keyword timeout: The floating point number of seconds to wait. Under Python 2, this is a gevent extension, and we simply return if it expires. Under Python 3, if this time elapses without finishing the process, :exc:`TimeoutExpired` is raised.""" result = self.result.wait(timeout=timeout) if PY3 and timeout is not None and not self.result.ready(): raise TimeoutExpired(self.args, timeout) return result def send_signal(self, sig): """Send a signal to the process """ os.kill(self.pid, sig) def terminate(self): """Terminate the process with SIGTERM """ self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ self.send_signal(signal.SIGKILL)
class ChildrenMonitor(object): """Simple monitor that monitors the children of a node and their content. """ _STOP_REQUEST = object() def __init__(self, client, path, into, factory, args, listener): self.client = client self.path = path self.into = into if into is not None else {} self.factory = factory if factory is not None else str self.args = args self.listener = listener or MonitorListener() self.started = AsyncResult() self.queue = Queue() self.stats = {} def _monitor(self): """Run the monitoring loop.""" def watcher(event): self.queue.put(event) while True: try: children = self.client.get_children(self.path, watcher) except zookeeper.NoNodeException: if not self.started.ready(): self.started.set(None) gevent.sleep(1) continue except Exception, err: if not self.started.ready(): self.started.set_exception(err) break for child in children: if not child in self.stats: data, stat = self.client.get(os.path.join(self.path, child)) self.into[child] = self.factory(data, *self.args) self.listener.created(child, self.into[child]) self.stats[child] = stat else: data, stat = self.client.get(os.path.join(self.path, child)) if stat['version'] != self.stats[child]['version']: self.into[child] = self.factory(data, *self.args) self.listener.modified(child, self.into[child]) self.stats[child] = stat for child in self.into.keys(): if child not in children: del self.into[child] del self.stats[child] self.listener.deleted(child) if not self.started.ready(): self.started.set(None) self.listener.commit() event = self.queue.get() if event is self._STOP_REQUEST: break
class TransactionDummy(ATransaction): ping_timeout = 5 # sec result_timeout = 20 # sec def __init__(self, callback_url: str, local_timeout: float, ping_timeout=None, result_timeout=None): super().__init__(ObjectId()) self.callback_url = callback_url self.local_timeout = local_timeout self.ping_timeout = ping_timeout if ping_timeout is not None else TransactionDummy.ping_timeout # type: float self.result_timeout = result_timeout if result_timeout is not None else TransactionDummy.result_timeout # type: float self.key = sha256( bytes(str(self.id) + str(int(time.time() * 10**6) ^ randint(0, 2**20)), encoding="utf-8")).hexdigest() debug_SSE.event({ "event": "init", "t": datetime.now(), "data": { "callback_url": self.callback_url, "local_timeout": self.local_timeout * 1000, "result_timeout": self.result_timeout * 1000, "ping_timeout": self.ping_timeout * 1000, "key": self.key, "_id": self.id } }) # DEBUG init self._ping = Event() self.result = AsyncResult() self.ping_timeout_thread_obj = None # type: Greenlet self.result_thread_obj = None # type: Greenlet @g_async def _spawn(self): self.ping_timeout_thread_obj = self.ping_timeout_thread( ) # THREAD:1, loop # wait((self.ready_commit, self.fail), timeout=self.local_timeout) # BLOCK, timeout # wait((self.commit, self.fail)) # BLOCK @g_async def ping_timeout_thread(self): while not (self.done.ready() or self.fail.ready()): debug_SSE.event({ "event": "wait_ping", "t": datetime.now(), "data": None }) # DEBUG wait_ping w = wait((self._ping, self.done, self.fail), count=1, timeout=self.ping_timeout * 2) # BLOCK, timeout if not len(w): debug_SSE.event({ "event": "fail", "t": datetime.now(), "data": "ping timeout" }) # DEBUG ping timeout self.fail.set() # EMIT(fail) break if self._ping.ready(): debug_SSE.event({ "event": "ping", "t": datetime.now(), "data": None }) # DEBUG ping self._ping.clear() # EMIT(-ping) sleep() def do_work(self, resource): self.result_thread_obj = self.result_thread(resource) # THREAD:1 @g_async def result_thread(self, resource): sleep(self.result_timeout) # BLOCK, sleep if not (self.ready_commit.ready() or self.fail.ready()): self.result.set(resource) # EMIT(result) self.ready_commit.set() # EMIT(ready_commit) debug_SSE.event({ "event": "ready_commit", "t": datetime.now(), "data": None }) # DEBUG ready_commit data = {"key": self.key, "response": {"data": self.result.get()}} rp = requests.put(self.callback_url, headers={"Connection": "close"}, json=data, timeout=5) # else: # raise Exception("error during work") def ping(self) -> bool: if not (self.fail.ready() or self.done.ready()): self._ping.set() # EMIT(ping) return True return False @g_async def do_commit(self): if not self.fail.ready(): if self.ready_commit.ready() and self.result.ready(): self.commit.set() # EMIT(ping) debug_SSE.event({ "event": "commit", "t": datetime.now(), "data": None }) # DEBUG commit else: raise Exception("Error during commit") sleep(randint(self.ping_timeout - 2, self.ping_timeout + 2)) data = {"key": self.key, "done": True} rp = requests.put(self.callback_url, headers={"Connection": "close"}, json=data) debug_SSE.event({ "event": "done", "t": datetime.now(), "data": None }) # DEBUG done @g_async def do_rollback(self): self.fail.set() # EMIT(fail) debug_SSE.event({ "event": "rollback", "t": datetime.now(), "data": None }) # DEBUG rollback
class GClient(object): """ A generic gevent-based network client, that implements common send and receive functionality. Useful members: group: A gevent.pool.Group() tied to the lifetime of the client. When stopping, all greenlets in the group will be killed. started: True if the client has been started stopped: True if the client has been stopped running: True if the client has been started but not stopped """ def __init__(self, logger=None): self.group = Group() self.started = False self._send_queue = Queue() self._stopping = False self._stopped = AsyncResult() if not hasattr(self, 'logger'): # let subclass overrride if they want if not logger: logger = logging.getLogger('gclient').getChild(type(self).__name__) self.logger = logger def start(self): """Start the client, performing some connection step and beginning processing.""" if self.started: raise Exception("Already started") self.started = True self.logger.debug("{} starting".format(self)) self._start() self._send_loop_worker = self.group.spawn(self._send_loop) self._recv_loop_worker = self.group.spawn(self._recv_loop) self.logger.info("{} started".format(self)) def _start(self): """Override this with code that creates and initializes a connection""" def stop(self, ex=None): """Stop the client, optionally referencing some exception. This will kill all greenlets in group and do any specific stop handling. Anyone waiting on the client stopping will have the exception raised, if any. """ if self._stopping: self.wait_for_stop() return if not self.started: self.started = True self._stopping = True if ex: self.logger.info("{} stopping with error".format(self), exc_info=True) else: self.logger.info("{} stopping".format(self)) # since the greenlet calling stop() might be in self.group, we make a new greenlet to do the work @gevent.spawn def stop_worker(): self.group.kill(block=True) while not self._send_queue.empty(): msg, event = self._send_queue.get(block=False) event.set() self._stop(ex) if ex: self._stopped.set_exception(ex) else: self._stopped.set(None) self.logger.debug("{} fully stopped".format(self)) stop_worker.get() def _stop(self, ex=None): """Optionally override this with specific cleanup code for stopping the client, such as closing the connection.""" pass def wait_for_stop(self): """Block until the client has stopped, re-raising the exception it was stopped with, if any.""" self._stopped.get() @property def stopped(self): return self._stopped.ready() @property def running(self): return self.started and not self.stopped def send(self, msg, block=False): """Enqueue some kind of message to be sent. If block=True, block until actually sent. If block=False, returns a gevent.event.Event() that will be set when actually sent, or the client is stopped. Note that messages are sent in order, so using either of these shouldn't often be needed. """ if self._stopping: raise Exception("Can't send to stopped client") event = Event() self._send_queue.put((msg, event)) if block: event.wait() else: return event def _send_loop(self): try: for msg, event in self._send_queue: self._send(msg) event.set() except Exception as ex: self.stop(ex) def _send(self, msg): """Override this with specific code for sending a message. It may raise to indicate a failure that will stop the client.""" def _recv_loop(self): try: self._receive() except Exception as ex: self.stop(ex) else: self.stop() def _receive(self): """Override this with code that receives data. It may return to indicate a graceful close,
class OutTimer(Collected): """Timer for timed outputs""" storage = OutTimers.storage q = None _timer = None _timer = None def __init__(self, parent, timer, nextval): global tseq tseq += 1 self.name = parent.name + (str(tseq), ) super(OutTimer, self).__init__() self.parent = parent self.end = timer self.val = nextval self.q = AsyncResult() self._start() def info(self): return "%s:%s" % (self.name, self.val) def list(self): n = now() yield super(OutTimer, self) yield ("output", self.parent.name) yield ("start", self.started) yield ("end", self.end) yield ("next value", self.val) def _start(self): if self._timer: self._timer.cancel() self.started = now() self._timer = callLater(False, self.end, self._timeout) def _timeout(self): self._timer = None try: self.parent.write(self.val) except Exception as ex: fix_exception(ex) self.q.set(ex) else: self.q.set(None) def done(self): """called externally via _tmwrite() when the external timer writes""" if self._timer: self._timer.cancel() self._timer = None if not self.q.ready(): self.q.set(None) def cancel(self): self.delete() def delete(self, ctx=None): if self._timer is not None: self._timer.cancel() self._timer = None self.q.set(DelayCancelled(self)) super(OutTimer, self).delete(ctx)
class Operation(BaseOperation): def __init__(self, monitor, callable, *args): super(Operation, self).__init__(monitor) self.id = (callable, args) self.callable = callable self.args = args self.node = getattr(callable, "__self__", None) self.method = getattr(callable, "__name__", None) self.result = AsyncResult() self.primary_parent = self.monitor.get_current() self.primary_parent.add_dependency(self) self.greenlet = Yaylet(callable, *args) self.greenlet.operation = self self.greenlet.link(self._operation_finish) def start(self): self.greenlet.start() def ready(self): return self.result.ready() def get(self): return self.result.get() def _operation_finish(self, source): # WARNING: This method will be caused in it's own greenlet. # Using self.monitor.execute from here will cause work to be owned by Root # Cycle breaking source.operation = None # Setup the AsyncResult so *new* calls will return immediately # But let's not notify the existing blocked greenlets until we # have run the paradox detector if source.successful(): self.result.value = source.value self.result._exception = None else: self.result.value = None self.result._exception = source.exception # Purge any operations that were cached during a peek operation checks = [] for p in self.peeks: for c, op in p.walk_children(): if op.method.startswith("as_"): checks.append(op) op.purge_one() if p.method.startswith("as_"): checks.append(p) p.purge_one() getcurrent().operation = self for op in checks: try: current_val = op.get() new_val = self.monitor.wait(getattr(op.node, op.method)) except Exception as e: self.result.set_exception(e) return if new_val != current_val: self.result.set_exception( errors.ParadoxError( "Inconsistent configuration detected - changed from %r to %r" % (current_val, new_val), anchor=op.node.anchor, ) ) getcurrent().operation = None return getcurrent().operation = None # Now notify all the other greenlets waiting for us that it is safe to continue if source.successful(): self.result.set(source.value) else: self.result.set_exception(source.exception) def __repr__(self): return "%s<%s>.%s(%r)" % (self.node.__class__.__name__, id(self), self.method, self.args)
class ProxyBot(Bot): def __init__(self, name, client, game, bot): self.name = name self.client = client self.bot = bot if bot: self.TIMEOUT = 1.0 else: self.TIMEOUT = None self.expecting = None self._vote = None self._select = None self._sabotage = None self._join = None self._part = None self.game = game def __call__(self, game, index, spy): """This function pretends to be a Builder, but in fact just configures this object in place as it's easier to setup and maintain.""" Player.__init__(self, self.name, index) self.state = game self.spy = spy self.channel = '%s-player-%i' % (self.game, index) self.client.send_message(message.Join(self.channel)) self.client.send_message(message.Join(self.game)) self._join = Event() # Use elegant /INVITE command for humans that have better clients. self.client.send_message(message.Command([self.name, self.channel], 'INVITE')) return self def bakeTeam(self, team): return ', '.join([str(p) for p in team]) def makeTeam(self, msg): for s in '\t,.!;?': msg = msg.replace(s, ' ') names = [n for n in msg.split(' ') if n] players = [] for n in names: players.append(self.makePlayer(n)) return players def makePlayer(self, name): for p in self.state.players: if str(p.index) in name: return p if name in p.name: return p assert False, "Can't find player for input name '%s'." % (name) def send(self, msg): self.client.msg(self.channel, msg) def onGameRevealed(self, players, spies): roles = {True: "Spy", False: "Resistance"} s = "" if self.spy: s = "; SPIES " + self.bakeTeam(spies) self._join.wait() self._join = None self.send('REVEAL %s; ROLE %s; PLAYERS %s%s.' % (self.game, roles[self.spy], self.bakeTeam(players), s)) def onMissionAttempt(self, mission, tries, leader): self.send('MISSION %i.%i; LEADER %s.' % (mission, tries, Player.__repr__(leader))) def select(self, players, count): self.send('SELECT %i!' % (count)) self._select = AsyncResult() self.state.count = count self.expecting = self.process_SELECTED return self._select.get(timeout=self.TIMEOUT) def process_SELECTED(self, msg): if 'select' in msg[1].lower(): msg = ' '.join(msg[2:]) else: msg = ' '.join(msg[1:]) team = self.makeTeam(msg) if len(team) != self.state.count: self.send('SELECT %i?' % (self.state.count)) else: self._select.set(team) self._select = None def onTeamSelected(self, leader, team): self.state.team = team[:] self.send("VOTE %s?" % (self.bakeTeam(team))) self._vote = AsyncResult() self.expecting = self.process_VOTED def vote(self, team): v = self._vote.get(timeout=self.TIMEOUT) self._vote = None return v def process_VOTED(self, msg): result = parseYesOrNo(' '.join(msg[1:])) if result is not None: self._vote.set(result) def onVoteComplete(self, votes): self.send("VOTES %s." % (', '.join([showYesOrNo(v) for v in votes]))) v = [b for b in votes if b] if self in self.state.team and len(v) > 2: self.send("SABOTAGE?") self._sabotage = AsyncResult() self.expecting = self.process_SABOTAGED else: self._sabotage = None def sabotage(self): assert self._sabotage is not None s = self._sabotage.get(timeout=self.TIMEOUT) self._sabotage = None return s def process_SABOTAGED(self, msg): result = parseYesOrNo(' '.join(msg[1:])) if result is not None: self._sabotage.set(result) def onMissionComplete(self, sabotaged): # Force synchronization in case sabotage() is not called due to the bot # being resistance. This helps hide human identity by having the same # input delay in Spy or Resistance cases. if self._sabotage and not self._sabotage.ready(): s = self._sabotage.get(timeout=self.TIMEOUT) assert not s, "Expecting sabotage() to be False if it was handled automatically." self.send("SABOTAGES %i." % (sabotaged)) self.expecting = None def onGameComplete(self, win, spies): self.send("RESULT %s; SPIES %s." % (showYesOrNo(win), self.bakeTeam(spies))) self.client.send_message(message.Command(self.game, 'PART')) self._part = Event() self._part.wait() self.client.send_message(message.Command(self.channel, 'PART'))
class _Socket(_original_Socket): """Green version of :class:`zmq.core.socket.Socket` The following methods are overridden: * send * recv To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or recieving is deferred to the hub if a ``zmq.EAGAIN`` (retry) error is raised. The `__state_changed` method is triggered when the zmq.FD for the socket is marked as readable and triggers the necessary read and write events (which are waited for in the recv and send methods). Some double underscore prefixes are used to minimize pollution of :class:`zmq.core.socket.Socket`'s namespace. """ def __init__(self, context, socket_type): self.__in_send_multipart = False self.__in_recv_multipart = False self.__setup_events() def __del__(self): self.close() def close(self, linger=None): super(_Socket, self).close(linger) self.__cleanup_events() def __cleanup_events(self): # close the _state_event event, keeps the number of active file descriptors down if getattr(self, '_state_event', None): _stop(self._state_event) self._state_event = None # if the socket has entered a close state resume any waiting greenlets self.__writable.set() self.__readable.set() def __setup_events(self): self.__readable = AsyncResult() self.__writable = AsyncResult() self.__readable.set() self.__writable.set() try: self._state_event = get_hub().loop.io(self.getsockopt(zmq.FD), 1) # read state watcher self._state_event.start(self.__state_changed) except AttributeError: # for gevent<1.0 compatibility from gevent.core import read_event self._state_event = read_event(self.getsockopt(zmq.FD), self.__state_changed, persist=True) def __state_changed(self, event=None, _evtype=None): if self.closed: self.__cleanup_events() return try: # avoid triggering __state_changed from inside __state_changed events = super(_Socket, self).getsockopt(zmq.EVENTS) except zmq.ZMQError as exc: self.__writable.set_exception(exc) self.__readable.set_exception(exc) else: if events & zmq.POLLOUT: self.__writable.set() if events & zmq.POLLIN: self.__readable.set() def _wait_write(self): assert self.__writable.ready( ), "Only one greenlet can be waiting on this event" self.__writable = AsyncResult() # timeout is because libzmq cannot be trusted to properly signal a new send event: # this is effectively a maximum poll interval of 1s tic = time.time() timeout = gevent.Timeout(seconds=1) try: timeout.start() self.__writable.get(block=True) except gevent.Timeout as t: if t is not timeout: raise toc = time.time() # gevent bug: get can raise timeout even on clean return # don't display zmq bug warning for gevent bug (this is getting ridiculous) if toc - tic > 0.9 and self.getsockopt(zmq.EVENTS) & zmq.POLLOUT: print("BUG: gevent missed a libzmq send event on %i!" % self.FD, file=sys.stderr) finally: timeout.cancel() self.__writable.set() def _wait_read(self): assert self.__readable.ready( ), "Only one greenlet can be waiting on this event" self.__readable = AsyncResult() # timeout is because libzmq cannot always be trusted to play nice with libevent. # I can only confirm that this actually happens for send, but lets be symmetrical # with our dirty hacks. # this is effectively a maximum poll interval of 1s tic = time.time() timeout = gevent.Timeout(seconds=1) try: timeout.start() self.__readable.get(block=True) except gevent.Timeout as t: if t is not timeout: raise toc = time.time() # gevent bug: get can raise timeout even on clean return # don't display zmq bug warning for gevent bug (this is getting ridiculous) if toc - tic > 0.9 and self.getsockopt(zmq.EVENTS) & zmq.POLLIN: print("BUG: gevent missed a libzmq recv event on %i!" % self.FD, file=sys.stderr) finally: timeout.cancel() self.__readable.set() def send(self, data, flags=0, copy=True, track=False): """send, which will only block current greenlet state_changed always fires exactly once (success or fail) at the end of this method. """ # if we're given the NOBLOCK flag act as normal and let the EAGAIN get raised if flags & zmq.NOBLOCK: try: msg = super(_Socket, self).send(data, flags, copy, track) finally: if not self.__in_send_multipart: self.__state_changed() return msg # ensure the zmq.NOBLOCK flag is part of flags flags |= zmq.NOBLOCK while True: # Attempt to complete this operation indefinitely, blocking the current greenlet try: # attempt the actual call msg = super(_Socket, self).send(data, flags, copy, track) except zmq.ZMQError as e: # if the raised ZMQError is not EAGAIN, reraise if e.errno != zmq.EAGAIN: if not self.__in_send_multipart: self.__state_changed() raise else: if not self.__in_send_multipart: self.__state_changed() return msg # defer to the event loop until we're notified the socket is writable self._wait_write() def recv(self, flags=0, copy=True, track=False): """recv, which will only block current greenlet state_changed always fires exactly once (success or fail) at the end of this method. """ if flags & zmq.NOBLOCK: try: msg = super(_Socket, self).recv(flags, copy, track) finally: if not self.__in_recv_multipart: self.__state_changed() return msg flags |= zmq.NOBLOCK while True: try: msg = super(_Socket, self).recv(flags, copy, track) except zmq.ZMQError as e: if e.errno != zmq.EAGAIN: if not self.__in_recv_multipart: self.__state_changed() raise else: if not self.__in_recv_multipart: self.__state_changed() return msg self._wait_read() def send_multipart(self, *args, **kwargs): """wrap send_multipart to prevent state_changed on each partial send""" self.__in_send_multipart = True try: msg = super(_Socket, self).send_multipart(*args, **kwargs) finally: self.__in_send_multipart = False self.__state_changed() return msg def recv_multipart(self, *args, **kwargs): """wrap recv_multipart to prevent state_changed on each partial recv""" self.__in_recv_multipart = True try: msg = super(_Socket, self).recv_multipart(*args, **kwargs) finally: self.__in_recv_multipart = False self.__state_changed() return msg def getsockopt(self, opt): """trigger state_changed on getsockopt(EVENTS)""" optval = super(_Socket, self).getsockopt(opt) if opt == zmq.EVENTS: self.__state_changed() return optval
def socket_send(context, dest_id, message, expect_reply=False): if dest_id is None: logger.error("socket_send dest ID %s is not valid" % ID_TO_STRING(dest_id)) return None if context is not None and context.direction == ONLY_FROM_TCP_SERVER and context.id == dest_id: # logger.debug("socket_send reuses socket since context ID is the same as dest_id %s" % str(dest_id)) next_hop_id = dest_id address = context.address sock = context.socket nonce = context.nonce else: next_hop = find_nexthop_for_id(dest_id) # logger.debug("socket_send next_hop is %s" % str(next_hop)) if next_hop is None: logger.error( "socket_send next hop for dest ID %s 0x%X cannot be found" % (ID_TO_STRING(dest_id), dest_id)) return None next_hop_id = next_hop.id address = next_hop.tcp_address sock = ConnectionManager.init().get_peer_by_id(next_hop_id) nonce = os.urandom(MPTN_TCP_NONCE_SIZE) if sock is None: # logger.debug("socket_send no socket found for ID %s"%ID_TO_STRING(next_hop_id)) sock = reconnect(address) if sock is None: # logger.error("socket_send cannot re-setup socket for next_hop_id=%s addr=%s msg is\n%s" % (ID_TO_STRING(next_hop_id), str(address), formatted_print(split_packet_to_list(message)))) return try: sock.send(self_id_net_endian_string) except Exception as e: logger.error( "socket_send self_id_net_endian_string error=%s. addr=%s, self_id_net_endian_string=%s, nonce=%s, message is\n%s\nerror=%s\n%s" % (str(address), ID_TO_STRING(self_id_net_endian_string), str(map(ord, nonce)), str(formatted_print(split_packet_to_list(message))), str(e), traceback.format_exc())) return gevent.spawn(socket_recv, sock, address, next_hop_id) ConnectionManager.init().add_peer( address, Peer(socket=sock, id=next_hop_id, address=address)) gevent.sleep(0) # logger.debug("socket_send message %s to ID %s" % (str(message), ID_TO_STRING(next_hop_id))) size = 0 try: sock.send(nonce) size = struct.pack("!L", socket.htonl(len(message))) sock.send(size) sock.sendall(message) except Exception as e: logger.error( "socket_send nonce addr=%s, self_id_net_endian_string=%s, nonce=%s, message is\n%s\nerror=%s\n%s" % (str(address), ID_TO_STRING(self_id_net_endian_string), str(map(ord, nonce)), str(formatted_print(split_packet_to_list(message))), str(e), traceback.format_exc())) ConnectionManager.init().remove_peer(address) return None if not expect_reply: return None callback = AsyncResult() ConnectionManager.init().add_nonce(nonce, NonceCallback(dest_id, callback)) if context is None: while not callback.ready(): gevent.sleep(0) return callback.get()
class _Socket(_original_Socket): """Green version of :class:`zmq.core.socket.Socket` The following methods are overridden: * send * recv To ensure that the ``zmq.NOBLOCK`` flag is set and that sending or receiving is deferred to the hub if a ``zmq.EAGAIN`` (retry) error is raised. The `__state_changed` method is triggered when the zmq.FD for the socket is marked as readable and triggers the necessary read and write events (which are waited for in the recv and send methods). Some double underscore prefixes are used to minimize pollution of :class:`zmq.core.socket.Socket`'s namespace. """ __in_send_multipart = False __in_recv_multipart = False __writable = None __readable = None _state_event = None _gevent_bug_timeout = 11.6 # timeout for not trusting gevent _debug_gevent = False # turn on if you think gevent is missing events _poller_class = _Poller def __init__(self, context, socket_type): _original_Socket.__init__(self, context, socket_type) self.__in_send_multipart = False self.__in_recv_multipart = False self.__setup_events() def __del__(self): self.close() def close(self, linger=None): super(_Socket, self).close(linger) self.__cleanup_events() def __cleanup_events(self): # close the _state_event event, keeps the number of active file descriptors down if getattr(self, '_state_event', None): _stop(self._state_event) self._state_event = None # if the socket has entered a close state resume any waiting greenlets self.__writable.set() self.__readable.set() def __setup_events(self): self.__readable = AsyncResult() self.__writable = AsyncResult() self.__readable.set() self.__writable.set() try: self._state_event = get_hub().loop.io(self.getsockopt(zmq.FD), 1) # read state watcher self._state_event.start(self.__state_changed) except AttributeError: # for gevent<1.0 compatibility from gevent.core import read_event self._state_event = read_event(self.getsockopt(zmq.FD), self.__state_changed, persist=True) def __state_changed(self, event=None, _evtype=None): if self.closed: self.__cleanup_events() return try: # avoid triggering __state_changed from inside __state_changed events = super(_Socket, self).getsockopt(zmq.EVENTS) except zmq.ZMQError as exc: self.__writable.set_exception(exc) self.__readable.set_exception(exc) else: if events & zmq.POLLOUT: self.__writable.set() if events & zmq.POLLIN: self.__readable.set() def _wait_write(self): assert self.__writable.ready(), "Only one greenlet can be waiting on this event" self.__writable = AsyncResult() # timeout is because libzmq cannot be trusted to properly signal a new send event: # this is effectively a maximum poll interval of 1s tic = time.time() dt = self._gevent_bug_timeout if dt: timeout = gevent.Timeout(seconds=dt) else: timeout = None try: if timeout: timeout.start() self.__writable.get(block=True) except gevent.Timeout as t: if t is not timeout: raise toc = time.time() # gevent bug: get can raise timeout even on clean return # don't display zmq bug warning for gevent bug (this is getting ridiculous) if self._debug_gevent and timeout and toc-tic > dt and \ self.getsockopt(zmq.EVENTS) & zmq.POLLOUT: print("BUG: gevent may have missed a libzmq send event on %i!" % self.FD, file=sys.stderr) finally: if timeout: timeout.cancel() self.__writable.set() def _wait_read(self): assert self.__readable.ready(), "Only one greenlet can be waiting on this event" self.__readable = AsyncResult() # timeout is because libzmq cannot always be trusted to play nice with libevent. # I can only confirm that this actually happens for send, but lets be symmetrical # with our dirty hacks. # this is effectively a maximum poll interval of 1s tic = time.time() dt = self._gevent_bug_timeout if dt: timeout = gevent.Timeout(seconds=dt) else: timeout = None try: if timeout: timeout.start() self.__readable.get(block=True) except gevent.Timeout as t: if t is not timeout: raise toc = time.time() # gevent bug: get can raise timeout even on clean return # don't display zmq bug warning for gevent bug (this is getting ridiculous) if self._debug_gevent and timeout and toc-tic > dt and \ self.getsockopt(zmq.EVENTS) & zmq.POLLIN: print("BUG: gevent may have missed a libzmq recv event on %i!" % self.FD, file=sys.stderr) finally: if timeout: timeout.cancel() self.__readable.set() def send(self, data, flags=0, copy=True, track=False): """send, which will only block current greenlet state_changed always fires exactly once (success or fail) at the end of this method. """ # if we're given the NOBLOCK flag act as normal and let the EAGAIN get raised if flags & zmq.NOBLOCK: try: msg = super(_Socket, self).send(data, flags, copy, track) finally: if not self.__in_send_multipart: self.__state_changed() return msg # ensure the zmq.NOBLOCK flag is part of flags flags |= zmq.NOBLOCK while True: # Attempt to complete this operation indefinitely, blocking the current greenlet try: # attempt the actual call msg = super(_Socket, self).send(data, flags, copy, track) except zmq.ZMQError as e: # if the raised ZMQError is not EAGAIN, reraise if e.errno != zmq.EAGAIN: if not self.__in_send_multipart: self.__state_changed() raise else: if not self.__in_send_multipart: self.__state_changed() return msg # defer to the event loop until we're notified the socket is writable self._wait_write() def recv(self, flags=0, copy=True, track=False): """recv, which will only block current greenlet state_changed always fires exactly once (success or fail) at the end of this method. """ if flags & zmq.NOBLOCK: try: msg = super(_Socket, self).recv(flags, copy, track) finally: if not self.__in_recv_multipart: self.__state_changed() return msg flags |= zmq.NOBLOCK while True: try: msg = super(_Socket, self).recv(flags, copy, track) except zmq.ZMQError as e: if e.errno != zmq.EAGAIN: if not self.__in_recv_multipart: self.__state_changed() raise else: if not self.__in_recv_multipart: self.__state_changed() return msg self._wait_read() def send_multipart(self, *args, **kwargs): """wrap send_multipart to prevent state_changed on each partial send""" self.__in_send_multipart = True try: msg = super(_Socket, self).send_multipart(*args, **kwargs) finally: self.__in_send_multipart = False self.__state_changed() return msg def recv_multipart(self, *args, **kwargs): """wrap recv_multipart to prevent state_changed on each partial recv""" self.__in_recv_multipart = True try: msg = super(_Socket, self).recv_multipart(*args, **kwargs) finally: self.__in_recv_multipart = False self.__state_changed() return msg def get(self, opt): """trigger state_changed on getsockopt(EVENTS)""" optval = super(_Socket, self).get(opt) if opt == zmq.EVENTS: self.__state_changed() return optval
class InputTable(scheme.Table): _table_name = 'input' _table_collection = input_tables id = scheme.Column('api') type = scheme.Column('api') parent = scheme.Column('api', lambda self, value: value and [value._table_name, value.id] or None) timeout = scheme.Column('api', lambda self, timeout: timeout and int(timeout.eta*1000) or None) elements = scheme.Column('api') result = scheme.Column('api') close_aborts = scheme.Column('api') ignore_api = False def __init__(self, type, parent, timeout, elements, close_aborts, ignore_api=False): self.type = type self.parent = parent self.timeout = None self.elements = [isinstance(e, list) and e or [e] for e in elements] self.close_aborts = close_aborts self.ignore_api = ignore_api if parent: parent.input = self self._result = AsyncResult() self.reset_timeout(timeout) def set_result(self, value): if self._result.ready(): #raise RuntimeError('result of input already set') return with scheme.transaction: self.result = value self.reset_timeout(None) self._result.set(value) event.fire("input:result", self) def set_error(self, value): if self._result.ready(): #raise RuntimeError('result of input already set') return with scheme.transaction: self.result = str(value) self.reset_timeout(None) self._result.set_exception(value) event.fire("input:error", self) def reset_timeout(self, timeout): with scheme.transaction: if self.timeout: self.timeout.kill() if timeout: self.timeout = gevent.spawn_later(timeout, self._timed_out) self.timeout.eta = time.time() + timeout elif self.timeout: self.timeout = None def _timed_out(self): with scheme.transaction: self.timeout = None self.set_error(InputTimeout())