def __init__(self, io, _startcount=2): """ initialize core gateway, using the given inputoutput object. """ self._io = io self._channelfactory = ChannelFactory(self, _startcount) self._cleanup.register(self) if _startcount == 1: # only import 'py' on the "client" side import py self.hook = py._com.HookRelay(ExecnetAPI, py._com.comregistry) else: self.hook = ExecnetAPI()
def __init__(self, io, _startcount=2): """ initialize core gateway, using the given inputoutput object. """ self._io = io self._channelfactory = ChannelFactory(self, _startcount) self._cleanup.register(self)
def __init__(self, io, execthreads=None, _startcount=2): """ initialize core gateway, using the given inputoutput object and 'execthreads' execution threads. """ global registered_cleanup self._execpool = WorkerPool(maxthreads=execthreads) self._io = io self._outgoing = Queue.Queue() self._channelfactory = ChannelFactory(self, _startcount) if not registered_cleanup: atexit.register(cleanup_atexit) registered_cleanup = True _active_sendqueues[self._outgoing] = True self._pool = NamedThreadPool(receiver = self._thread_receiver, sender = self._thread_sender)
class Gateway(object): class _StopExecLoop(Exception): pass _ThreadOut = ThreadOut remoteaddress = "" _requestqueue = None _cleanup = GatewayCleanup() def __init__(self, io, _startcount=2): """ initialize core gateway, using the given inputoutput object. """ self._io = io self._channelfactory = ChannelFactory(self, _startcount) self._cleanup.register(self) def _initreceive(self, requestqueue=False): if requestqueue: self._requestqueue = Queue.Queue() self._receiverthread = threading.Thread(name="receiver", target=self._thread_receiver) self._receiverthread.setDaemon(1) self._receiverthread.start() def __repr__(self): """ return string representing gateway type and status. """ addr = self.remoteaddress if addr: addr = '[%s]' % (addr,) else: addr = '' try: r = (self._receiverthread.isAlive() and "receiving" or "not receiving") s = "sending" # XXX i = len(self._channelfactory.channels()) except AttributeError: r = s = "uninitialized" i = "no" return "<%s%s %s/%s (%s active channels)>" %( self.__class__.__name__, addr, r, s, i) def _trace(self, *args): if debug: try: l = "\n".join(args).split(os.linesep) id = getid(self) for x in l: print >>debug, x debug.flush() except sysex: raise except: traceback.print_exc() def _traceex(self, excinfo): try: l = traceback.format_exception(*excinfo) errortext = "".join(l) except: errortext = '%s: %s' % (excinfo[0].__name__, excinfo[1]) self._trace(errortext) def _thread_receiver(self): """ thread to read and handle Messages half-sync-half-async. """ try: from sys import exc_info while 1: try: msg = Message.readfrom(self._io) self._trace("received <- %r" % msg) msg.received(self) except sysex: break except EOFError: break except: self._traceex(exc_info()) break finally: self._stopexec() self._stopsend() self._channelfactory._finished_receiving() self._trace('leaving %r' % threading.currentThread()) from sys import exc_info def _send(self, msg): if msg is None: self._io.close_write() else: try: msg.writeto(self._io) except: excinfo = self.exc_info() self._traceex(excinfo) msg.post_sent(self, excinfo) else: msg.post_sent(self) self._trace('sent -> %r' % msg) def _local_redirect_thread_output(self, outid, errid): l = [] for name, id in ('stdout', outid), ('stderr', errid): if id: channel = self._channelfactory.new(outid) out = self._ThreadOut(sys, name) out.setwritefunc(channel.send) l.append((out, channel)) def close(): for out, channel in l: out.delwritefunc() channel.close() return close def _local_schedulexec(self, channel, sourcetask): if self._requestqueue is not None: self._requestqueue.put((channel, sourcetask)) else: # we will not execute, let's send back an error # to inform the other side channel.close("execution disallowed") def _servemain(self, joining=True): from sys import exc_info self._initreceive(requestqueue=True) try: while 1: item = self._requestqueue.get() if item is None: self._stopsend() break try: self._executetask(item) except self._StopExecLoop: break finally: self._trace("_servemain finished") if joining: self.join() def remote_init_threads(self, num=None): """ start up to 'num' threads for subsequent remote_exec() invocations to allow concurrent execution. """ if hasattr(self, '_remotechannelthread'): raise IOError("remote threads already running") from py.__.thread import pool source = py.code.Source(pool, """ execpool = WorkerPool(maxthreads=%r) gw = channel.gateway while 1: task = gw._requestqueue.get() if task is None: gw._stopsend() execpool.shutdown() execpool.join() raise gw._StopExecLoop execpool.dispatch(gw._executetask, task) """ % num) self._remotechannelthread = self.remote_exec(source) def _executetask(self, item): """ execute channel/source items. """ from sys import exc_info channel, (source, outid, errid) = item try: loc = { 'channel' : channel } self._trace("execution starts:", repr(source)[:50]) close = self._local_redirect_thread_output(outid, errid) try: co = compile(source+'\n', '', 'exec', __future__.CO_GENERATOR_ALLOWED) exec co in loc finally: close() self._trace("execution finished:", repr(source)[:50]) except (KeyboardInterrupt, SystemExit): pass except self._StopExecLoop: channel.close() raise except: excinfo = exc_info() l = traceback.format_exception(*excinfo) errortext = "".join(l) channel.close(errortext) self._trace(errortext) else: channel.close() def _newredirectchannelid(self, callback): if callback is None: return if hasattr(callback, 'write'): callback = callback.write assert callable(callback) chan = self.newchannel() chan.setcallback(callback) return chan.id # _____________________________________________________________________ # # High Level Interface # _____________________________________________________________________ # def newchannel(self): """ return new channel object. """ return self._channelfactory.new() def remote_exec(self, source, stdout=None, stderr=None): """ return channel object and connect it to a remote execution thread where the given 'source' executes and has the sister 'channel' object in its global namespace. The callback functions 'stdout' and 'stderr' get called on receival of remote stdout/stderr output strings. """ try: source = str(Source(source)) except NameError: try: import py source = str(py.code.Source(source)) except ImportError: pass channel = self.newchannel() outid = self._newredirectchannelid(stdout) errid = self._newredirectchannelid(stderr) self._send(Message.CHANNEL_OPEN( channel.id, (source, outid, errid))) return channel def _remote_redirect(self, stdout=None, stderr=None): """ return a handle representing a redirection of a remote end's stdout to a local file object. with handle.close() the redirection will be reverted. """ clist = [] for name, out in ('stdout', stdout), ('stderr', stderr): if out: outchannel = self.newchannel() outchannel.setcallback(getattr(out, 'write', out)) channel = self.remote_exec(""" import sys outchannel = channel.receive() outchannel.gateway._ThreadOut(sys, %r).setdefaultwriter(outchannel.send) """ % name) channel.send(outchannel) clist.append(channel) for c in clist: c.waitclose() class Handle: def close(_): for name, out in ('stdout', stdout), ('stderr', stderr): if out: c = self.remote_exec(""" import sys channel.gateway._ThreadOut(sys, %r).resetdefault() """ % name) c.waitclose() return Handle() def exit(self): """ Try to stop all exec and IO activity. """ self._cleanup.unregister(self) self._stopexec() self._stopsend() def _stopsend(self): self._send(None) def _stopexec(self): if self._requestqueue is not None: self._requestqueue.put(None) def join(self, joinexec=True): """ Wait for all IO (and by default all execution activity) to stop. the joinexec parameter is obsolete. """ current = threading.currentThread() if self._receiverthread.isAlive(): self._trace("joining receiver thread") self._receiverthread.join()
class Gateway(object): _ThreadOut = ThreadOut remoteaddress = "" def __init__(self, io, execthreads=None, _startcount=2): """ initialize core gateway, using the given inputoutput object and 'execthreads' execution threads. """ global registered_cleanup self._execpool = WorkerPool(maxthreads=execthreads) self._io = io self._outgoing = Queue.Queue() self._channelfactory = ChannelFactory(self, _startcount) if not registered_cleanup: atexit.register(cleanup_atexit) registered_cleanup = True _active_sendqueues[self._outgoing] = True self._pool = NamedThreadPool(receiver = self._thread_receiver, sender = self._thread_sender) def __repr__(self): """ return string representing gateway type and status. """ addr = self.remoteaddress if addr: addr = '[%s]' % (addr,) else: addr = '' try: r = (len(self._pool.getstarted('receiver')) and "receiving" or "not receiving") s = (len(self._pool.getstarted('sender')) and "sending" or "not sending") i = len(self._channelfactory.channels()) except AttributeError: r = s = "uninitialized" i = "no" return "<%s%s %s/%s (%s active channels)>" %( self.__class__.__name__, addr, r, s, i) ## def _local_trystopexec(self): ## self._execpool.shutdown() def _trace(self, *args): if debug: try: l = "\n".join(args).split(os.linesep) id = getid(self) for x in l: print >>debug, x debug.flush() except sysex: raise except: traceback.print_exc() def _traceex(self, excinfo): try: l = traceback.format_exception(*excinfo) errortext = "".join(l) except: errortext = '%s: %s' % (excinfo[0].__name__, excinfo[1]) self._trace(errortext) def _thread_receiver(self): """ thread to read and handle Messages half-sync-half-async. """ try: from sys import exc_info while 1: try: msg = Message.readfrom(self._io) self._trace("received <- %r" % msg) msg.received(self) except sysex: break except EOFError: break except: self._traceex(exc_info()) break finally: self._send(None) self._channelfactory._finished_receiving() self._trace('leaving %r' % threading.currentThread()) def _send(self, msg): self._outgoing.put(msg) def _thread_sender(self): """ thread to send Messages over the wire. """ try: from sys import exc_info while 1: msg = self._outgoing.get() try: if msg is None: self._io.close_write() break msg.writeto(self._io) except: excinfo = exc_info() self._traceex(excinfo) if msg is not None: msg.post_sent(self, excinfo) break else: self._trace('sent -> %r' % msg) msg.post_sent(self) finally: self._trace('leaving %r' % threading.currentThread()) def _local_redirect_thread_output(self, outid, errid): l = [] for name, id in ('stdout', outid), ('stderr', errid): if id: channel = self._channelfactory.new(outid) out = self._ThreadOut(sys, name) out.setwritefunc(channel.send) l.append((out, channel)) def close(): for out, channel in l: out.delwritefunc() channel.close() return close def _thread_executor(self, channel, (source, outid, errid)): """ worker thread to execute source objects from the execution queue. """ from sys import exc_info try: loc = { 'channel' : channel } self._trace("execution starts:", repr(source)[:50]) close = self._local_redirect_thread_output(outid, errid) try: co = compile(source+'\n', '', 'exec', __future__.CO_GENERATOR_ALLOWED) exec co in loc finally: close() self._trace("execution finished:", repr(source)[:50]) except (KeyboardInterrupt, SystemExit): pass except: excinfo = exc_info() l = traceback.format_exception(*excinfo) errortext = "".join(l) channel.close(errortext) self._trace(errortext) else: channel.close()
class Gateway(object): class _StopExecLoop(Exception): pass _ThreadOut = ThreadOut remoteaddress = "" _requestqueue = None _cleanup = GatewayCleanup() def __init__(self, io, _startcount=2): """ initialize core gateway, using the given inputoutput object. """ self._io = io self._channelfactory = ChannelFactory(self, _startcount) self._cleanup.register(self) if _startcount == 1: # only import 'py' on the "client" side import py self.hook = py._com.HookRelay(ExecnetAPI, py._com.comregistry) else: self.hook = ExecnetAPI() def _initreceive(self, requestqueue=False): if requestqueue: self._requestqueue = Queue.Queue() self._receiverthread = threading.Thread(name="receiver", target=self._thread_receiver) self._receiverthread.setDaemon(1) self._receiverthread.start() def __repr__(self): """ return string representing gateway type and status. """ addr = self.remoteaddress if addr: addr = '[%s]' % (addr,) else: addr = '' try: r = (self._receiverthread.isAlive() and "receiving" or "not receiving") s = "sending" # XXX i = len(self._channelfactory.channels()) except AttributeError: r = s = "uninitialized" i = "no" return "<%s%s %s/%s (%s active channels)>" %( self.__class__.__name__, addr, r, s, i) def _trace(self, *args): if debug: try: l = "\n".join(args).split(os.linesep) id = getid(self) for x in l: print >>debug, x debug.flush() except sysex: raise except: traceback.print_exc() def _traceex(self, excinfo): try: l = traceback.format_exception(*excinfo) errortext = "".join(l) except: errortext = '%s: %s' % (excinfo[0].__name__, excinfo[1]) self._trace(errortext) def _thread_receiver(self): """ thread to read and handle Messages half-sync-half-async. """ try: from sys import exc_info while 1: try: msg = Message.readfrom(self._io) self._trace("received <- %r" % msg) msg.received(self) except sysex: break except EOFError: break except: self._traceex(exc_info()) break finally: # XXX we need to signal fatal error states to # channels/callbacks, particularly ones # where the other side just died. self._stopexec() try: self._stopsend() except IOError: self._trace('IOError on _stopsend()') self._channelfactory._finished_receiving() if threading: # might be None during shutdown/finalization self._trace('leaving %r' % threading.currentThread()) from sys import exc_info def _send(self, msg): if msg is None: self._io.close_write() else: try: msg.writeto(self._io) except: excinfo = self.exc_info() self._traceex(excinfo) msg.post_sent(self, excinfo) else: msg.post_sent(self) self._trace('sent -> %r' % msg) def _local_redirect_thread_output(self, outid, errid): l = [] for name, id in ('stdout', outid), ('stderr', errid): if id: channel = self._channelfactory.new(outid) out = self._ThreadOut(sys, name) out.setwritefunc(channel.send) l.append((out, channel)) def close(): for out, channel in l: out.delwritefunc() channel.close() return close def _local_schedulexec(self, channel, sourcetask): if self._requestqueue is not None: self._requestqueue.put((channel, sourcetask)) else: # we will not execute, let's send back an error # to inform the other side channel.close("execution disallowed") def _servemain(self, joining=True): from sys import exc_info self._initreceive(requestqueue=True) try: while 1: item = self._requestqueue.get() if item is None: self._stopsend() break try: self._executetask(item) except self._StopExecLoop: break finally: self._trace("_servemain finished") if joining: self.join() def _executetask(self, item): """ execute channel/source items. """ from sys import exc_info channel, (source, outid, errid) = item try: loc = { 'channel' : channel, '__name__': '__channelexec__'} self._trace("execution starts:", repr(source)[:50]) close = self._local_redirect_thread_output(outid, errid) try: co = compile(source+'\n', '', 'exec', __future__.CO_GENERATOR_ALLOWED) exec co in loc finally: close() self._trace("execution finished:", repr(source)[:50]) except (KeyboardInterrupt, SystemExit): pass except self._StopExecLoop: channel.close() raise except: excinfo = exc_info() l = traceback.format_exception(*excinfo) errortext = "".join(l) channel.close(errortext) self._trace(errortext) else: channel.close() def _newredirectchannelid(self, callback): if callback is None: return if hasattr(callback, 'write'): callback = callback.write assert callable(callback) chan = self.newchannel() chan.setcallback(callback) return chan.id def _rinfo(self, update=False): """ return some sys/env information from remote. """ if update or not hasattr(self, '_cache_rinfo'): class RInfo: def __init__(self, **kwargs): self.__dict__.update(kwargs) def __repr__(self): info = ", ".join(["%s=%s" % item for item in self.__dict__.items()]) return "<RInfo %r>" % info self._cache_rinfo = RInfo(**self.remote_exec(""" import sys, os channel.send(dict( executable = sys.executable, version_info = sys.version_info, platform = sys.platform, cwd = os.getcwd(), pid = os.getpid(), )) """).receive()) return self._cache_rinfo # _____________________________________________________________________ # # High Level Interface # _____________________________________________________________________ # def remote_exec(self, source, stdout=None, stderr=None): """ return channel object and connect it to a remote execution thread where the given 'source' executes and has the sister 'channel' object in its global namespace. The callback functions 'stdout' and 'stderr' get called on receival of remote stdout/stderr output strings. """ try: source = str(Source(source)) except NameError: try: import py source = str(py.code.Source(source)) except ImportError: pass channel = self.newchannel() outid = self._newredirectchannelid(stdout) errid = self._newredirectchannelid(stderr) self._send(Message.CHANNEL_OPEN( channel.id, (source, outid, errid))) return channel def remote_init_threads(self, num=None): """ start up to 'num' threads for subsequent remote_exec() invocations to allow concurrent execution. """ if hasattr(self, '_remotechannelthread'): raise IOError("remote threads already running") from py.__.thread import pool source = py.code.Source(pool, """ execpool = WorkerPool(maxthreads=%r) gw = channel.gateway while 1: task = gw._requestqueue.get() if task is None: gw._stopsend() execpool.shutdown() execpool.join() raise gw._StopExecLoop execpool.dispatch(gw._executetask, task) """ % num) self._remotechannelthread = self.remote_exec(source) def newchannel(self): """ return new channel object. """ return self._channelfactory.new() def join(self, joinexec=True): """ Wait for all IO (and by default all execution activity) to stop. the joinexec parameter is obsolete. """ current = threading.currentThread() if self._receiverthread.isAlive(): self._trace("joining receiver thread") self._receiverthread.join() def exit(self): """ Try to stop all exec and IO activity. """ self._cleanup.unregister(self) self._stopexec() self._stopsend() self.hook.pyexecnet_gateway_exit(gateway=self) def _remote_redirect(self, stdout=None, stderr=None): """ return a handle representing a redirection of a remote end's stdout to a local file object. with handle.close() the redirection will be reverted. """ clist = [] for name, out in ('stdout', stdout), ('stderr', stderr): if out: outchannel = self.newchannel() outchannel.setcallback(getattr(out, 'write', out)) channel = self.remote_exec(""" import sys outchannel = channel.receive() outchannel.gateway._ThreadOut(sys, %r).setdefaultwriter(outchannel.send) """ % name) channel.send(outchannel) clist.append(channel) for c in clist: c.waitclose() class Handle: def close(_): for name, out in ('stdout', stdout), ('stderr', stderr): if out: c = self.remote_exec(""" import sys channel.gateway._ThreadOut(sys, %r).resetdefault() """ % name) c.waitclose() return Handle() def _stopsend(self): self._send(None) def _stopexec(self): if self._requestqueue is not None: self._requestqueue.put(None)