def _remove_write_fd(self, fd): if not isinstance(fd, (int, long)): fd, f_obj = fd.fileno(), fd else: f_obj = None fd = fd.fileno() if fd < 0: # With proper sequencing of removing vs. closing in the # reactor thread, this should not happen - however, in # case it does, we check here whether f_obj is registered # with self.__robj or self.__wobj and if so remove the associated # (no longer valid) descriptor. if f_obj: self.__purge_old_fd_obj(f_obj) _v_silent(IOError('Tried to _remove_read_fd invalid descriptor')) return if fd not in self.__wfd: return try: ev = kevent(fd, _KQ_FILTER_WRITE, _KQ_EV_DELETE) self.__kqueue.control([ev], 0) except Exception as e: raise IOError('Failed to update kqueue object: %s' % e) finally: self.__wfd.discard(fd) self.__wobj.pop(fd, None)
def __del__(self): if self._sock_close_cback and not self._sock_sent_close_cback: try: self._sock_close_cback() except Exception as e: self.log.debug('Close callback failed') _v_silent(e)
def _sock_shutdown_read(self): """Internal call to shut down the socket for reading.""" try: self.stop_reading() self.sock.shutdown(socket.SHUT_RD) except socket.error as e: _v_silent(e)
def _sock_shutdown_write(self): """Internal call to shut down the socket for writing.""" try: self.stop_writing() self.sock.shutdown(socket.SHUT_WR) except socket.error as e: _v_silent(e)
def _remove_write_fd(self, fd): if not isinstance(fd, int): fd, f_obj = fd.fileno(), fd else: f_obj = None fd = fd.fileno() if fd < 0: # With proper sequencing of removing vs. closing in the # reactor thread, this should not happen - however, in # case it does, we check here whether f_obj is registered # with self.__robj or self.__wobj and if so remove the associated # (no longer valid) descriptor. if f_obj: self.__purge_old_fd_obj(f_obj) _v_silent(IOError('Tried to _remove_read_fd invalid descriptor')) return if fd not in self.__wfd: return try: if fd in self.__rfd: self.__poll.modify(fd, _POLLIN | _POLLERR) else: try: self.__poll.unregister(fd) except IOError as e: _v_silent(e) except: raise IOError('Failed to update poll object') finally: self.__wfd.discard(fd) self.__wobj.pop(fd, None)
def run(self): """See :meth:`versile.reactor.IVCoreReactor.run`\ .""" if self.__finished or self.__thread: raise RuntimeError('Can only start reactor once') self.__thread = threading.current_thread() self.started() try: self._run() finally: # Clean up and free resources self.__finished = True if self.__ctrl_is_pipe: for pipe_fd in self.__ctrl_r, self.__ctrl_w: try: os.close(pipe_fd) except Exception as e: _v_silent(e) else: for pipe_f in self.__ctrl_r, self.__ctrl_w: try: pipe_f.close() except Exception as e: _v_silent(e) self.__readers.clear() self.__writers.clear() self.__ctrl_queue.clear() self.__scheduled_calls = [] self.__grouped_calls.clear() self._fd_done()
def close_io(self, reason): """See :meth:`versile.reactor.io.IVByteHandle.close_io`\ .""" if not (self._sock_in_closed and self._sock_out_closed): in_was_closed = self._sock_in_closed out_was_closed = self._sock_out_closed #self.log.debug('rw shutdown') self._sock_shutdown_rw() self.log.debug('close()') try: self.sock.close() except socket.error: pass self._sock_in_closed = self._sock_out_closed = True self._sock_in_closed_reason = self._sock_out_closed_reason = reason if not in_was_closed: self._input_was_closed(reason) if not out_was_closed: self._output_was_closed(reason) if (self._sock_close_cback and self._sock_in_closed and self._sock_out_closed): try: self._sock_close_cback() except Exception as e: self.log.debug('Close callback failed') _v_silent(e) finally: self._sock_sent_close_cback = True return self._sock_in_closed and self._sock_out_closed
def __init__(self, activates=False, add_types=True): super(VNativeModule, self).__init__() self.__lock = Lock() self.__activates = activates self.__handlers = dict( ) # native typ tag -> (obj_decoder, exc_decoder) # Add decoder for conversion from VNativeObject _decoder = self._vse_obj_decoder _entry = VSECodes.NATIVE_OBJECT.mod_decoder(_decoder) self.add_decoder(_entry) # Add decoder for conversion from VNativeException _decoder = self._vse_exc_decoder _entry = VSECodes.NATIVE_EXCEPTION.mod_decoder(_decoder) self.add_decoder(_entry) if add_types: # Register Python Native Type try: import versile.vse.native.python as py_native except Exception as e: _v_silent(e) else: # Register python 2.x objects tag = 'vse-python-2.x' obj_dec = py_native.VPython2._v_vse_obj_decoder obj_exc = py_native.VPython2Exception._v_vse_exc_decoder self.add_native_handler(tag, obj_dec, obj_exc) # Register python 3.x objects tag = 'vse-python-3.x' obj_dec = py_native.VPython3._v_vse_obj_decoder obj_exc = py_native.VPython3Exception._v_vse_exc_decoder self.add_native_handler(tag, obj_dec, obj_exc)
def __fire_callback(self): try: super(VPendingList, self).callback(self.__results) except VPendingException as e: # Just ignore exceptions here - this should mean an error was # already propagated, e.g. the deferred was cancelled _v_silent(e)
def close_io(self, reason): """See :meth:`versile.reactor.io.IVByteHandle.close_io`\ .""" self._sock_shutdown_rw() self.log.debug('close()') try: self.sock.close() except socket.error as e: _v_silent(e) return True
def request(): try: _cons = self.__link.entity_consume if not _cons.producer: _v_silent(Exception('Notif. w/o producer')) return _cons.producer.control.req_producer_state(_cons) except VIOMissingControl: pass
def _fd_wait(self, timeout): s_rsel, s_wsel, s_sel = self.__rsel, self.__wsel, self.__sel try: rsel, wsel, xsel = select.select(s_rsel, s_wsel, s_sel, timeout) except ValueError as e: bad_fds = set() for fd in s_sel: if isinstance(fd, int): continue if fd.fileno() < 0: bad_fds.add(fd) if bad_fds: for fd in bad_fds: return ((self._FD_ERROR, fd),) else: raise except select.error as e: # Unspecified error, iterate all descriptors to identify events = deque() for _sock in s_rsel: try: select.select([_sock], [], [_sock], 0.0) except: events.append((self._FD_READ_ERROR, _sock)) for _sock in s_wsel: try: select.select([], [_sock], [_sock], 0.0) except: events.append((self._FD_WRITE_ERROR, _sock)) if events: return events else: # PLATFORM - IronPython - select.select sometimes # triggers this situation, unclear why - ignore for now self.__rlog.debug('unhandled select.error') _v_silent(Exception('unhandled select.error')) except IOError as e: # Return if interrupted by signal handler, not catching can # interfere e.g. with interactive use in a python interpreter if e.errno == errno.EINTR: return deque() else: events = deque() for fd in xsel: if fd not in s_sel: continue events.append((self._FD_ERROR, fd)) for fd in rsel: if fd not in s_rsel: continue events.append((self._FD_READ, fd)) for fd in wsel: if fd not in s_wsel: continue events.append((self._FD_WRITE, fd)) return events
def _sock_shutdown_rw(self): """Internal call to shut down the socket for reading and writing.""" try: if not self._sock_in_closed: self.stop_reading() if not self._sock_out_closed: self.stop_writing() self.sock.shutdown(socket.SHUT_RDWR) except socket.error as e: _v_silent(e)
def __del__(self): if self._fd >= 0: try: os.close(self._fd) except OSError as e: _v_silent(e) if self._close_cback and not self._sent_close_cback: try: self._close_cback() except Exception as e: self.log.debug('Close callback failed') _v_silent(e)
def __error_cback(self, exc, msg_id, failback=None): """Internal callback handling of calls which raised exception.""" self._lock.acquire() try: self._pending -= 1 finally: self._lock.release() if failback: try: failback(exc) except Exception as e: _v_silent(e) self.__process_queue()
def __result_cback(self, value, msg_id, callback=None): """Internal callback handling of completed calls.""" self._lock.acquire() try: self._pending -= 1 finally: self._lock.release() if callback: try: callback(value) except Exception as e: _v_silent(e) self.__process_queue()
def add_writer(self, writer, internal=False): """See :meth:`versile.reactor.IVDescriptorReactor.add_writer`\ .""" if self.__finished: raise VReactorStopped('Reactor was stopped.') elif internal or self.__is_reactor_thread(): # TROUBLESHOOT - see add_reader comments regarding 'internal' try: self._add_write_fd(writer) except IOError as e: _v_silent(e) # Ignoring for now else: self.__writers.add(writer) else: self.__msg_add_writer(writer)
def _watch(self, log_entry): msg = self.__formatter.format_entry(log_entry) # ISSUE - the .encode() step has some times been observed to # block without throwing exceptions, even though input is # valid. Suspect there is a threading issue involving GIL data = msg.encode(self.__encoding) self.__file.write(data) try: self.__file.flush() except AttributeError as e: # Some sandbox environments provide a file-like object for # writing which provides write() but not flush(), if flush() # is not available we just ignore it _v_silent(e)
def _cancel(self): with self: if self._c_result: result, self._c_result = self._c_result, None try: result.cancel() except VResultException as e: _v_silent(e) if self._r_result: result, self._r_result = self._r_result, None try: result.cancel() except VResultException as e: _v_silent(e)
def __init__(self, processor=None): super(VExternal, self).__init__(processor=processor) self.__methods = dict() # external name -> method_data self.__method_names = dict() # method_func -> external name self.__methods_lock = RLock() self.__meta = dict() # meta name -> meta_data self.__meta_names = dict() # method_func -> meta name for _attr_name in dir(self): try: method = getattr(self, _attr_name) except Exception as e: _v_silent(e) continue if not isinstance(method, types.MethodType): continue if hasattr(method, 'external') and method.external: method_func, instance_method = self._v_unwind_method(method) try: doc = method.external_doc except AttributeError: doc = None try: ctx = not method.external_noctx except AttributeError: ctx = True try: show = method.external_show except AttributeError: show = False method_data = _VMethodData(method_func, instance_method, doc, ctx, show) name = method.external_name if name is None: name = unicode(method.__name__) self.__methods[name] = method_data self.__method_names[method_func] = name if hasattr(method, 'meta') and method.meta: name = method.meta_name if name is None: name = unicode(method.__name__) method_func, instance_method = self._v_unwind_method(method) meta_data = _VMetaData(method_func, instance_method) self.__meta[name] = meta_data self.__meta_names[method_func] = name
def close_output(self, reason): """See :meth:`versile.reactor.io.IVByteHandleOutput.close_output`\ .""" if not self._out_closed: self.log.debug('write shutdown') try: os.close(self._fd) except OSError as e: _v_silent(e) finally: self._fd = -1 self._out_closed = True self._out_closed_reason = reason self._output_was_closed(reason) if self.peer is not None or not self.half_close_policy.half_out: self.close_io(reason)
def __purge_old_fd_obj(self, fd_obj): for f, o in self.__robj.items(): if o is fd_obj: try: self._remove_read_fd(f) except Exception as e: _v_silent(e) break for f, o in self.__wobj.items(): if o is fd_obj: try: self._remove_write_fd(f) except Exception as e: _v_silent(e) break
def close_io(self, reason): """See :meth:`versile.reactor.io.IVByteHandle.close_io`\ .""" if self._out_closed and self.peer is None: return if self.peer: _peer, self._peer = self.peer, None if _peer: _peer.close_input(reason) if not self._out_closed: self.close_output() if self._close_cback and not self._sent_close_cback: try: self._close_cback() except Exception as e: self.log.debug('Close callback failed') _v_silent(e) finally: self._sent_close_cback = True return True
def add_reader(self, reader, internal=False): """See :meth:`versile.reactor.IVDescriptorReactor.add_reader`\ .""" if self.__finished: raise VReactorStopped('Reactor was stopped.') elif internal or self.__is_reactor_thread(): # TROUBLESHOOT - if 'internal' is set True from a # non-reactor thread, there can be all kinds of thread conflicts, # as the reactor relies on the 'internal parameter' as a promise. # In case of odd errors that indicate thread conflicts, can # perform a 'if internal and not self.__is_reactor_thread()' # debug check here try: self._add_read_fd(reader) except IOError as e: _v_silent(e) # Ignoring for now else: self.__readers.add(reader) else: self.__msg_add_reader(reader)
def run(self): processor = self.__processor while True: try: with processor: try: call, start_cback, done_cback = processor._pop_call() except VProcessorNoCall: processor.wait() continue except VProcessorStopWorker: break else: if start_cback: try: start_cback() except Exception as e: if self.__log: self.__log.warn('worker start_cback fail') self.__log.log_trace(lvl=self.__log.WARN) else: _v_silent(e) call._execute() if self.__log: try: call.result() except Exception as e: self.__log.warn('worker execute raised exception') self.__log.log_trace(lvl=self.__log.WARN) processor._call_done() if done_cback: try: done_cback() except Exception as e: if self.__log: self.__log.warn('worker done_cback failed') self.__log.log_trace(lvl=self.__log.WARN) else: _v_silent(e) finally: # Make sure to dereference any object we don't need call = start_cback = done_cback = None
def connect(self, peer): """Connects to a peer. :param peer: peer to connect to :type peer: :class:`versile.common.peer.VSocketPeer` :raises: :exc:`versile.reactor.io.VIOError` Default creates a new native socket which is set up with *peer* socket parameters and performs :func:`sock.connect` on *peer.address*. The :class:`VClientSocket` then registers itself with the the reactor for writing to detect connect success. Upon immediate failure the socket is closed. Derived classes can override. """ try: sock = peer.create_socket() sock.setblocking(False) # The discarded socket is discarded, so we close before replacing if self.sock: try: self.sock.close() except socket.error as e: _v_silent(e) self._sock_replace(sock) if not self._can_connect(peer): self.close_io(VFIOLost()) raise VIOError('Not allowed to connect to host') self._sock_peer = peer self._set_sock_enabled() self.sock.connect(peer.address) except IOError as e: if e.errno in _errno_connect: self.start_writing() else: raise except Exception as e: self.log.debug('Connect exception') self.close_io(VFIOLost()) else: self._set_sock_connected()
def __shutdown_input(self): with self._status_cond: self.__closing_input = True # Abort consumer interface input producer = self.entity_consume.producer if producer: self.reactor.schedule(0.0, producer.abort) # Pass exception to all local calls waiting for a result self._ref_calls_lock.acquire() try: calls = self._ref_calls.values() self._ref_calls.clear() finally: self._ref_calls_lock.release() for w_call in calls: call = w_call() if call: try: call.push_exception(VCallError()) except VResultException as e: _v_silent(e)
def active_do_read(self): """Detects connecting client sockets as 'read' events. Accepts an incoming connection and calls :meth:`accepted` with the native client socket of the new client connection. """ if not self._can_accept: self.stop_reading() try: sock, address = self.sock.accept() except Exception as e: self.log.debug('Socket accept() exception, dropping connection') else: if self._controlled: self._can_accept = False self.stop_reading() self.accepted(sock, address) if self._accept_cback: try: self._accept_cback() except Exception as e: self.log.debug('Accept callback failed') _v_silent(e)
def __purge_old_fd_obj(self, fd_obj): for f, o in self.__robj.items(): if o is fd_obj: old_rd_fd = f break else: old_rd_fd = None for f, o in self.__wobj.items(): if o is fd_obj: old_wr_fd = f break else: old_wr_fd = None # Return if no matches if old_rd_fd is None and old_wr_fd is None: return # Unregister file descriptor from poll object if old_rd_fd is not None: self.__rfd.discard(old_rd_fd) self.__robj.pop(old_rd_fd, None) try: self.__poll.unregister(old_rd_fd) except IOError as e: _v_silent(e) if old_wr_fd is not None: self.__wfd.discard(old_wr_fd) self.__wobj.pop(old_wr_fd, None) if old_wr_fd != old_rd_fd: try: self.__poll.unregister(old_wr_fd) except IOError as e: _v_silent(e) # Check if file descriptor used for another object in the other queue # and if so re-enable try: if old_rd_fd in self.__wfd: self.__poll.register(old_rd_fd, _POLLOUT | _POLLERR) elif old_wr_fd in self.__rfd: self.__poll.register(old_wr_fd, _POLLIN | _POLLERR) except: _v_silent(IOError('Problem unregistering old descriptor'))
The module exports the reactor it believes to be the best performing on the system as the class :class:`VReactor`. """ from __future__ import print_function, unicode_literals from versile.internal import _vexport, _v_silent _reactors = [] _r_cls = None try: from versile.reactor.selectr import VSelectReactor except ImportError as e: _v_silent(e) else: _reactors.append('VSelectReactor') _r_cls = VSelectReactor try: from versile.reactor.pollr import VPollReactor except ImportError as e: _v_silent(e) else: _reactors.append('VPollReactor') _r_cls = VPollReactor try: from versile.reactor.kqueuer import VKqueueReactor except ImportError as e: