def _remove_write_fd(self, fd):
if not isinstance(fd, (int, long)):
fd, f_obj = fd.fileno(), fd
else:
f_obj = None
fd = fd.fileno()
if fd < 0:
# With proper sequencing of removing vs. closing in the
# reactor thread, this should not happen - however, in
# case it does, we check here whether f_obj is registered
# with self.__robj or self.__wobj and if so remove the associated
# (no longer valid) descriptor.
if f_obj:
self.__purge_old_fd_obj(f_obj)
_v_silent(IOError('Tried to _remove_read_fd invalid descriptor'))
return
if fd not in self.__wfd:
return
try:
ev = kevent(fd, _KQ_FILTER_WRITE, _KQ_EV_DELETE)
self.__kqueue.control([ev], 0)
except Exception as e:
raise IOError('Failed to update kqueue object: %s' % e)
finally:
self.__wfd.discard(fd)
self.__wobj.pop(fd, None)
def __del__(self):
if self._sock_close_cback and not self._sock_sent_close_cback:
try:
self._sock_close_cback()
except Exception as e:
self.log.debug('Close callback failed')
_v_silent(e)
def _sock_shutdown_read(self):
"""Internal call to shut down the socket for reading."""
try:
self.stop_reading()
self.sock.shutdown(socket.SHUT_RD)
except socket.error as e:
_v_silent(e)
def _sock_shutdown_write(self):
"""Internal call to shut down the socket for writing."""
try:
self.stop_writing()
self.sock.shutdown(socket.SHUT_WR)
except socket.error as e:
_v_silent(e)
def _remove_write_fd(self, fd):
if not isinstance(fd, int):
fd, f_obj = fd.fileno(), fd
else:
f_obj = None
fd = fd.fileno()
if fd < 0:
# With proper sequencing of removing vs. closing in the
# reactor thread, this should not happen - however, in
# case it does, we check here whether f_obj is registered
# with self.__robj or self.__wobj and if so remove the associated
# (no longer valid) descriptor.
if f_obj:
self.__purge_old_fd_obj(f_obj)
_v_silent(IOError('Tried to _remove_read_fd invalid descriptor'))
return
if fd not in self.__wfd:
return
try:
if fd in self.__rfd:
self.__poll.modify(fd, _POLLIN | _POLLERR)
else:
try:
self.__poll.unregister(fd)
except IOError as e:
_v_silent(e)
except:
raise IOError('Failed to update poll object')
finally:
self.__wfd.discard(fd)
self.__wobj.pop(fd, None)
def run(self):
"""See :meth:`versile.reactor.IVCoreReactor.run`\ ."""
if self.__finished or self.__thread:
raise RuntimeError('Can only start reactor once')
self.__thread = threading.current_thread()
self.started()
try:
self._run()
finally:
# Clean up and free resources
self.__finished = True
if self.__ctrl_is_pipe:
for pipe_fd in self.__ctrl_r, self.__ctrl_w:
try:
os.close(pipe_fd)
except Exception as e:
_v_silent(e)
else:
for pipe_f in self.__ctrl_r, self.__ctrl_w:
try:
pipe_f.close()
except Exception as e:
_v_silent(e)
self.__readers.clear()
self.__writers.clear()
self.__ctrl_queue.clear()
self.__scheduled_calls = []
self.__grouped_calls.clear()
self._fd_done()
def close_io(self, reason):
"""See :meth:`versile.reactor.io.IVByteHandle.close_io`\ ."""
if not (self._sock_in_closed and self._sock_out_closed):
in_was_closed = self._sock_in_closed
out_was_closed = self._sock_out_closed
#self.log.debug('rw shutdown')
self._sock_shutdown_rw()
self.log.debug('close()')
try:
self.sock.close()
except socket.error:
pass
self._sock_in_closed = self._sock_out_closed = True
self._sock_in_closed_reason = self._sock_out_closed_reason = reason
if not in_was_closed:
self._input_was_closed(reason)
if not out_was_closed:
self._output_was_closed(reason)
if (self._sock_close_cback and self._sock_in_closed
and self._sock_out_closed):
try:
self._sock_close_cback()
except Exception as e:
self.log.debug('Close callback failed')
_v_silent(e)
finally:
self._sock_sent_close_cback = True
return self._sock_in_closed and self._sock_out_closed
def __init__(self, activates=False, add_types=True):
super(VNativeModule, self).__init__()
self.__lock = Lock()
self.__activates = activates
self.__handlers = dict(
) # native typ tag -> (obj_decoder, exc_decoder)
# Add decoder for conversion from VNativeObject
_decoder = self._vse_obj_decoder
_entry = VSECodes.NATIVE_OBJECT.mod_decoder(_decoder)
self.add_decoder(_entry)
# Add decoder for conversion from VNativeException
_decoder = self._vse_exc_decoder
_entry = VSECodes.NATIVE_EXCEPTION.mod_decoder(_decoder)
self.add_decoder(_entry)
if add_types:
# Register Python Native Type
try:
import versile.vse.native.python as py_native
except Exception as e:
_v_silent(e)
else:
# Register python 2.x objects
tag = 'vse-python-2.x'
obj_dec = py_native.VPython2._v_vse_obj_decoder
obj_exc = py_native.VPython2Exception._v_vse_exc_decoder
self.add_native_handler(tag, obj_dec, obj_exc)
# Register python 3.x objects
tag = 'vse-python-3.x'
obj_dec = py_native.VPython3._v_vse_obj_decoder
obj_exc = py_native.VPython3Exception._v_vse_exc_decoder
self.add_native_handler(tag, obj_dec, obj_exc)
def __fire_callback(self):
try:
super(VPendingList, self).callback(self.__results)
except VPendingException as e:
# Just ignore exceptions here - this should mean an error was
# already propagated, e.g. the deferred was cancelled
_v_silent(e)
def close_io(self, reason):
"""See :meth:`versile.reactor.io.IVByteHandle.close_io`\ ."""
self._sock_shutdown_rw()
self.log.debug('close()')
try:
self.sock.close()
except socket.error as e:
_v_silent(e)
return True
def request():
try:
_cons = self.__link.entity_consume
if not _cons.producer:
_v_silent(Exception('Notif. w/o producer'))
return
_cons.producer.control.req_producer_state(_cons)
except VIOMissingControl:
pass
def _fd_wait(self, timeout):
s_rsel, s_wsel, s_sel = self.__rsel, self.__wsel, self.__sel
try:
rsel, wsel, xsel = select.select(s_rsel, s_wsel, s_sel, timeout)
except ValueError as e:
bad_fds = set()
for fd in s_sel:
if isinstance(fd, int):
continue
if fd.fileno() < 0:
bad_fds.add(fd)
if bad_fds:
for fd in bad_fds:
return ((self._FD_ERROR, fd),)
else:
raise
except select.error as e:
# Unspecified error, iterate all descriptors to identify
events = deque()
for _sock in s_rsel:
try:
select.select([_sock], [], [_sock], 0.0)
except:
events.append((self._FD_READ_ERROR, _sock))
for _sock in s_wsel:
try:
select.select([], [_sock], [_sock], 0.0)
except:
events.append((self._FD_WRITE_ERROR, _sock))
if events:
return events
else:
# PLATFORM - IronPython - select.select sometimes
# triggers this situation, unclear why - ignore for now
self.__rlog.debug('unhandled select.error')
_v_silent(Exception('unhandled select.error'))
except IOError as e:
# Return if interrupted by signal handler, not catching can
# interfere e.g. with interactive use in a python interpreter
if e.errno == errno.EINTR:
return deque()
else:
events = deque()
for fd in xsel:
if fd not in s_sel:
continue
events.append((self._FD_ERROR, fd))
for fd in rsel:
if fd not in s_rsel:
continue
events.append((self._FD_READ, fd))
for fd in wsel:
if fd not in s_wsel:
continue
events.append((self._FD_WRITE, fd))
return events
def _sock_shutdown_rw(self):
"""Internal call to shut down the socket for reading and writing."""
try:
if not self._sock_in_closed:
self.stop_reading()
if not self._sock_out_closed:
self.stop_writing()
self.sock.shutdown(socket.SHUT_RDWR)
except socket.error as e:
_v_silent(e)
def __del__(self):
if self._fd >= 0:
try:
os.close(self._fd)
except OSError as e:
_v_silent(e)
if self._close_cback and not self._sent_close_cback:
try:
self._close_cback()
except Exception as e:
self.log.debug('Close callback failed')
_v_silent(e)
def __error_cback(self, exc, msg_id, failback=None):
"""Internal callback handling of calls which raised exception."""
self._lock.acquire()
try:
self._pending -= 1
finally:
self._lock.release()
if failback:
try:
failback(exc)
except Exception as e:
_v_silent(e)
self.__process_queue()
def __result_cback(self, value, msg_id, callback=None):
"""Internal callback handling of completed calls."""
self._lock.acquire()
try:
self._pending -= 1
finally:
self._lock.release()
if callback:
try:
callback(value)
except Exception as e:
_v_silent(e)
self.__process_queue()
def add_writer(self, writer, internal=False):
"""See :meth:`versile.reactor.IVDescriptorReactor.add_writer`\ ."""
if self.__finished:
raise VReactorStopped('Reactor was stopped.')
elif internal or self.__is_reactor_thread():
# TROUBLESHOOT - see add_reader comments regarding 'internal'
try:
self._add_write_fd(writer)
except IOError as e:
_v_silent(e) # Ignoring for now
else:
self.__writers.add(writer)
else:
self.__msg_add_writer(writer)
def _watch(self, log_entry):
msg = self.__formatter.format_entry(log_entry)
# ISSUE - the .encode() step has some times been observed to
# block without throwing exceptions, even though input is
# valid. Suspect there is a threading issue involving GIL
data = msg.encode(self.__encoding)
self.__file.write(data)
try:
self.__file.flush()
except AttributeError as e:
# Some sandbox environments provide a file-like object for
# writing which provides write() but not flush(), if flush()
# is not available we just ignore it
_v_silent(e)
def _cancel(self):
with self:
if self._c_result:
result, self._c_result = self._c_result, None
try:
result.cancel()
except VResultException as e:
_v_silent(e)
if self._r_result:
result, self._r_result = self._r_result, None
try:
result.cancel()
except VResultException as e:
_v_silent(e)
def __init__(self, processor=None):
super(VExternal, self).__init__(processor=processor)
self.__methods = dict() # external name -> method_data
self.__method_names = dict() # method_func -> external name
self.__methods_lock = RLock()
self.__meta = dict() # meta name -> meta_data
self.__meta_names = dict() # method_func -> meta name
for _attr_name in dir(self):
try:
method = getattr(self, _attr_name)
except Exception as e:
_v_silent(e)
continue
if not isinstance(method, types.MethodType):
continue
if hasattr(method, 'external') and method.external:
method_func, instance_method = self._v_unwind_method(method)
try:
doc = method.external_doc
except AttributeError:
doc = None
try:
ctx = not method.external_noctx
except AttributeError:
ctx = True
try:
show = method.external_show
except AttributeError:
show = False
method_data = _VMethodData(method_func, instance_method, doc,
ctx, show)
name = method.external_name
if name is None:
name = unicode(method.__name__)
self.__methods[name] = method_data
self.__method_names[method_func] = name
if hasattr(method, 'meta') and method.meta:
name = method.meta_name
if name is None:
name = unicode(method.__name__)
method_func, instance_method = self._v_unwind_method(method)
meta_data = _VMetaData(method_func, instance_method)
self.__meta[name] = meta_data
self.__meta_names[method_func] = name
def close_output(self, reason):
"""See :meth:`versile.reactor.io.IVByteHandleOutput.close_output`\ ."""
if not self._out_closed:
self.log.debug('write shutdown')
try:
os.close(self._fd)
except OSError as e:
_v_silent(e)
finally:
self._fd = -1
self._out_closed = True
self._out_closed_reason = reason
self._output_was_closed(reason)
if self.peer is not None or not self.half_close_policy.half_out:
self.close_io(reason)
def __purge_old_fd_obj(self, fd_obj):
for f, o in self.__robj.items():
if o is fd_obj:
try:
self._remove_read_fd(f)
except Exception as e:
_v_silent(e)
break
for f, o in self.__wobj.items():
if o is fd_obj:
try:
self._remove_write_fd(f)
except Exception as e:
_v_silent(e)
break
def close_io(self, reason):
"""See :meth:`versile.reactor.io.IVByteHandle.close_io`\ ."""
if self._out_closed and self.peer is None:
return
if self.peer:
_peer, self._peer = self.peer, None
if _peer:
_peer.close_input(reason)
if not self._out_closed:
self.close_output()
if self._close_cback and not self._sent_close_cback:
try:
self._close_cback()
except Exception as e:
self.log.debug('Close callback failed')
_v_silent(e)
finally:
self._sent_close_cback = True
return True
def add_reader(self, reader, internal=False):
"""See :meth:`versile.reactor.IVDescriptorReactor.add_reader`\ ."""
if self.__finished:
raise VReactorStopped('Reactor was stopped.')
elif internal or self.__is_reactor_thread():
# TROUBLESHOOT - if 'internal' is set True from a
# non-reactor thread, there can be all kinds of thread conflicts,
# as the reactor relies on the 'internal parameter' as a promise.
# In case of odd errors that indicate thread conflicts, can
# perform a 'if internal and not self.__is_reactor_thread()'
# debug check here
try:
self._add_read_fd(reader)
except IOError as e:
_v_silent(e) # Ignoring for now
else:
self.__readers.add(reader)
else:
self.__msg_add_reader(reader)
def run(self):
processor = self.__processor
while True:
try:
with processor:
try:
call, start_cback, done_cback = processor._pop_call()
except VProcessorNoCall:
processor.wait()
continue
except VProcessorStopWorker:
break
else:
if start_cback:
try:
start_cback()
except Exception as e:
if self.__log:
self.__log.warn('worker start_cback fail')
self.__log.log_trace(lvl=self.__log.WARN)
else:
_v_silent(e)
call._execute()
if self.__log:
try:
call.result()
except Exception as e:
self.__log.warn('worker execute raised exception')
self.__log.log_trace(lvl=self.__log.WARN)
processor._call_done()
if done_cback:
try:
done_cback()
except Exception as e:
if self.__log:
self.__log.warn('worker done_cback failed')
self.__log.log_trace(lvl=self.__log.WARN)
else:
_v_silent(e)
finally:
# Make sure to dereference any object we don't need
call = start_cback = done_cback = None
def connect(self, peer):
"""Connects to a peer.
:param peer: peer to connect to
:type peer: :class:`versile.common.peer.VSocketPeer`
:raises: :exc:`versile.reactor.io.VIOError`
Default creates a new native socket which is set up with
*peer* socket parameters and performs :func:`sock.connect` on
*peer.address*. The :class:`VClientSocket` then registers
itself with the the reactor for writing to detect connect
success. Upon immediate failure the socket is closed. Derived
classes can override.
"""
try:
sock = peer.create_socket()
sock.setblocking(False)
# The discarded socket is discarded, so we close before replacing
if self.sock:
try:
self.sock.close()
except socket.error as e:
_v_silent(e)
self._sock_replace(sock)
if not self._can_connect(peer):
self.close_io(VFIOLost())
raise VIOError('Not allowed to connect to host')
self._sock_peer = peer
self._set_sock_enabled()
self.sock.connect(peer.address)
except IOError as e:
if e.errno in _errno_connect:
self.start_writing()
else:
raise
except Exception as e:
self.log.debug('Connect exception')
self.close_io(VFIOLost())
else:
self._set_sock_connected()
def __shutdown_input(self):
with self._status_cond:
self.__closing_input = True
# Abort consumer interface input
producer = self.entity_consume.producer
if producer:
self.reactor.schedule(0.0, producer.abort)
# Pass exception to all local calls waiting for a result
self._ref_calls_lock.acquire()
try:
calls = self._ref_calls.values()
self._ref_calls.clear()
finally:
self._ref_calls_lock.release()
for w_call in calls:
call = w_call()
if call:
try:
call.push_exception(VCallError())
except VResultException as e:
_v_silent(e)
def active_do_read(self):
"""Detects connecting client sockets as 'read' events.
Accepts an incoming connection and calls :meth:`accepted` with
the native client socket of the new client connection.
"""
if not self._can_accept:
self.stop_reading()
try:
sock, address = self.sock.accept()
except Exception as e:
self.log.debug('Socket accept() exception, dropping connection')
else:
if self._controlled:
self._can_accept = False
self.stop_reading()
self.accepted(sock, address)
if self._accept_cback:
try:
self._accept_cback()
except Exception as e:
self.log.debug('Accept callback failed')
_v_silent(e)
def __purge_old_fd_obj(self, fd_obj):
for f, o in self.__robj.items():
if o is fd_obj:
old_rd_fd = f
break
else:
old_rd_fd = None
for f, o in self.__wobj.items():
if o is fd_obj:
old_wr_fd = f
break
else:
old_wr_fd = None
# Return if no matches
if old_rd_fd is None and old_wr_fd is None:
return
# Unregister file descriptor from poll object
if old_rd_fd is not None:
self.__rfd.discard(old_rd_fd)
self.__robj.pop(old_rd_fd, None)
try:
self.__poll.unregister(old_rd_fd)
except IOError as e:
_v_silent(e)
if old_wr_fd is not None:
self.__wfd.discard(old_wr_fd)
self.__wobj.pop(old_wr_fd, None)
if old_wr_fd != old_rd_fd:
try:
self.__poll.unregister(old_wr_fd)
except IOError as e:
_v_silent(e)
# Check if file descriptor used for another object in the other queue
# and if so re-enable
try:
if old_rd_fd in self.__wfd:
self.__poll.register(old_rd_fd, _POLLOUT | _POLLERR)
elif old_wr_fd in self.__rfd:
self.__poll.register(old_wr_fd, _POLLIN | _POLLERR)
except:
_v_silent(IOError('Problem unregistering old descriptor'))
The module exports the reactor it believes to be the best performing
on the system as the class :class:`VReactor`.
"""
from __future__ import print_function, unicode_literals
from versile.internal import _vexport, _v_silent
_reactors = []
_r_cls = None
try:
from versile.reactor.selectr import VSelectReactor
except ImportError as e:
_v_silent(e)
else:
_reactors.append('VSelectReactor')
_r_cls = VSelectReactor
try:
from versile.reactor.pollr import VPollReactor
except ImportError as e:
_v_silent(e)
else:
_reactors.append('VPollReactor')
_r_cls = VPollReactor
try:
from versile.reactor.kqueuer import VKqueueReactor
except ImportError as e:
