def _pull_data(self, pulling_done):
"""Pull data from SshShell connection."""
logging.getLogger("moler_threads").debug("ENTER {}".format(self))
heartbeat = tracked_thread.report_alive()
already_notified = False
while not pulling_done.is_set():
if next(heartbeat):
logging.getLogger("moler_threads").debug(
"ALIVE {}".format(self))
try:
data = self.receive()
if data:
self.data_received(data, datetime.datetime.now()) # (3)
except ConnectionTimeout:
continue
except RemoteEndpointNotConnected:
break
except RemoteEndpointDisconnected:
self._notify_on_disconnect()
already_notified = True
break
except Exception as err:
err_msg = "Unexpected {!r} during pulling for data in {}".format(
err, self)
if self.sshshell.logger:
self.sshshell.logger.exception(err_msg)
else:
print("ERROR: {}".format(err_msg))
break
was_open = self._shell_channel is not None
self.sshshell.close()
is_closed = self._shell_channel is None
if was_open and is_closed and (not already_notified):
self._notify_on_disconnect()
logging.getLogger("moler_threads").debug("EXIT {}".format(self))
def _feed_loop(self, connection_observer, stop_feeding, observer_lock):
start_time = connection_observer.life_status.start_time
heartbeat = tracked_thread.report_alive()
while True:
if next(heartbeat):
logging.getLogger("moler_threads").debug(
"ALIVE {}".format(connection_observer))
if stop_feeding.is_set():
# TODO: should it be renamed to 'cancelled' to be in sync with initial action?
self.logger.debug("stopped {}".format(connection_observer))
break
if connection_observer.done():
self.logger.debug("done {}".format(connection_observer))
break
current_time = time.time()
run_duration = current_time - start_time
# we need to check connection_observer.timeout at each round since timeout may change
# during lifetime of connection_observer
timeout = connection_observer.timeout
if connection_observer.life_status.in_terminating:
timeout = connection_observer.life_status.terminating_timeout
if (timeout is not None) and (run_duration >= timeout):
if connection_observer.life_status.in_terminating:
msg = "{} underlying real command failed to finish during {} seconds. It will be forcefully" \
" terminated".format(connection_observer, timeout)
self.logger.info(msg)
connection_observer.set_end_of_life()
else:
self.logger.debug(
">>> Entering {}. conn-obs '{}' runner '{}'".format(
observer_lock, connection_observer, self))
with observer_lock:
self.logger.debug(
">>> Entered {}. conn-obs '{}' runner '{}'".
format(observer_lock, connection_observer, self))
time_out_observer(connection_observer,
timeout=connection_observer.timeout,
passed_time=run_duration,
runner_logger=self.logger)
if connection_observer.life_status.terminating_timeout >= 0.0:
start_time = time.time()
connection_observer.life_status.in_terminating = True
else:
break
self.logger.debug(
">>> Exited {}. conn-obs '{}' runner '{}'".format(
observer_lock, connection_observer, self))
else:
self._call_on_inactivity(
connection_observer=connection_observer,
current_time=current_time)
if self._in_shutdown:
self.logger.debug(
"shutdown so cancelling {}".format(connection_observer))
connection_observer.cancel()
time.sleep(self._tick) # give moler_conn a chance to feed observer
def pull_data(self, pulling_done):
"""Pull data from FIFO buffer."""
logging.getLogger("moler_threads").debug("ENTER {}".format(self))
heartbeat = tracked_thread.report_alive()
while not pulling_done.is_set():
if next(heartbeat):
logging.getLogger("moler_threads").debug(
"ALIVE {}".format(self))
self.read() # internally forwards to embedded Moler connection
try:
data, delay = self.injections.get_nowait()
if delay:
time.sleep(delay)
self._inject(data)
self.injections.task_done()
except Empty:
time.sleep(0.01) # give FIFO chance to get data
logging.getLogger("moler_threads").debug("EXIT {}".format(self))
def pull_data(self, pulling_done):
"""Pull data from ThreadedTerminal connection."""
logging.getLogger("moler_threads").debug("ENTER {}".format(self))
heartbeat = tracked_thread.report_alive()
reads = []
while not pulling_done.is_set():
if next(heartbeat):
logging.getLogger("moler_threads").debug(
"ALIVE {}".format(self))
try:
reads, _, _ = select.select([self._terminal.fd], [], [],
self._select_timeout)
except ValueError as exc:
self.logger.warning("'{}: {}'".format(exc.__class__, exc))
self._notify_on_disconnect()
pulling_done.set()
if self._terminal.fd in reads:
try:
data = self._terminal.read(self._read_buffer_size)
if self.debug_hex_on_all_chars:
self.logger.debug("incoming data: '{}'.".format(
all_chars_to_hex(data)))
if self.debug_hex_on_non_printable_chars:
self.logger.debug("incoming data: '{}'.".format(
non_printable_chars_to_hex(data)))
if self._shell_operable.is_set():
self.data_received(data=data,
recv_time=datetime.datetime.now())
else:
self._verify_shell_is_operable(data)
except EOFError:
self._notify_on_disconnect()
pulling_done.set()
logging.getLogger("moler_threads").debug("EXIT {}".format(self))
def _loop_for_observer(self):
"""
Loop to pass data (put by method feed) to observer.
:return: None
"""
logging.getLogger("moler_threads").debug("ENTER {}".format(
self._observer))
heartbeat = tracked_thread.report_alive()
while not self._request_end.is_set():
if next(heartbeat):
logging.getLogger("moler_threads").debug("ALIVE")
try:
data, timestamp = self._queue.get(
True, self._timeout_for_get_from_queue)
try:
self.logger.log(level=TRACE,
msg=r'notifying {}({!r})'.format(
self._observer, repr(data)))
except ReferenceError:
self._request_end.set() # self._observer is no more valid.
try:
if self._observer_self:
self._observer(self._observer_self, data, timestamp)
else:
self._observer(data, timestamp)
except ReferenceError:
self._request_end.set() # self._observer is no more valid.
except Exception as ex:
self._handle_unexpected_error_from_observer(
exception=ex, data=data, timestamp=timestamp)
except queue.Empty:
pass # No incoming data within self._timeout_for_get_from_queue
self._observer = None
self._observer_self = None
logging.getLogger("moler_threads").debug("EXIT")