def test_connection_observer_exception_do_not_remove():
ConnectionObserver.get_unraised_exceptions(True)
time.sleep(0.1)
from moler.cmd.unix.ls import Ls
from moler.exceptions import CommandTimeout
from moler.exceptions import WrongUsage
cmd = Ls(None)
none_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 0 == len(none_exceptions)
cmd.set_exception(CommandTimeout(cmd, 0.1))
cmd._is_done = True
active_exceptions = ConnectionObserver.get_unraised_exceptions(False)
assert 1 == len(active_exceptions)
cmd = Ls(None)
ctoe = CommandTimeout(cmd, 0.1)
cwue = WrongUsage(cmd, "Another exception")
cmd.set_exception(ctoe)
cmd._is_done = True
cmd.set_exception(cwue)
active_exceptions = ConnectionObserver.get_unraised_exceptions(False)
assert ctoe == active_exceptions[1]
assert 2 == len(active_exceptions)
active_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 2 == len(active_exceptions)
none_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 0 == len(none_exceptions)
def _add_command_to_connection(self, cmd, wait_for_slot=True):
"""
Adds command to execute on connection.
:param cmd: Command object to add to connection
:param wait_for_slot: If True then waits till command timeout or there is free slot to execute command. If False
then returns immediately regardless there is free slot or not.
:return: True if command was marked as current executed, False if command cannot be set as current executed.
"""
if self._add_command_to_execute(cmd=cmd):
self._submit(cmd)
return True
else:
if wait_for_slot:
self._add_command_to_queue(cmd=cmd)
start_time = cmd.start_time
if self._wait_for_slot_for_command(cmd=cmd):
self._submit(connection_observer=cmd)
return True
# If we are here it means command timeout before it really starts.
cmd.set_exception(CommandTimeout(cmd,
timeout=cmd.timeout,
kind="scheduler.await_done",
passed_time=time.time() - start_time))
cmd.set_end_of_life()
self._remove_command(cmd=cmd)
return False
def wait_for(self,
connection_observer,
connection_observer_future,
timeout=None):
"""
Await for connection_observer running in background or timeout.
:param connection_observer: The one we are awaiting for.
:param connection_observer_future: Future of connection-observer returned from submit().
:param timeout: Max time (in float seconds) you want to await before you give up. If None then taken from connection_observer
:return:
"""
self.logger.debug("go foreground: {!r} - await max. {} [sec]".format(
connection_observer, timeout))
start_time = connection_observer.start_time
remain_time = connection_observer.timeout
check_timeout_from_observer = True
wait_tick = 0.1
if timeout:
remain_time = timeout
check_timeout_from_observer = False
wait_tick = remain_time
while remain_time > 0.0:
done, not_done = wait([connection_observer_future],
timeout=wait_tick)
if connection_observer_future in done:
connection_observer_future._stop()
result = connection_observer_future.result()
self.logger.debug("{} returned {}".format(
connection_observer, result))
return None
if check_timeout_from_observer:
timeout = connection_observer.timeout
remain_time = timeout - (time.time() - start_time)
# code below is for timed out observer
passed = time.time() - start_time
self.logger.debug("timed out {}".format(connection_observer))
connection_observer_future.cancel()
# TODO: rethink - on timeout we raise while on other exceptions we expect observers
# just to call observer.set_exception() - so, no raise before calling observer.result()
if connection_observer.is_command():
exception = CommandTimeout(connection_observer,
timeout,
kind="await_done",
passed_time=passed)
else:
exception = ConnectionObserverTimeout(connection_observer,
timeout,
kind="await_done",
passed_time=passed)
connection_observer.set_exception(exception)
connection_observer.on_timeout()
connection_observer._log(
logging.INFO,
"'{}.{}' has timed out after '{:.2f}' seconds.".format(
connection_observer.__class__.__module__,
connection_observer.__class__.__name__,
time.time() - start_time))
return None
def _timeout_observer(self, connection_observer, timeout, passed_time, runner_logger, kind="background_run"):
"""
Set connection_observer status to timed-out
:param connection_observer: ConnectionObserver instance (command or event)
:param timeout: timeout
:param passed_time: passed time
:param runner_logger: runner logger
:param kind: Kind of running
:return: None
"""
if not connection_observer.life_status.was_on_timeout_called:
connection_observer.life_status.was_on_timeout_called = True
if not connection_observer.done():
if connection_observer.is_command():
exception = CommandTimeout(connection_observer=connection_observer,
timeout=timeout, kind=kind, passed_time=passed_time)
else:
exception = ConnectionObserverTimeout(connection_observer=connection_observer,
timeout=timeout, kind=kind, passed_time=passed_time)
connection_observer.set_exception(exception)
connection_observer.on_timeout()
observer_info = "{}.{}".format(connection_observer.__class__.__module__, connection_observer)
timeout_msg = "has timed out after {:.2f} seconds.".format(passed_time)
msg = "{} {}".format(observer_info, timeout_msg)
# levels_to_go_up: extract caller info to log where .time_out_observer has been called from
connection_observer._log(logging.INFO, msg, levels_to_go_up=2)
log_into_logger(runner_logger, level=logging.DEBUG,
msg="{} {}".format(connection_observer, timeout_msg),
levels_to_go_up=1)
def test_connection_observer_exception_do_not_remove():
ConnectionObserver.get_unraised_exceptions(True)
time.sleep(0.1)
from moler.cmd.unix.ls import Ls
from moler.exceptions import CommandTimeout
cmd = Ls(None)
none_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 0 == len(none_exceptions)
cmd.set_exception(CommandTimeout(cmd, 0.1))
active_exceptions = ConnectionObserver.get_unraised_exceptions(False)
assert 1 == len(active_exceptions)
cmd = Ls(None)
cmd.set_exception(CommandTimeout(cmd, 0.1))
active_exceptions = ConnectionObserver.get_unraised_exceptions(False)
assert 2 == len(active_exceptions)
active_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 2 == len(active_exceptions)
none_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 0 == len(none_exceptions)
def wait_for(self,
connection_observer,
connection_observer_future,
timeout=None):
"""
Await for connection_observer running in background or timeout.
:param connection_observer: The one we are awaiting for.
:param connection_observer_future: Future of connection-observer returned from submit().
:param timeout: Max time (in float seconds) you want to await before you give up. If None then taken from connection_observer
:return:
"""
self.logger.debug("go foreground: {!r} - await max. {} [sec]".format(
connection_observer, timeout))
start_time = time.time()
remain_time = connection_observer.timeout
check_timeout_from_observer = True
wait_tick = 0.1
if timeout:
remain_time = timeout
check_timeout_from_observer = False
wait_tick = remain_time
while remain_time > 0.0:
done, not_done = wait([connection_observer_future],
timeout=wait_tick)
if connection_observer_future in done:
self.shutdown()
result = connection_observer_future.result()
self.logger.debug("{} returned {}".format(
connection_observer, result))
return result
if check_timeout_from_observer:
timeout = connection_observer.timeout
remain_time = timeout - (time.time() - start_time)
moler_conn = connection_observer.connection
moler_conn.unsubscribe(connection_observer.data_received)
passed = time.time() - start_time
self.logger.debug("timeouted {}".format(connection_observer))
connection_observer.cancel()
connection_observer_future.cancel()
self.shutdown()
connection_observer.on_timeout()
if hasattr(connection_observer, "command_string"):
raise CommandTimeout(connection_observer,
timeout,
kind="await_done",
passed_time=passed)
else:
raise ConnectionObserverTimeout(connection_observer,
timeout,
kind="await_done",
passed_time=passed)
def test_connection_observer_one_exception():
ConnectionObserver.get_unraised_exceptions(True)
time.sleep(0.1)
from moler.cmd.unix.ls import Ls
from moler.exceptions import CommandTimeout
cmd = Ls(None)
none_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 0 == len(none_exceptions)
cmd.set_exception(CommandTimeout(cmd, 0.1))
active_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 1 == len(active_exceptions)
try:
cmd.result()
except CommandTimeout:
pass
none_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 0 == len(none_exceptions)
none_exceptions = ConnectionObserver.get_unraised_exceptions(True)
assert 0 == len(none_exceptions)
def time_out_observer(connection_observer,
timeout,
passed_time,
runner_logger,
kind="background_run"):
"""Set connection_observer status to timed-out"""
if not connection_observer.life_status.was_on_timeout_called:
connection_observer.life_status.was_on_timeout_called = True
if not connection_observer.done():
if hasattr(connection_observer, "command_string"):
exception = CommandTimeout(
connection_observer=connection_observer,
timeout=timeout,
kind=kind,
passed_time=passed_time)
else:
exception = ConnectionObserverTimeout(
connection_observer=connection_observer,
timeout=timeout,
kind=kind,
passed_time=passed_time)
# TODO: secure_data_received() may change status of connection_observer
# TODO: and if secure_data_received() runs inside threaded connection - we have race
connection_observer.set_exception(exception)
connection_observer.on_timeout()
observer_info = "{}.{}".format(
connection_observer.__class__.__module__, connection_observer)
timeout_msg = "has timed out after {:.2f} seconds.".format(
passed_time)
msg = "{} {}".format(observer_info, timeout_msg)
# levels_to_go_up: extract caller info to log where .time_out_observer has been called from
connection_observer._log(logging.INFO, msg, levels_to_go_up=2)
log_into_logger(runner_logger,
level=logging.DEBUG,
msg="{} {}".format(connection_observer,
timeout_msg),
levels_to_go_up=1)
