def test_time_out_observer_sets_exception_inside_observer_before_calling_on_timeout(conn_observer,
observer_runner):
from moler.runner import time_out_observer
from moler.exceptions import ConnectionObserverTimeout
def on_timeout_handler(self):
with pytest.raises(ConnectionObserverTimeout):
self.result()
with mock.patch.object(conn_observer.__class__, "on_timeout", on_timeout_handler):
with observer_runner:
conn_observer.start_time = time.time()
observer_runner.submit(conn_observer)
time_out_observer(conn_observer, timeout=2.3, passed_time=2.32, runner_logger=mock.MagicMock())
def test_runner_doesnt_impact_unrised_observer_exception_while_taking_observer_result(connection_observer,
observer_runner):
from moler.runner import time_out_observer, result_for_runners
from moler.exceptions import ConnectionObserverTimeout
with observer_runner:
connection_observer.start_time = time.time() # must start observer lifetime before runner.submit()
observer_runner.submit(connection_observer)
time_out_observer(connection_observer, timeout=2.3, passed_time=2.32, runner_logger=mock.MagicMock())
timeout = connection_observer._exception
assert timeout in ConnectionObserver._not_raised_exceptions
try:
result_for_runners(connection_observer)
except ConnectionObserverTimeout as timeout:
assert timeout in ConnectionObserver._not_raised_exceptions
def _wait_for_time_out(self, connection_observer,
connection_observer_future, timeout):
passed = time.time() - connection_observer.start_time
future = connection_observer_future or connection_observer._future
if future:
with future.observer_lock:
time_out_observer(connection_observer=connection_observer,
timeout=timeout,
passed_time=passed,
runner_logger=self.logger,
kind="await_done")
else:
# sorry, we don't have lock yet (it is created by runner.submit()
time_out_observer(connection_observer=connection_observer,
timeout=timeout,
passed_time=passed,
runner_logger=self.logger,
kind="await_done")
def test_time_out_observer_can_set_proper_exception_inside_observer(conn_observer,
observer_runner):
from moler.runner import time_out_observer
from moler.exceptions import CommandTimeout
from moler.exceptions import ConnectionObserverTimeout
if conn_observer.is_command():
expected_timeout_class = CommandTimeout
else:
expected_timeout_class = ConnectionObserverTimeout
with observer_runner:
conn_observer.start_time = time.time()
observer_runner.submit(conn_observer)
time_out_observer(conn_observer, timeout=2.3, passed_time=2.32, runner_logger=mock.MagicMock())
assert conn_observer.done()
with pytest.raises(expected_timeout_class):
conn_observer.result()
def wait_for(self,
connection_observer,
connection_observer_future,
timeout=None):
"""
Await for connection_observer running in background or timeout.
:param connection_observer: The one we are awaiting for.
:param connection_observer_future: Future of connection-observer returned from submit().
:param timeout: Max time (in float seconds) to await before give up. None - use connection_observer.timeout
:return:
"""
self.logger.debug("go foreground: {!r} - await max. {} [sec]".format(
connection_observer, timeout))
if connection_observer.done(
): # may happen when failed to start observer feeder
return None
start_time = time.time()
async def wait_for_connection_observer_done():
result_of_future = await connection_observer_future # feed() always returns None
return result_of_future
thread4async = get_asyncio_loop_thread()
try:
event_loop, its_new = thread_secure_get_event_loop()
if event_loop.is_running():
# wait_for() should not be called from 'async def'
self._raise_wrong_usage_of_wait_for(connection_observer)
# If we have have timeout=None then concurrent.futures will wait infinitely
# and feed() inside asyncio-loop will work on connection_observer.timeout
#
# If timeout is given then it defines max timeout (from "now") that concurrent.futures
# may use to shorten lifetime of feed().
# In such case we have concurrent.futures and asyncio race here - race about timeouts.
thread4async.run_async_coroutine(
wait_for_connection_observer_done(), timeout=timeout)
# If feed() inside asyncio-loop handles timeout as first - we exit here.
return None
except MolerTimeout:
# If run_async_coroutine() times out - we follow from here.
pass
except concurrent.futures.CancelledError:
connection_observer.cancel()
return None
except Exception as err:
err_msg = "{} raised {!r}".format(connection_observer, err)
self.logger.debug(err_msg)
if connection_observer._exception != err:
connection_observer.set_exception(err)
return None # will be reraised during call to connection_observer.result()
finally:
# protect against leaking coroutines
if not connection_observer_future.done():
async def conn_observer_fut_cancel():
connection_observer_future.cancel()
thread4async.start_async_coroutine(conn_observer_fut_cancel())
# handle timeout
passed = time.time() - start_time
fired_timeout = timeout if timeout else connection_observer.timeout
time_out_observer(connection_observer=connection_observer,
timeout=fired_timeout,
passed_time=passed,
runner_logger=self.logger,
kind="await_done")
return None
async def feed(self, connection_observer, subscribed_data_receiver,
observer_lock):
"""
Feeds connection_observer by transferring data from connection and passing it to connection_observer.
Should be called from background-processing of connection observer.
"""
remain_time, msg = his_remaining_time(
"remaining",
timeout=connection_observer.timeout,
from_start_time=connection_observer.start_time)
self.logger.debug("{} started, {}".format(connection_observer, msg))
connection_observer._log(
logging.INFO,
"{} started, {}".format(connection_observer.get_long_desc(), msg))
if not subscribed_data_receiver:
subscribed_data_receiver = self._start_feeding(
connection_observer, observer_lock)
await asyncio.sleep(0.005
) # give control back before we start processing
start_time = connection_observer.start_time
moler_conn = connection_observer.connection
try:
while True:
if connection_observer.done():
self.logger.debug("done {}".format(connection_observer))
break
run_duration = time.time() - start_time
# we need to check connection_observer.timeout at each round since timeout may change
# during lifetime of connection_observer
if (connection_observer.timeout is not None) and (
run_duration >= connection_observer.timeout):
with observer_lock:
time_out_observer(connection_observer,
timeout=connection_observer.timeout,
passed_time=run_duration,
runner_logger=self.logger)
break
if self._in_shutdown:
self.logger.debug("shutdown so cancelling {}".format(
connection_observer))
connection_observer.cancel()
await asyncio.sleep(
0.005) # give moler_conn a chance to feed observer
#
# main purpose of feed() is to progress observer-life by time
# firing timeout should do: observer.set_exception(Timeout)
#
# second responsibility: subscribe, unsubscribe observer from connection (build/break data path)
# third responsibility: react on external stop request via observer.cancel() or runner.shutdown()
# There is no need to put observer's result/exception into future:
# Future's goal is to feed observer (by data or time) - exiting future here means observer is already fed.
#
# Moreover, putting observer's exception here, in future, causes problem at asyncio shutdown:
# we get logs like: "Task exception was never retrieved" with bunch of stacktraces.
# That is correct behaviour of asyncio to not exit silently when future/task has gone wrong.
# However, feed() task worked fine since it correctly handled observer's exception.
# Another words - it is not feed's exception but observer's exception so, it should not be raised here.
#
except asyncio.CancelledError:
self.logger.debug("cancelling {}.feed".format(self))
# cancelling connection_observer is done inside handle_cancelled_feeder()
raise # need to reraise to inform "I agree for cancellation"
finally:
self.logger.debug("unsubscribing {}".format(connection_observer))
moler_conn.unsubscribe(subscribed_data_receiver)
# feed_done.set()
remain_time, msg = his_remaining_time(
"remaining",
timeout=connection_observer.timeout,
from_start_time=connection_observer.start_time)
connection_observer._log(
logging.INFO,
"{} finished, {}".format(connection_observer.get_short_desc(),
msg))
self.logger.debug("{} finished, {}".format(connection_observer,
msg))
return None