async def test_waitable_monitor():
"""Test if `WaitableMonitor.wait_for_event()` respects event ordering."""
monitor = EventMonitor()
monitor.start()
events = []
async def wait_for_events():
events.append(await monitor.wait_for_event(lambda e: e == 1))
events.append(await monitor.wait_for_event(lambda e: e == 2))
events.append(await monitor.wait_for_event(lambda e: e == 3))
await monitor.add_event(0)
await monitor.add_event(1)
await monitor.add_event(2)
await monitor.add_event(0)
task = asyncio.create_task(wait_for_events())
await asyncio.sleep(0.1)
assert events == [1, 2]
await monitor.add_event(3)
await asyncio.sleep(0.1)
assert events == [1, 2, 3]
assert task.done()
await monitor.stop()
async def test_waitable_monitor_timeout_error():
"""Test if `WaitableMonitor.wait_for_event()` raises `TimeoutError` on timeout."""
monitor = EventMonitor()
monitor.start()
with pytest.raises(asyncio.TimeoutError):
await monitor.wait_for_event(lambda e: e == 1, timeout=0.1)
await monitor.stop()
async def test_stopped_raises_on_add_event():
"""Test whether `add_event()` invoked after stopping the monitor raises error."""
monitor = EventMonitor()
monitor.start()
await monitor.stop()
with pytest.raises(RuntimeError):
await monitor.add_event(1)
async def test_add_assertion_while_checking():
"""Test if adding an assertion while iterating over existing assertions works.
See https://github.com/golemfactory/goth/issues/464
"""
monitor = EventMonitor()
async def long_running_assertion_1(stream: Events):
async for _ in stream:
await asyncio.sleep(0.05)
async def long_running_assertion_2(stream: Events):
async for _ in stream:
await asyncio.sleep(0.05)
monitor.add_assertion(long_running_assertion_1)
monitor.add_assertion(long_running_assertion_2)
monitor.start()
await monitor.add_event(1)
await asyncio.sleep(0)
# Add a new assertion while long_running_assertions are still being checked
monitor.add_assertion(assert_all_positive)
await monitor.stop()
async def test_waitable_monitor_timeout_success():
"""Test if `WaitableMonitor.wait_for_event()` return success before timeout."""
monitor = EventMonitor()
monitor.start()
async def worker_task():
await asyncio.sleep(0.1)
await monitor.add_event(1)
asyncio.create_task(worker_task())
await monitor.wait_for_event(lambda e: e == 1, timeout=1.0)
await monitor.stop()
def __init__(self, monitor: Optional[EventMonitor[APIEvent]] = None):
self._monitor = monitor or EventMonitor()
if not self._monitor.is_running():
self._monitor.start()
self._pending_requests = {}
self._num_requests = 0
self._logger = logging.getLogger(__name__)
async def test_assertions():
"""Test a dummy set of assertions against a list of int's."""
monitor: EventMonitor[int] = EventMonitor()
monitor.add_assertions([
assert_all_positive,
assert_increasing,
assert_eventually_five,
assert_fancy_property,
])
monitor.start()
for n in [1, 3, 4, 6, 3, 8, 9, 10]:
await monitor.add_event(n)
# Need this sleep to make sure the assertions consume the events
await asyncio.sleep(0.1)
failed = {a.name.rsplit(".", 1)[-1] for a in monitor.failed}
assert failed == {"assert_increasing"}
satisfied = {a.name.rsplit(".", 1)[-1] for a in monitor.satisfied}
assert satisfied == {"assert_fancy_property"}
# Certain assertions can only accept/fail after the monitor is stopped
await monitor.stop()
failed = {a.name.rsplit(".", 1)[-1] for a in monitor.failed}
assert failed == {"assert_increasing", "assert_eventually_five"}
satisfied = {a.name.rsplit(".", 1)[-1] for a in monitor.satisfied}
assert satisfied == {"assert_all_positive", "assert_fancy_property"}
async def test_not_started_raises_on_add_event():
"""Test whether `add_event()` invoked before starting the monitor raises error."""
monitor = EventMonitor()
with pytest.raises(RuntimeError):
await monitor.add_event(1)
def __init__(
self,
node_names: Mapping[str, str],
ports: Mapping[str, dict],
assertions_module: Optional[str] = None,
):
self._node_names = node_names
self._ports = ports
self._logger = logging.getLogger(__name__)
self._proxy_thread = threading.Thread(target=self._run_mitmproxy,
name="ProxyThread",
daemon=True)
self._server_ready = threading.Event()
self._mitmproxy_runner = None
self.monitor = EventMonitor("rest", self._logger)
if assertions_module:
self.monitor.load_assertions(assertions_module)
async def test_check_assertions(caplog):
"""Test the `Runner.check_assertion_errors()` method."""
runner = Runner(
base_log_dir=Path(tempfile.mkdtemp()),
compose_config=Mock(),
)
async def assertion(events):
async for _ in events:
break
async for _ in events:
raise AssertionError("Just failing")
idle_monitor = EventMonitor()
idle_monitor.start()
busy_monitor = EventMonitor()
busy_monitor.add_assertion(assertion)
busy_monitor.start()
await asyncio.sleep(0.1)
runner.check_assertion_errors(idle_monitor, busy_monitor)
await busy_monitor.add_event(1)
await asyncio.sleep(0.1)
runner.check_assertion_errors(idle_monitor, busy_monitor)
await busy_monitor.add_event(2)
await asyncio.sleep(0.1)
# Assertion failure should be logged at this point
assert any(record.levelname == "ERROR" for record in caplog.records)
# And `check_assertion_errors()` should raise an exception
with pytest.raises(TemporalAssertionError):
runner.check_assertion_errors(idle_monitor, busy_monitor)
await busy_monitor.stop()
await idle_monitor.stop()
with pytest.raises(TemporalAssertionError):
runner.check_assertion_errors(idle_monitor, busy_monitor)
async def test_assertion_results_reported(caplog):
"""Test that assertion success and failure are logged.
This used to be a problem for assertions that do not succeed or fail
immediately after consuming an event. For example if an assertion
contains `asyncio.wait_for()` then it may raise an exception some time
after it consumed any event. After the failure, the monitor will not
feed new events to the assertion. But it should report the failure
(exactly once).
"""
monitor = EventMonitor()
async def never_accept(events):
async for _ in events:
pass
async def await_impossible(events):
await asyncio.wait_for(never_accept(events), timeout=0.1)
async def await_inevitable(events):
try:
await asyncio.wait_for(never_accept(events), timeout=0.1)
except asyncio.TimeoutError:
return "I'm fine!"
monitor.add_assertion(await_impossible)
monitor.add_assertion(await_inevitable)
monitor.start()
await monitor.add_event(1)
# At this point the assertions are still alive
assert not monitor.done
await asyncio.sleep(0.3)
# The assertions should be done now
assert monitor.failed
assert monitor.satisfied
# Stopping the monitor should trigger logging assertion success and
# failure messages
await monitor.stop()
assert any(record.levelname == "ERROR" and "failed" in record.message
for record in caplog.records)
assert any(record.levelname == "INFO" and "I'm fine!" in record.message
for record in caplog.records)
class Proxy:
"""Proxy using mitmproxy to generate events out of http calls."""
monitor: EventMonitor[APIEvent]
_proxy_thread: threading.Thread
_logger: logging.Logger
_mitmproxy_runner: Optional[dump.DumpMaster]
_node_names: Mapping[str, str]
_server_ready: threading.Event
"""Mapping of IP addresses to node names"""
_ports: Mapping[str, dict]
"""Mapping of IP addresses to their port mappings"""
def __init__(
self,
node_names: Mapping[str, str],
ports: Mapping[str, dict],
assertions_module: Optional[str] = None,
):
self._node_names = node_names
self._ports = ports
self._logger = logging.getLogger(__name__)
self._proxy_thread = threading.Thread(target=self._run_mitmproxy,
name="ProxyThread",
daemon=True)
self._server_ready = threading.Event()
self._mitmproxy_runner = None
self.monitor = EventMonitor("rest", self._logger)
if assertions_module:
self.monitor.load_assertions(assertions_module)
def start(self):
"""Start the proxy thread."""
self.monitor.start()
self._proxy_thread.start()
self._server_ready.wait()
async def stop(self):
"""Stop the proxy thread and the monitor."""
if self._mitmproxy_runner:
self._mitmproxy_runner.shutdown()
self._proxy_thread.join()
self._logger.info("The mitmproxy thread has finished")
await self.monitor.stop()
def _run_mitmproxy(self):
"""Run by `self.proxy_thread`."""
# This class is nested since it needs to refer to the `monitor` attribute
# of the enclosing instance of `Proxy`.
class MITMProxyRunner(dump.DumpMaster):
def __init__(inner_self, opts: options.Options) -> None:
super().__init__(opts)
inner_self.addons.add(
RouterAddon(self._node_names, self._ports))
inner_self.addons.add(MonitorAddon(self.monitor))
def start(inner_self):
super().start()
self._mitmproxy_runner = inner_self
self._logger.info("Embedded mitmproxy started")
self._server_ready.set()
try:
loop = asyncio.new_event_loop()
# Monkey patch the loop to set its `add_signal_handler` method to no-op.
# The original method would raise error since the loop will run in
# a non-main thread and hence cannot have signal handlers installed.
loop.add_signal_handler = lambda *args_: None
asyncio.set_event_loop(loop)
self._logger.info("Starting embedded mitmproxy...")
args = f"-q --mode reverse:http://127.0.0.1 --listen-port {MITM_PROXY_PORT}"
_main.run(MITMProxyRunner, cmdline.mitmdump, args.split())
except Exception:
self._logger.exception("Exception in mitmproxy thread")
self._logger.info("Embedded mitmproxy exited")
async def test_demand_resubscription(log_dir: Path, goth_config_path: Path,
monkeypatch) -> None:
"""Test that checks that a demand is re-submitted after its previous submission expires."""
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
goth_config = load_yaml(goth_config_path, [("nodes", nodes)])
vm_package = await vm.repo(
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
env = dict(os.environ)
env.update(requestor.get_agent_env_vars())
# Setup the environment for the requestor
for key, val in env.items():
monkeypatch.setenv(key, val)
monitor = EventMonitor()
monitor.add_assertion(assert_demand_resubscribed)
monitor.start()
# The requestor
enable_default_logger()
async def worker(work_ctx, tasks):
async for task in tasks:
script = work_ctx.new_script()
script.run("/bin/sleep", "5")
yield script
task.accept_result()
async with Golem(
budget=10.0,
event_consumer=monitor.add_event_sync,
) as golem:
task: Task # mypy needs this for some reason
async for task in golem.execute_tasks(
worker,
[Task(data=n) for n in range(20)],
vm_package,
max_workers=1,
timeout=timedelta(seconds=30),
):
logger.info("Task %d computed", task.data)
await monitor.stop()
for a in monitor.failed:
raise a.result()