async def test_add_assertion_while_checking():
"""Test if adding an assertion while iterating over existing assertions works.
See https://github.com/golemfactory/goth/issues/464
"""
monitor = EventMonitor()
async def long_running_assertion_1(stream: Events):
async for _ in stream:
await asyncio.sleep(0.05)
async def long_running_assertion_2(stream: Events):
async for _ in stream:
await asyncio.sleep(0.05)
monitor.add_assertion(long_running_assertion_1)
monitor.add_assertion(long_running_assertion_2)
monitor.start()
await monitor.add_event(1)
await asyncio.sleep(0)
# Add a new assertion while long_running_assertions are still being checked
monitor.add_assertion(assert_all_positive)
await monitor.stop()
async def test_assertion_results_reported(caplog):
"""Test that assertion success and failure are logged.
This used to be a problem for assertions that do not succeed or fail
immediately after consuming an event. For example if an assertion
contains `asyncio.wait_for()` then it may raise an exception some time
after it consumed any event. After the failure, the monitor will not
feed new events to the assertion. But it should report the failure
(exactly once).
"""
monitor = EventMonitor()
async def never_accept(events):
async for _ in events:
pass
async def await_impossible(events):
await asyncio.wait_for(never_accept(events), timeout=0.1)
async def await_inevitable(events):
try:
await asyncio.wait_for(never_accept(events), timeout=0.1)
except asyncio.TimeoutError:
return "I'm fine!"
monitor.add_assertion(await_impossible)
monitor.add_assertion(await_inevitable)
monitor.start()
await monitor.add_event(1)
# At this point the assertions are still alive
assert not monitor.done
await asyncio.sleep(0.3)
# The assertions should be done now
assert monitor.failed
assert monitor.satisfied
# Stopping the monitor should trigger logging assertion success and
# failure messages
await monitor.stop()
assert any(record.levelname == "ERROR" and "failed" in record.message
for record in caplog.records)
assert any(record.levelname == "INFO" and "I'm fine!" in record.message
for record in caplog.records)
async def test_check_assertions(caplog):
"""Test the `Runner.check_assertion_errors()` method."""
runner = Runner(
base_log_dir=Path(tempfile.mkdtemp()),
compose_config=Mock(),
)
async def assertion(events):
async for _ in events:
break
async for _ in events:
raise AssertionError("Just failing")
idle_monitor = EventMonitor()
idle_monitor.start()
busy_monitor = EventMonitor()
busy_monitor.add_assertion(assertion)
busy_monitor.start()
await asyncio.sleep(0.1)
runner.check_assertion_errors(idle_monitor, busy_monitor)
await busy_monitor.add_event(1)
await asyncio.sleep(0.1)
runner.check_assertion_errors(idle_monitor, busy_monitor)
await busy_monitor.add_event(2)
await asyncio.sleep(0.1)
# Assertion failure should be logged at this point
assert any(record.levelname == "ERROR" for record in caplog.records)
# And `check_assertion_errors()` should raise an exception
with pytest.raises(TemporalAssertionError):
runner.check_assertion_errors(idle_monitor, busy_monitor)
await busy_monitor.stop()
await idle_monitor.stop()
with pytest.raises(TemporalAssertionError):
runner.check_assertion_errors(idle_monitor, busy_monitor)
async def test_demand_resubscription(log_dir: Path, goth_config_path: Path,
monkeypatch) -> None:
"""Test that checks that a demand is re-submitted after its previous submission expires."""
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
goth_config = load_yaml(goth_config_path, [("nodes", nodes)])
vm_package = await vm.repo(
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
env = dict(os.environ)
env.update(requestor.get_agent_env_vars())
# Setup the environment for the requestor
for key, val in env.items():
monkeypatch.setenv(key, val)
monitor = EventMonitor()
monitor.add_assertion(assert_demand_resubscribed)
monitor.start()
# The requestor
enable_default_logger()
async def worker(work_ctx, tasks):
async for task in tasks:
script = work_ctx.new_script()
script.run("/bin/sleep", "5")
yield script
task.accept_result()
async with Golem(
budget=10.0,
event_consumer=monitor.add_event_sync,
) as golem:
task: Task # mypy needs this for some reason
async for task in golem.execute_tasks(
worker,
[Task(data=n) for n in range(20)],
vm_package,
max_workers=1,
timeout=timedelta(seconds=30),
):
logger.info("Task %d computed", task.data)
await monitor.stop()
for a in monitor.failed:
raise a.result()