async def test_e2e_wasi(
common_assets: Path,
config_overrides: List[Override],
log_dir: Path,
):
"""Test successful flow requesting WASM tasks with goth REST API client."""
goth_config = load_yaml(common_assets / "goth-config.yml", config_overrides)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
web_root_path=common_assets / "web-root",
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
providers = runner.get_probes(probe_type=ProviderProbe)
# Market
demand = DemandBuilder(requestor).props_from_template(wasi_task_package).build()
agreement_providers = await negotiate_agreements(
requestor,
demand,
providers,
lambda p: p.properties.get("golem.runtime.name") == "wasmtime",
)
# Activity
exe_script = wasi_exe_script(runner)
num_commands = len(exe_script)
for agreement_id, provider in agreement_providers:
logger.info("Running activity on %s", provider.name)
activity_id = await requestor.create_activity(agreement_id)
await provider.wait_for_exeunit_started()
batch_id = await requestor.call_exec(activity_id, json.dumps(exe_script))
await requestor.collect_results(
activity_id, batch_id, num_commands, timeout=30
)
await requestor.destroy_activity(activity_id)
await provider.wait_for_exeunit_finished()
# Payment
for agreement_id, provider in agreement_providers:
await provider.wait_for_invoice_sent()
invoices = await requestor.gather_invoices(agreement_id)
assert all(inv.agreement_id == agreement_id for inv in invoices)
# TODO:
await requestor.pay_invoices(invoices)
await provider.wait_for_invoice_paid()
async def test_renegotiation(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
test_script_path = str(Path(__file__).parent / "requestor.py")
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(test_script_path,
env=os.environ) as (
_cmd_task,
cmd_monitor,
_process_monitor,
):
await cmd_monitor.wait_for_pattern(r"\[.+\] Renegotiating",
timeout=50)
await cmd_monitor.wait_for_pattern(
r"agreement.terminate\(\): True", timeout=50)
# assert not "Main timeout triggered :("
await cmd_monitor.wait_for_pattern(r"All done", timeout=50)
async def test_run_yacat(
log_dir: Path,
project_dir: Path,
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(
Path(__file__).parent / "assets" / "goth-config.yml")
examples_dir = project_dir / "examples"
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"npm run --prefix {examples_dir} ts:yacat -- -d "
r"--mask ?a?a --hash \$P\$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0 "
"--subnet-tag goth --number-of-providers 2",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
# Add assertions to the command output monitor `cmd_monitor`:
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent)
all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
await cmd_monitor.wait_for_pattern(
f".*Keyspace size computed. Keyspace size = {EXPECTED_KEYSPACE_SIZE}",
timeout=120)
logger.info("Keyspace found")
await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ",
timeout=20)
logger.info("Received proposals")
await all_sent.wait_for_result(timeout=120)
logger.info("All tasks sent")
await all_computed.wait_for_result(timeout=120)
logger.info("All tasks computed")
await cmd_monitor.wait_for_pattern(".*Password found: yo",
timeout=60)
logger.info("Password found, waiting for Executor shutdown")
await cmd_monitor.wait_for_pattern(".*Executor has shut down",
timeout=180)
logger.info("Requestor script finished")
async def test_run_blender(project_dir: Path, log_dir: Path,
goth_config_path: Path,
config_overrides: List[Override]) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
blender_path = project_dir / "examples" / "blender" / "blender.py"
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{blender_path} --subnet-tag goth --min-cpu-threads 1",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
# Add assertions to the command output monitor `cmd_monitor`:
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
all_sent = cmd_monitor.add_assertion(assert_all_tasks_started)
all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ",
timeout=20)
logger.info("Received proposals")
await cmd_monitor.wait_for_pattern(".*Agreement proposed ",
timeout=10)
logger.info("Agreement proposed")
await cmd_monitor.wait_for_pattern(".*Agreement confirmed ",
timeout=10)
logger.info("Agreement confirmed")
await all_sent.wait_for_result(timeout=120)
logger.info("All tasks sent")
await all_computed.wait_for_result(timeout=120)
logger.info("All tasks computed, waiting for Golem shutdown")
await cmd_monitor.wait_for_pattern(
f".*{SummaryLogger.GOLEM_SHUTDOWN_SUCCESSFUL_MESSAGE}",
timeout=120)
logger.info("Requestor script finished")
async def test_run_blender(
log_dir: Path,
project_dir: Path,
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(
Path(__file__).parent / "assets" / "goth-config.yml")
examples_dir = project_dir / "examples"
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"npm run --prefix {examples_dir} ts:blender -- -d --subnet-tag goth",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
# Add assertions to the command output monitor `cmd_monitor`:
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent)
all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
await cmd_monitor.wait_for_pattern(".*Agreement proposed ",
timeout=20)
logger.info("Agreement proposed")
await cmd_monitor.wait_for_pattern(".*Agreement confirmed ",
timeout=20)
logger.info("Agreement confirmed")
await all_sent.wait_for_result(timeout=120)
logger.info("All tasks sent")
await all_computed.wait_for_result(timeout=120)
logger.info("All tasks computed, waiting for Executor shutdown")
await cmd_monitor.wait_for_pattern(".*Executor has shut down",
timeout=180)
logger.info("Requestor script finished")
async def start_network(
configuration: Configuration,
log_dir: Optional[Path] = None,
):
"""Start a test network described by `configuration`."""
runner = Runner(
base_log_dir=log_dir,
compose_config=configuration.compose_config,
test_name="interactive",
api_assertions_module=None,
web_root_path=configuration.web_root,
cancellation_callback=lambda: logger.info("The runner was cancelled"),
)
async with runner(configuration.containers):
providers = runner.get_probes(probe_type=ProviderProbe)
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
# Some test steps may be included in the interactive test as well
for provider in providers:
await provider.provider_agent.wait_for_log("Subscribed offer")
print("\n\033[33;1mNow run your requestor agent as follows:\n")
env = {"PATH": "$PATH"}
requestor.set_agent_env_vars(env)
env_vars = " ".join([f"{key}={val}" for key, val in env.items()])
subnet = providers[0].provider_agent.subnet
print(f"$ {env_vars} examples/blender/blender.py --subnet {subnet}")
print(
"\nPress Ctrl+C at any moment to stop the test harness.\033[0m\n")
while True:
await asyncio.sleep(5)
async def test_agreement_termination(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
test_script_path = str(Path(__file__).parent / "requestor.py")
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(test_script_path,
env=os.environ) as (
_cmd_task,
cmd_monitor,
_process_monitor,
):
cmd_monitor.add_assertion(assert_all_tasks_computed)
# Wait for worker failure due to command error
assertion = cmd_monitor.add_assertion(assert_command_error)
agr_id = await assertion.wait_for_result(timeout=60)
logger.info("Detected command error in activity for agreement %s",
agr_id)
# Make sure no new tasks are sent and the agreement is terminated
assertion = cmd_monitor.add_assertion(
partial(assert_agreement_cancelled, agr_id),
name=f"assert_agreement_cancelled({agr_id})",
)
await assertion.wait_for_result(timeout=10)
# Wait for executor shutdown
await cmd_monitor.wait_for_pattern("ShutdownFinished", timeout=60)
logger.info("Requestor script finished")
async def test_multiactivity_agreement(project_dir: Path, log_dir: Path,
config_overrides) -> None:
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
config_overrides.append(("nodes", nodes))
goth_config = goth.configuration.load_yaml(
project_dir / "tests" / "goth" / "assets" / "goth-config.yml",
config_overrides,
)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
requestor_path = project_dir / "tests" / "goth" / "test_multiactivity_agreement" / "requestor.js"
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"node {requestor_path}",
env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor):
# Wait for agreement
assertion = cmd_monitor.add_assertion(assert_agreement_created)
agr_id = await assertion.wait_for_result(timeout=120)
# Wait for multiple workers run for the agreement
assertion = cmd_monitor.add_assertion(
partial(assert_multiple_workers_run, agr_id),
name=f"assert_multiple_workers_run({agr_id})",
)
await assertion.wait_for_result(timeout=120)
async def test_async_task_generation(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[goth.configuration.Override],
) -> None:
"""Run the `requestor.py` and make sure that it's standard output is as expected."""
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
config_overrides.append(("nodes", nodes))
goth_config = goth.configuration.load_yaml(goth_config_path,
config_overrides)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
str(Path(__file__).parent / "requestor.py"),
env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor):
# The requestor should print "task result: 3" once ...
await cmd_monitor.wait_for_pattern("task result: 3", timeout=60)
# ... then "task result: 2" twice ...
for _ in range(3):
await cmd_monitor.wait_for_pattern("task result: 2",
timeout=10)
# ... and "task result: 1" six times.
for _ in range(6):
await cmd_monitor.wait_for_pattern("task result: 1",
timeout=10)
await cmd_monitor.wait_for_pattern("all done!", timeout=10)
async def test_concurrent_executors(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[goth.configuration.Override],
) -> None:
"""Run the `requestor.py` and make sure that it's standard output is as expected."""
configure_logging(log_dir)
goth_config = goth.configuration.load_yaml(goth_config_path,
config_overrides)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
str(Path(__file__).parent / "requestor.py"),
env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor):
# Wait for job ALEF summary
await cmd_monitor.wait_for_pattern(".*ALEF.* Job finished",
timeout=60)
await cmd_monitor.wait_for_pattern(
".*ALEF.* Negotiated 2 agreements", timeout=5)
await cmd_monitor.wait_for_pattern(
".*ALEF.* Provider .* computed 8 tasks", timeout=5)
await cmd_monitor.wait_for_pattern(
".*ALEF.* Activity failed 1 time", timeout=5)
# Wait for job BET summary
await cmd_monitor.wait_for_pattern(".*BET.* Job finished",
timeout=60)
await cmd_monitor.wait_for_pattern(
".*BET.* Negotiated 1 agreement", timeout=5)
await cmd_monitor.wait_for_pattern(
".*BET.* Provider .* computed 8 tasks", timeout=5)
await cmd_monitor.wait_for_pattern(".*All jobs have finished",
timeout=20)
async def test_agreement_termination(
project_dir: Path,
log_dir: Path,
config_overrides,
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(
project_dir / "tests" / "goth" / "assets" / "goth-config.yml",
config_overrides,
)
test_script_path = project_dir / "tests" / "goth" / "test_agreement_termination" / "requestor.js"
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(f"node {test_script_path}",
env=os.environ) as (
_cmd_task,
cmd_monitor,
_process_monitor,
):
cmd_monitor.add_assertion(assert_all_tasks_computed)
# Make sure no new tasks are sent and the agreement is terminated
assertion = cmd_monitor.add_assertion(assert_agreement_cancelled)
await assertion.wait_for_result(timeout=120)
# Wait for executor shutdown
await cmd_monitor.wait_for_pattern(".*Shutdown complete.*",
timeout=120)
logger.info("Requestor script finished")
async def test_payment_driver_list(
common_assets: Path,
config_overrides: List[Override],
log_dir: Path,
):
"""Test just the requestor's CLI command, no need to setup provider."""
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
]
config_overrides.append(("nodes", nodes))
config_overrides.append(
("docker-compose.build-environment.commit-hash", "29b7f85"))
goth_config = load_yaml(common_assets / "goth-config.yml",
config_overrides)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
web_root_path=common_assets / "web-root",
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
res = requestor.cli.payment_drivers()
assert res and res.items()
driver = next(iter(res.values()), None)
assert driver
assert driver.default_network, "Default network should be set"
network = driver.networks.get(driver.default_network, None)
assert network, "Network should belong to the Driver"
assert network.default_token, "Default taken should be set"
token = network.tokens.get(network.default_token, None)
assert token, "Token should belong to the Network"
async def test_run_custom_usage_counter(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
requestor_path = project_dir / "examples" / "custom-usage-counter" / "custom_usage_counter.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{requestor_path} --running-time {RUNNING_TIME} --subnet-tag {SUBNET_TAG}",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
cmd_monitor.add_assertion(assert_correct_startup_and_shutdown)
cmd_monitor.add_assertion(assert_counter_not_decremented)
await cmd_monitor.wait_for_pattern(".*All jobs have finished",
timeout=300)
logger.info(f"Requestor script finished")
async def test_run_ssh(
log_dir: Path,
project_dir: Path,
config_overrides: List[Override],
ssh_verify_connection: bool,
) -> None:
websocat_check = pexpect.spawn("/usr/bin/which websocat")
exit_code = websocat_check.wait()
if exit_code != 0:
raise ProcessLookupError(
"websocat binary not found, please install it or check your PATH.")
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(
Path(__file__).parent / "assets" / "goth-config.yml")
requestor_path = project_dir / "examples" / "ssh" / "ssh.js"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"node {requestor_path} --subnet-tag {SUBNET_TAG} --timeout 15",
env=os.environ,
) as (_cmd_task, cmd_monitor, process_monitor):
start_time = time.time()
def elapsed_time():
return f"time: {(time.time() - start_time):.1f}"
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
await cmd_monitor.wait_for_pattern(".*Created network", timeout=20)
logger.info(f"Network created")
await cmd_monitor.wait_for_pattern(".*Agreement proposed ",
timeout=20)
logger.info("Agreement proposed")
await cmd_monitor.wait_for_pattern(".*Agreement confirmed ",
timeout=20)
logger.info("Agreement confirmed")
ssh_connections = []
# # A longer timeout to account for downloading a VM image
for i in range(2):
ssh_string = await cmd_monitor.wait_for_pattern(
"ssh -o ProxyCommand", timeout=120)
matches = re.match("ssh -o ProxyCommand=('.*') (root@.*)",
ssh_string)
# the default API port goes through a proxy that logs REST requests
# but does not support websocket connections
# hence, we're replacing it with a port that connects directly
# to the daemon's port in the requestor's Docker container
proxy_cmd = re.sub(":16(\\d\\d\\d)", ":6\\1", matches.group(1))
auth_str = matches.group(2)
password = re.sub(
"Password: "******"", await
cmd_monitor.wait_for_pattern("Password:"******"Skipping SSH connection check. Use `--ssh-verify-connection` to perform it."
)
else:
for proxy_cmd, auth_str, password in ssh_connections:
args = [
"ssh",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"StrictHostKeyChecking=no",
"-o",
"ProxyCommand=" + proxy_cmd,
auth_str,
"uname -v",
]
logger.debug("running ssh with: %s", args)
ssh = pexpect.spawn(" ".join(args))
ssh.expect("[pP]assword:", timeout=5)
ssh.sendline(password)
ssh.expect("#1-Alpine SMP", timeout=5)
ssh.expect(pexpect.EOF, timeout=5)
logger.info("Connection to %s confirmed.", auth_str)
logger.info("SSH connections confirmed.")
for _ in range(2):
await cmd_monitor.wait_for_pattern("Task .* completed",
timeout=20)
await cmd_monitor.wait_for_pattern(".*Computation finished",
timeout=20)
await cmd_monitor.wait_for_pattern(".*Removed network", timeout=20)
logger.info(f"Network removed")
await cmd_monitor.wait_for_pattern(".*Executor has shut down",
timeout=20)
logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_run_ssh(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
ssh_verify_connection: bool,
) -> None:
if ssh_verify_connection:
websocat_check = pexpect.spawn("/usr/bin/which websocat")
exit_code = websocat_check.wait()
if exit_code != 0:
raise ProcessLookupError(
"websocat binary not found, please install it or check your PATH. "
"You may also skip the connection check by omitting `--ssh-verify-connection`."
)
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
# disable the mitm proxy used to capture the requestor agent -> daemon API calls
# because it doesn't support websockets which are neeeded to enable VPN connections
requestor = [c for c in goth_config.containers if c.name == "requestor"][0]
requestor.use_proxy = False
requestor_path = project_dir / "examples" / "ssh" / "ssh.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{requestor_path} --subnet-tag {SUBNET_TAG}",
env=os.environ,
) as (_cmd_task, cmd_monitor, process_monitor):
start_time = time.time()
def elapsed_time():
return f"time: {(time.time() - start_time):.1f}"
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
ssh_connections = []
# A longer timeout to account for downloading a VM image
for i in range(2):
ssh_string = await cmd_monitor.wait_for_pattern(
"ssh -o ProxyCommand", timeout=120)
matches = re.match("ssh -o ProxyCommand=('.*') (root@.*)",
ssh_string)
proxy_cmd = matches.group(1)
auth_str = matches.group(2)
password = re.sub(
"password: "******"", await
cmd_monitor.wait_for_pattern("password:"******".*SshService running on provider.*SshService running on provider",
timeout=10)
if not ssh_verify_connection:
logger.warning(
"Skipping SSH connection check. Use `--ssh-verify-connection` to perform it."
)
else:
for proxy_cmd, auth_str, password in ssh_connections:
args = [
"ssh",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"StrictHostKeyChecking=no",
"-o",
"ProxyCommand=" + proxy_cmd,
auth_str,
"uname -v",
]
logger.debug("running ssh with: %s", args)
ssh = pexpect.spawn(" ".join(args))
ssh.expect("[pP]assword:", timeout=5)
ssh.sendline(password)
ssh.expect("#1-Alpine SMP", timeout=5)
ssh.expect(pexpect.EOF, timeout=5)
logger.info("Connection to %s confirmed.", auth_str)
logger.info("SSH connections confirmed.")
proc: asyncio.subprocess.Process = await process_monitor.get_process(
)
proc.send_signal(signal.SIGINT)
logger.info("Sent SIGINT...")
for _ in range(2):
await cmd_monitor.wait_for_pattern(
".*SshService terminated on provider", timeout=120)
logger.info(
f"The instances have been terminated ({elapsed_time()})")
await cmd_monitor.wait_for_pattern(".*All jobs have finished",
timeout=20)
logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_instance_restart(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[goth.configuration.Override],
) -> None:
"""Run the `requestor.py` and make sure that it's standard output is as expected."""
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
config_overrides.append(("nodes", nodes))
goth_config = goth.configuration.load_yaml(goth_config_path,
config_overrides)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
str(Path(__file__).parent / "requestor.py"),
env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor):
cmd_monitor.add_assertion(count_instances)
# The first attempt to create an instance should fail
await cmd_monitor.wait_for_pattern("STARTING 1$", timeout=60)
await cmd_monitor.wait_for_pattern(".*CommandExecutionError",
timeout=20)
# The second one should successfully start and fail in `running` state
await cmd_monitor.wait_for_pattern("STARTING 2$", timeout=20)
await cmd_monitor.wait_for_pattern("RUNNING 2$", timeout=20)
await cmd_monitor.wait_for_pattern(".*CommandExecutionError",
timeout=20)
await cmd_monitor.wait_for_pattern("STOPPING 2$", timeout=20)
# The third instance should be started, but not running
await cmd_monitor.wait_for_pattern("STARTING 3$", timeout=20)
await cmd_monitor.wait_for_pattern("Cluster stopped$", timeout=60)
assert instances_started == {
1, 2, 3
}, ("Expected to see instances 1, 2, 3 starting, saw instances "
f"{', '.join(str(n) for n in instances_started)} instead")
assert instances_running == {
2
}, ("Expected to see only instance 2 running, saw instances "
f"{', '.join(str(n) for n in instances_running)} instead")
async def test_zero_amount_invoice(
common_assets: Path,
config_overrides: List[Override],
log_dir: Path,
):
"""Test successful flow requesting WASM tasks with goth REST API client."""
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
config_overrides.append(("nodes", nodes))
goth_config = load_yaml(common_assets / "goth-config.yml",
config_overrides)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
web_root_path=common_assets / "web-root",
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
provider = runner.get_probes(probe_type=ProviderProbe)[0]
# Market
demand = DemandBuilder(requestor).props_from_template(
wasi_task_package).build()
agreement_providers = await negotiate_agreements(
requestor,
demand,
[provider],
lambda p: p.properties.get("golem.runtime.name") == "wasmtime",
)
agreement_id = agreement_providers[0][0]
# Zero-amount invoice is issued when agreement is terminated
# without activity
await requestor.wait_for_approval(agreement_id)
await requestor.terminate_agreement(agreement_id, None)
# Payment
await provider.wait_for_invoice_sent()
invoices = await requestor.gather_invoices(agreement_id)
await requestor.pay_invoices(invoices)
await provider.wait_for_invoice_paid()
# verify requestor's invoice is settled
invoice = (await requestor.gather_invoices(agreement_id))[0]
assert invoice.amount == "0"
assert invoice.status == InvoiceStatus.SETTLED
async def test_e2e_vm(
common_assets: Path,
config_overrides: List[Override],
log_dir: Path,
):
"""Test successful flow requesting a Blender task with goth REST API client."""
goth_config = load_yaml(common_assets / "goth-config.yml",
config_overrides)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
web_root_path=Path(__file__).parent / "assets",
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
providers = runner.get_probes(probe_type=ProviderProbe)
# Market
demand = DemandBuilder(requestor).props_from_template(
vm_task_package).build()
agreement_providers = await negotiate_agreements(
requestor,
demand,
providers,
lambda proposal: proposal.properties.get("golem.runtime.name") ==
"vm",
)
# Activity
output_file = "out0000.png"
output_path = Path(runner.web_root_path) / "upload" / output_file
if output_path.exists():
os.remove(output_path)
exe_script = vm_exe_script(runner, output_file)
num_commands = len(exe_script)
for agreement_id, provider in agreement_providers:
logger.info("Running activity on %s", provider.name)
activity_id = await requestor.create_activity(agreement_id)
await provider.wait_for_exeunit_started()
batch_id = await requestor.call_exec(activity_id,
json.dumps(exe_script))
await requestor.collect_results(activity_id,
batch_id,
num_commands,
timeout=300)
await requestor.destroy_activity(activity_id)
await provider.wait_for_exeunit_finished()
assert output_path.is_file()
assert output_path.stat().st_size > 0
# Payment
for agreement_id, provider in agreement_providers:
await provider.wait_for_invoice_sent()
invoices = await requestor.gather_invoices(agreement_id)
assert all(inv.agreement_id == agreement_id for inv in invoices)
# TODO:
await requestor.pay_invoices(invoices)
await provider.wait_for_invoice_paid()
async def test_demand_resubscription(log_dir: Path, goth_config_path: Path,
monkeypatch) -> None:
"""Test that checks that a demand is re-submitted after its previous submission expires."""
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
goth_config = load_yaml(goth_config_path, [("nodes", nodes)])
vm_package = await vm.repo(
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
env = dict(os.environ)
env.update(requestor.get_agent_env_vars())
# Setup the environment for the requestor
for key, val in env.items():
monkeypatch.setenv(key, val)
monitor = EventMonitor()
monitor.add_assertion(assert_demand_resubscribed)
monitor.start()
# The requestor
enable_default_logger()
async def worker(work_ctx, tasks):
async for task in tasks:
script = work_ctx.new_script()
script.run("/bin/sleep", "5")
yield script
task.accept_result()
async with Golem(
budget=10.0,
event_consumer=monitor.add_event_sync,
) as golem:
task: Task # mypy needs this for some reason
async for task in golem.execute_tasks(
worker,
[Task(data=n) for n in range(20)],
vm_package,
max_workers=1,
timeout=timedelta(seconds=30),
):
logger.info("Task %d computed", task.data)
await monitor.stop()
for a in monitor.failed:
raise a.result()
async def test_run_webapp(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
# disable the mitm proxy used to capture the requestor agent -> daemon API calls
# because it doesn't support websockets which are needed by the VPN (and the Local HTTP Proxy)
requestor = [c for c in goth_config.containers if c.name == "requestor"][0]
requestor.use_proxy = False
requestor_path = project_dir / "examples" / "webapp" / "webapp.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{requestor_path} --subnet-tag {SUBNET_TAG}",
env=os.environ,
) as (_cmd_task, cmd_monitor, process_monitor):
start_time = time.time()
def elapsed_time():
return f"time: {(time.time() - start_time):.1f}"
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
logger.info("Waiting for the instances to start")
# A longer timeout to account for downloading a VM image
await cmd_monitor.wait_for_pattern("DB instance started.*", timeout=240)
logger.info("Db instance started")
await cmd_monitor.wait_for_pattern("Local HTTP server listening on.*", timeout=120)
logger.info("HTTP instance started")
requests.post(ONELINER_URL, data={"message": ONELINER_ENTRY})
r = requests.get(ONELINER_URL)
assert r.status_code == 200
assert ONELINER_ENTRY in r.text
logger.info("DB write confirmed :)")
proc: asyncio.subprocess.Process = await process_monitor.get_process()
proc.send_signal(signal.SIGINT)
logger.info("Sent SIGINT...")
for i in range(2):
await cmd_monitor.wait_for_pattern(".*Service terminated.*", timeout=20)
logger.info(f"The instances have been terminated ({elapsed_time()})")
await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20)
logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_run_yacat(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
yacat_path = project_dir / "examples" / "yacat" / "yacat.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
logfile = f"hashcat-yapapi-{datetime.now().strftime('%Y-%m-%d_%H.%M.%S')}.log"
async with requestor.run_command_on_host(
f"{yacat_path} --mask ?a?a --hash $P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0 "
f"--subnet-tag goth --chunk-size {CHUNK_SIZE} --max-workers {PROVIDER_COUNT} "
f"--log-file {(log_dir / logfile).resolve()}",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
# Add assertions to the command output monitor `cmd_monitor`:
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
all_sent = cmd_monitor.add_assertion(assert_all_tasks_started)
all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
await cmd_monitor.wait_for_pattern(".*Received proposals",
timeout=30)
logger.info("Received proposals")
await cmd_monitor.wait_for_pattern(
f".*The keyspace size is {EXPECTED_KEYSPACE_SIZE}",
timeout=120)
logger.info("Keyspace found")
await all_sent.wait_for_result(timeout=60)
logger.info("All tasks sent")
await all_computed.wait_for_result(timeout=120)
logger.info("All tasks computed")
await cmd_monitor.wait_for_pattern(".*Password found: yo",
timeout=60)
logger.info("Password found, waiting for Golem shutdown")
await cmd_monitor.wait_for_pattern(
f".*{SummaryLogger.GOLEM_SHUTDOWN_SUCCESSFUL_MESSAGE}",
timeout=120)
logger.info("Requestor script finished")
async def test_run_scan(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
requestor_path = project_dir / "examples" / "scan" / "scan.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{requestor_path} --subnet-tag {SUBNET_TAG} --scan-size 3",
env=os.environ,
) as (_cmd_task, cmd_monitor, process_monitor):
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
# ensure this line is produced twice with a differing provider and task info:
# Task finished by provider 'provider-N', task data: M
providers = set()
tasks = set()
for i in range(2):
output = await cmd_monitor.wait_for_pattern(
".*Task finished by provider", timeout=120)
matches = re.match(
".*by provider 'provider-(\d)', task data: (\d)", output)
providers.add(matches.group(1))
tasks.add(matches.group(2))
assert providers == {"1", "2"}
assert tasks == {"0", "1"}
logger.info(
f"Scanner tasks completed for the two providers in the network."
)
# ensure no more tasks are executed by the two providers
logger.info("Waiting to see if another task gets started...")
await asyncio.sleep(30)
tasks_finished = [
e for e in cmd_monitor._events
if re.match(".*Task finished by provider", e)
]
assert len(tasks_finished) == 2
logger.info(
f"As expected, no more tasks started. Issuing a break...")
proc: asyncio.subprocess.Process = await process_monitor.get_process(
)
proc.send_signal(signal.SIGINT)
logger.info("SIGINT sent...")
await cmd_monitor.wait_for_pattern(".*All jobs have finished",
timeout=20)
logger.info(f"Requestor script finished.")
async def test_provider_multi_activity(
common_assets: Path,
config_overrides: List[Override],
log_dir: Path,
):
"""Test provider handling multiple activities in single Agreement."""
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
config_overrides.append(("nodes", nodes))
goth_config = load_yaml(common_assets / "goth-config.yml",
config_overrides)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
web_root_path=common_assets / "web-root",
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
providers = runner.get_probes(probe_type=ProviderProbe)
# Market
demand = (DemandBuilder(
requestor).props_from_template(wasi_task_package).property(
"golem.srv.caps.multi-activity",
True).constraints("(&(golem.com.pricing.model=linear)\
(golem.srv.caps.multi-activity=true)\
(golem.runtime.name=wasmtime))").build())
agreement_providers = await negotiate_agreements(
requestor,
demand,
providers,
lambda p: p.properties.get("golem.runtime.name") == "wasmtime",
)
# Activity
exe_script = wasi_exe_script(runner)
for agreement_id, provider in agreement_providers:
for i in range(0, 3):
logger.info("Running activity %s-th time on %s", i,
provider.name)
activity_id = await requestor.create_activity(agreement_id)
await provider.wait_for_exeunit_started()
batch_id = await requestor.call_exec(activity_id,
json.dumps(exe_script))
await requestor.collect_results(activity_id,
batch_id,
len(exe_script),
timeout=30)
await requestor.destroy_activity(activity_id)
await provider.wait_for_exeunit_finished()
await requestor.terminate_agreement(agreement_id, None)
await provider.wait_for_agreement_terminated()
# Payment
await pay_all(requestor, agreement_providers)