async def test_renegotiation(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
test_script_path = str(Path(__file__).parent / "requestor.py")
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(test_script_path,
env=os.environ) as (
_cmd_task,
cmd_monitor,
_process_monitor,
):
await cmd_monitor.wait_for_pattern(r"\[.+\] Renegotiating",
timeout=50)
await cmd_monitor.wait_for_pattern(
r"agreement.terminate\(\): True", timeout=50)
# assert not "Main timeout triggered :("
await cmd_monitor.wait_for_pattern(r"All done", timeout=50)
async def test_run_yacat(
log_dir: Path,
project_dir: Path,
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(
Path(__file__).parent / "assets" / "goth-config.yml")
examples_dir = project_dir / "examples"
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"npm run --prefix {examples_dir} ts:yacat -- -d "
r"--mask ?a?a --hash \$P\$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0 "
"--subnet-tag goth --number-of-providers 2",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
# Add assertions to the command output monitor `cmd_monitor`:
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent)
all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
await cmd_monitor.wait_for_pattern(
f".*Keyspace size computed. Keyspace size = {EXPECTED_KEYSPACE_SIZE}",
timeout=120)
logger.info("Keyspace found")
await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ",
timeout=20)
logger.info("Received proposals")
await all_sent.wait_for_result(timeout=120)
logger.info("All tasks sent")
await all_computed.wait_for_result(timeout=120)
logger.info("All tasks computed")
await cmd_monitor.wait_for_pattern(".*Password found: yo",
timeout=60)
logger.info("Password found, waiting for Executor shutdown")
await cmd_monitor.wait_for_pattern(".*Executor has shut down",
timeout=180)
logger.info("Requestor script finished")
def log_dir() -> Path:
base_dir = Path("/", "tmp", "goth-tests")
date_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z")
log_dir = base_dir / f"goth_{date_str}"
log_dir.mkdir(parents=True)
configure_logging(log_dir)
return log_dir
async def test_run_blender(project_dir: Path, log_dir: Path,
goth_config_path: Path,
config_overrides: List[Override]) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
blender_path = project_dir / "examples" / "blender" / "blender.py"
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{blender_path} --subnet-tag goth --min-cpu-threads 1",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
# Add assertions to the command output monitor `cmd_monitor`:
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
all_sent = cmd_monitor.add_assertion(assert_all_tasks_started)
all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ",
timeout=20)
logger.info("Received proposals")
await cmd_monitor.wait_for_pattern(".*Agreement proposed ",
timeout=10)
logger.info("Agreement proposed")
await cmd_monitor.wait_for_pattern(".*Agreement confirmed ",
timeout=10)
logger.info("Agreement confirmed")
await all_sent.wait_for_result(timeout=120)
logger.info("All tasks sent")
await all_computed.wait_for_result(timeout=120)
logger.info("All tasks computed, waiting for Golem shutdown")
await cmd_monitor.wait_for_pattern(
f".*{SummaryLogger.GOLEM_SHUTDOWN_SUCCESSFUL_MESSAGE}",
timeout=120)
logger.info("Requestor script finished")
def start(args):
"""Start the test network using a configuration read from `args.config_file`."""
configuration = load_yaml(args.config_file)
base_log_dir = args.log_dir or DEFAULT_LOG_DIR
log_dir = make_logs_dir(Path(base_log_dir))
configure_logging(log_dir, args.log_level)
loop = asyncio.get_event_loop()
task = start_network(configuration, log_dir=log_dir)
loop.run_until_complete(task)
async def test_run_blender(
log_dir: Path,
project_dir: Path,
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(
Path(__file__).parent / "assets" / "goth-config.yml")
examples_dir = project_dir / "examples"
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"npm run --prefix {examples_dir} ts:blender -- -d --subnet-tag goth",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
# Add assertions to the command output monitor `cmd_monitor`:
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent)
all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
await cmd_monitor.wait_for_pattern(".*Agreement proposed ",
timeout=20)
logger.info("Agreement proposed")
await cmd_monitor.wait_for_pattern(".*Agreement confirmed ",
timeout=20)
logger.info("Agreement confirmed")
await all_sent.wait_for_result(timeout=120)
logger.info("All tasks sent")
await all_computed.wait_for_result(timeout=120)
logger.info("All tasks computed, waiting for Executor shutdown")
await cmd_monitor.wait_for_pattern(".*Executor has shut down",
timeout=180)
logger.info("Requestor script finished")
async def test_agreement_termination(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
test_script_path = str(Path(__file__).parent / "requestor.py")
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(test_script_path,
env=os.environ) as (
_cmd_task,
cmd_monitor,
_process_monitor,
):
cmd_monitor.add_assertion(assert_all_tasks_computed)
# Wait for worker failure due to command error
assertion = cmd_monitor.add_assertion(assert_command_error)
agr_id = await assertion.wait_for_result(timeout=60)
logger.info("Detected command error in activity for agreement %s",
agr_id)
# Make sure no new tasks are sent and the agreement is terminated
assertion = cmd_monitor.add_assertion(
partial(assert_agreement_cancelled, agr_id),
name=f"assert_agreement_cancelled({agr_id})",
)
await assertion.wait_for_result(timeout=10)
# Wait for executor shutdown
await cmd_monitor.wait_for_pattern("ShutdownFinished", timeout=60)
logger.info("Requestor script finished")
async def test_multiactivity_agreement(project_dir: Path, log_dir: Path,
config_overrides) -> None:
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
config_overrides.append(("nodes", nodes))
goth_config = goth.configuration.load_yaml(
project_dir / "tests" / "goth" / "assets" / "goth-config.yml",
config_overrides,
)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
requestor_path = project_dir / "tests" / "goth" / "test_multiactivity_agreement" / "requestor.js"
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"node {requestor_path}",
env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor):
# Wait for agreement
assertion = cmd_monitor.add_assertion(assert_agreement_created)
agr_id = await assertion.wait_for_result(timeout=120)
# Wait for multiple workers run for the agreement
assertion = cmd_monitor.add_assertion(
partial(assert_multiple_workers_run, agr_id),
name=f"assert_multiple_workers_run({agr_id})",
)
await assertion.wait_for_result(timeout=120)
async def test_async_task_generation(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[goth.configuration.Override],
) -> None:
"""Run the `requestor.py` and make sure that it's standard output is as expected."""
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
config_overrides.append(("nodes", nodes))
goth_config = goth.configuration.load_yaml(goth_config_path,
config_overrides)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
str(Path(__file__).parent / "requestor.py"),
env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor):
# The requestor should print "task result: 3" once ...
await cmd_monitor.wait_for_pattern("task result: 3", timeout=60)
# ... then "task result: 2" twice ...
for _ in range(3):
await cmd_monitor.wait_for_pattern("task result: 2",
timeout=10)
# ... and "task result: 1" six times.
for _ in range(6):
await cmd_monitor.wait_for_pattern("task result: 1",
timeout=10)
await cmd_monitor.wait_for_pattern("all done!", timeout=10)
async def test_concurrent_executors(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[goth.configuration.Override],
) -> None:
"""Run the `requestor.py` and make sure that it's standard output is as expected."""
configure_logging(log_dir)
goth_config = goth.configuration.load_yaml(goth_config_path,
config_overrides)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
str(Path(__file__).parent / "requestor.py"),
env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor):
# Wait for job ALEF summary
await cmd_monitor.wait_for_pattern(".*ALEF.* Job finished",
timeout=60)
await cmd_monitor.wait_for_pattern(
".*ALEF.* Negotiated 2 agreements", timeout=5)
await cmd_monitor.wait_for_pattern(
".*ALEF.* Provider .* computed 8 tasks", timeout=5)
await cmd_monitor.wait_for_pattern(
".*ALEF.* Activity failed 1 time", timeout=5)
# Wait for job BET summary
await cmd_monitor.wait_for_pattern(".*BET.* Job finished",
timeout=60)
await cmd_monitor.wait_for_pattern(
".*BET.* Negotiated 1 agreement", timeout=5)
await cmd_monitor.wait_for_pattern(
".*BET.* Provider .* computed 8 tasks", timeout=5)
await cmd_monitor.wait_for_pattern(".*All jobs have finished",
timeout=20)
async def test_agreement_termination(
project_dir: Path,
log_dir: Path,
config_overrides,
) -> None:
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(
project_dir / "tests" / "goth" / "assets" / "goth-config.yml",
config_overrides,
)
test_script_path = project_dir / "tests" / "goth" / "test_agreement_termination" / "requestor.js"
configure_logging(log_dir)
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(f"node {test_script_path}",
env=os.environ) as (
_cmd_task,
cmd_monitor,
_process_monitor,
):
cmd_monitor.add_assertion(assert_all_tasks_computed)
# Make sure no new tasks are sent and the agreement is terminated
assertion = cmd_monitor.add_assertion(assert_agreement_cancelled)
await assertion.wait_for_result(timeout=120)
# Wait for executor shutdown
await cmd_monitor.wait_for_pattern(".*Shutdown complete.*",
timeout=120)
logger.info("Requestor script finished")
async def test_run_custom_usage_counter(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
requestor_path = project_dir / "examples" / "custom-usage-counter" / "custom_usage_counter.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{requestor_path} --running-time {RUNNING_TIME} --subnet-tag {SUBNET_TAG}",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
cmd_monitor.add_assertion(assert_correct_startup_and_shutdown)
cmd_monitor.add_assertion(assert_counter_not_decremented)
await cmd_monitor.wait_for_pattern(".*All jobs have finished",
timeout=300)
logger.info(f"Requestor script finished")
async def test_run_scan(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
requestor_path = project_dir / "examples" / "scan" / "scan.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{requestor_path} --subnet-tag {SUBNET_TAG} --scan-size 3",
env=os.environ,
) as (_cmd_task, cmd_monitor, process_monitor):
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
# ensure this line is produced twice with a differing provider and task info:
# Task finished by provider 'provider-N', task data: M
providers = set()
tasks = set()
for i in range(2):
output = await cmd_monitor.wait_for_pattern(
".*Task finished by provider", timeout=120)
matches = re.match(
".*by provider 'provider-(\d)', task data: (\d)", output)
providers.add(matches.group(1))
tasks.add(matches.group(2))
assert providers == {"1", "2"}
assert tasks == {"0", "1"}
logger.info(
f"Scanner tasks completed for the two providers in the network."
)
# ensure no more tasks are executed by the two providers
logger.info("Waiting to see if another task gets started...")
await asyncio.sleep(30)
tasks_finished = [
e for e in cmd_monitor._events
if re.match(".*Task finished by provider", e)
]
assert len(tasks_finished) == 2
logger.info(
f"As expected, no more tasks started. Issuing a break...")
proc: asyncio.subprocess.Process = await process_monitor.get_process(
)
proc.send_signal(signal.SIGINT)
logger.info("SIGINT sent...")
await cmd_monitor.wait_for_pattern(".*All jobs have finished",
timeout=20)
logger.info(f"Requestor script finished.")
async def test_run_webapp(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
# disable the mitm proxy used to capture the requestor agent -> daemon API calls
# because it doesn't support websockets which are needed by the VPN (and the Local HTTP Proxy)
requestor = [c for c in goth_config.containers if c.name == "requestor"][0]
requestor.use_proxy = False
requestor_path = project_dir / "examples" / "webapp" / "webapp.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{requestor_path} --subnet-tag {SUBNET_TAG}",
env=os.environ,
) as (_cmd_task, cmd_monitor, process_monitor):
start_time = time.time()
def elapsed_time():
return f"time: {(time.time() - start_time):.1f}"
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
logger.info("Waiting for the instances to start")
# A longer timeout to account for downloading a VM image
await cmd_monitor.wait_for_pattern("DB instance started.*", timeout=240)
logger.info("Db instance started")
await cmd_monitor.wait_for_pattern("Local HTTP server listening on.*", timeout=120)
logger.info("HTTP instance started")
requests.post(ONELINER_URL, data={"message": ONELINER_ENTRY})
r = requests.get(ONELINER_URL)
assert r.status_code == 200
assert ONELINER_ENTRY in r.text
logger.info("DB write confirmed :)")
proc: asyncio.subprocess.Process = await process_monitor.get_process()
proc.send_signal(signal.SIGINT)
logger.info("Sent SIGINT...")
for i in range(2):
await cmd_monitor.wait_for_pattern(".*Service terminated.*", timeout=20)
logger.info(f"The instances have been terminated ({elapsed_time()})")
await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20)
logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_run_ssh(
log_dir: Path,
project_dir: Path,
config_overrides: List[Override],
ssh_verify_connection: bool,
) -> None:
websocat_check = pexpect.spawn("/usr/bin/which websocat")
exit_code = websocat_check.wait()
if exit_code != 0:
raise ProcessLookupError(
"websocat binary not found, please install it or check your PATH.")
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(
Path(__file__).parent / "assets" / "goth-config.yml")
requestor_path = project_dir / "examples" / "ssh" / "ssh.js"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"node {requestor_path} --subnet-tag {SUBNET_TAG} --timeout 15",
env=os.environ,
) as (_cmd_task, cmd_monitor, process_monitor):
start_time = time.time()
def elapsed_time():
return f"time: {(time.time() - start_time):.1f}"
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
await cmd_monitor.wait_for_pattern(".*Created network", timeout=20)
logger.info(f"Network created")
await cmd_monitor.wait_for_pattern(".*Agreement proposed ",
timeout=20)
logger.info("Agreement proposed")
await cmd_monitor.wait_for_pattern(".*Agreement confirmed ",
timeout=20)
logger.info("Agreement confirmed")
ssh_connections = []
# # A longer timeout to account for downloading a VM image
for i in range(2):
ssh_string = await cmd_monitor.wait_for_pattern(
"ssh -o ProxyCommand", timeout=120)
matches = re.match("ssh -o ProxyCommand=('.*') (root@.*)",
ssh_string)
# the default API port goes through a proxy that logs REST requests
# but does not support websocket connections
# hence, we're replacing it with a port that connects directly
# to the daemon's port in the requestor's Docker container
proxy_cmd = re.sub(":16(\\d\\d\\d)", ":6\\1", matches.group(1))
auth_str = matches.group(2)
password = re.sub(
"Password: "******"", await
cmd_monitor.wait_for_pattern("Password:"******"Skipping SSH connection check. Use `--ssh-verify-connection` to perform it."
)
else:
for proxy_cmd, auth_str, password in ssh_connections:
args = [
"ssh",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"StrictHostKeyChecking=no",
"-o",
"ProxyCommand=" + proxy_cmd,
auth_str,
"uname -v",
]
logger.debug("running ssh with: %s", args)
ssh = pexpect.spawn(" ".join(args))
ssh.expect("[pP]assword:", timeout=5)
ssh.sendline(password)
ssh.expect("#1-Alpine SMP", timeout=5)
ssh.expect(pexpect.EOF, timeout=5)
logger.info("Connection to %s confirmed.", auth_str)
logger.info("SSH connections confirmed.")
for _ in range(2):
await cmd_monitor.wait_for_pattern("Task .* completed",
timeout=20)
await cmd_monitor.wait_for_pattern(".*Computation finished",
timeout=20)
await cmd_monitor.wait_for_pattern(".*Removed network", timeout=20)
logger.info(f"Network removed")
await cmd_monitor.wait_for_pattern(".*Executor has shut down",
timeout=20)
logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_run_ssh(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
ssh_verify_connection: bool,
) -> None:
if ssh_verify_connection:
websocat_check = pexpect.spawn("/usr/bin/which websocat")
exit_code = websocat_check.wait()
if exit_code != 0:
raise ProcessLookupError(
"websocat binary not found, please install it or check your PATH. "
"You may also skip the connection check by omitting `--ssh-verify-connection`."
)
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
# disable the mitm proxy used to capture the requestor agent -> daemon API calls
# because it doesn't support websockets which are neeeded to enable VPN connections
requestor = [c for c in goth_config.containers if c.name == "requestor"][0]
requestor.use_proxy = False
requestor_path = project_dir / "examples" / "ssh" / "ssh.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
f"{requestor_path} --subnet-tag {SUBNET_TAG}",
env=os.environ,
) as (_cmd_task, cmd_monitor, process_monitor):
start_time = time.time()
def elapsed_time():
return f"time: {(time.time() - start_time):.1f}"
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
ssh_connections = []
# A longer timeout to account for downloading a VM image
for i in range(2):
ssh_string = await cmd_monitor.wait_for_pattern(
"ssh -o ProxyCommand", timeout=120)
matches = re.match("ssh -o ProxyCommand=('.*') (root@.*)",
ssh_string)
proxy_cmd = matches.group(1)
auth_str = matches.group(2)
password = re.sub(
"password: "******"", await
cmd_monitor.wait_for_pattern("password:"******".*SshService running on provider.*SshService running on provider",
timeout=10)
if not ssh_verify_connection:
logger.warning(
"Skipping SSH connection check. Use `--ssh-verify-connection` to perform it."
)
else:
for proxy_cmd, auth_str, password in ssh_connections:
args = [
"ssh",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"StrictHostKeyChecking=no",
"-o",
"ProxyCommand=" + proxy_cmd,
auth_str,
"uname -v",
]
logger.debug("running ssh with: %s", args)
ssh = pexpect.spawn(" ".join(args))
ssh.expect("[pP]assword:", timeout=5)
ssh.sendline(password)
ssh.expect("#1-Alpine SMP", timeout=5)
ssh.expect(pexpect.EOF, timeout=5)
logger.info("Connection to %s confirmed.", auth_str)
logger.info("SSH connections confirmed.")
proc: asyncio.subprocess.Process = await process_monitor.get_process(
)
proc.send_signal(signal.SIGINT)
logger.info("Sent SIGINT...")
for _ in range(2):
await cmd_monitor.wait_for_pattern(
".*SshService terminated on provider", timeout=120)
logger.info(
f"The instances have been terminated ({elapsed_time()})")
await cmd_monitor.wait_for_pattern(".*All jobs have finished",
timeout=20)
logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_instance_restart(
log_dir: Path,
goth_config_path: Path,
config_overrides: List[goth.configuration.Override],
) -> None:
"""Run the `requestor.py` and make sure that it's standard output is as expected."""
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
config_overrides.append(("nodes", nodes))
goth_config = goth.configuration.load_yaml(goth_config_path,
config_overrides)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
async with requestor.run_command_on_host(
str(Path(__file__).parent / "requestor.py"),
env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor):
cmd_monitor.add_assertion(count_instances)
# The first attempt to create an instance should fail
await cmd_monitor.wait_for_pattern("STARTING 1$", timeout=60)
await cmd_monitor.wait_for_pattern(".*CommandExecutionError",
timeout=20)
# The second one should successfully start and fail in `running` state
await cmd_monitor.wait_for_pattern("STARTING 2$", timeout=20)
await cmd_monitor.wait_for_pattern("RUNNING 2$", timeout=20)
await cmd_monitor.wait_for_pattern(".*CommandExecutionError",
timeout=20)
await cmd_monitor.wait_for_pattern("STOPPING 2$", timeout=20)
# The third instance should be started, but not running
await cmd_monitor.wait_for_pattern("STARTING 3$", timeout=20)
await cmd_monitor.wait_for_pattern("Cluster stopped$", timeout=60)
assert instances_started == {
1, 2, 3
}, ("Expected to see instances 1, 2, 3 starting, saw instances "
f"{', '.join(str(n) for n in instances_started)} instead")
assert instances_running == {
2
}, ("Expected to see only instance 2 running, saw instances "
f"{', '.join(str(n) for n in instances_running)} instead")
async def test_demand_resubscription(log_dir: Path, goth_config_path: Path,
monkeypatch) -> None:
"""Test that checks that a demand is re-submitted after its previous submission expires."""
configure_logging(log_dir)
# Override the default test configuration to create only one provider node
nodes = [
{
"name": "requestor",
"type": "Requestor"
},
{
"name": "provider-1",
"type": "VM-Wasm-Provider",
"use-proxy": True
},
]
goth_config = load_yaml(goth_config_path, [("nodes", nodes)])
vm_package = await vm.repo(
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
runner = Runner(base_log_dir=log_dir,
compose_config=goth_config.compose_config)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
env = dict(os.environ)
env.update(requestor.get_agent_env_vars())
# Setup the environment for the requestor
for key, val in env.items():
monkeypatch.setenv(key, val)
monitor = EventMonitor()
monitor.add_assertion(assert_demand_resubscribed)
monitor.start()
# The requestor
enable_default_logger()
async def worker(work_ctx, tasks):
async for task in tasks:
script = work_ctx.new_script()
script.run("/bin/sleep", "5")
yield script
task.accept_result()
async with Golem(
budget=10.0,
event_consumer=monitor.add_event_sync,
) as golem:
task: Task # mypy needs this for some reason
async for task in golem.execute_tasks(
worker,
[Task(data=n) for n in range(20)],
vm_package,
max_workers=1,
timeout=timedelta(seconds=30),
):
logger.info("Task %d computed", task.data)
await monitor.stop()
for a in monitor.failed:
raise a.result()
async def test_run_yacat(
log_dir: Path,
project_dir: Path,
goth_config_path: Path,
config_overrides: List[Override],
) -> None:
configure_logging(log_dir)
# This is the default configuration with 2 wasm/VM providers
goth_config = load_yaml(goth_config_path, config_overrides)
yacat_path = project_dir / "examples" / "yacat" / "yacat.py"
runner = Runner(
base_log_dir=log_dir,
compose_config=goth_config.compose_config,
)
async with runner(goth_config.containers):
requestor = runner.get_probes(probe_type=RequestorProbe)[0]
logfile = f"hashcat-yapapi-{datetime.now().strftime('%Y-%m-%d_%H.%M.%S')}.log"
async with requestor.run_command_on_host(
f"{yacat_path} --mask ?a?a --hash $P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0 "
f"--subnet-tag goth --chunk-size {CHUNK_SIZE} --max-workers {PROVIDER_COUNT} "
f"--log-file {(log_dir / logfile).resolve()}",
env=os.environ,
) as (_cmd_task, cmd_monitor, _process_monitor):
# Add assertions to the command output monitor `cmd_monitor`:
cmd_monitor.add_assertion(assert_no_errors)
cmd_monitor.add_assertion(assert_all_invoices_accepted)
all_sent = cmd_monitor.add_assertion(assert_all_tasks_started)
all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed)
await cmd_monitor.wait_for_pattern(".*Received proposals",
timeout=30)
logger.info("Received proposals")
await cmd_monitor.wait_for_pattern(
f".*The keyspace size is {EXPECTED_KEYSPACE_SIZE}",
timeout=120)
logger.info("Keyspace found")
await all_sent.wait_for_result(timeout=60)
logger.info("All tasks sent")
await all_computed.wait_for_result(timeout=120)
logger.info("All tasks computed")
await cmd_monitor.wait_for_pattern(".*Password found: yo",
timeout=60)
logger.info("Password found, waiting for Golem shutdown")
await cmd_monitor.wait_for_pattern(
f".*{SummaryLogger.GOLEM_SHUTDOWN_SUCCESSFUL_MESSAGE}",
timeout=120)
logger.info("Requestor script finished")
