async def test_renegotiation( log_dir: Path, goth_config_path: Path, config_overrides: List[Override], ) -> None: # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) test_script_path = str(Path(__file__).parent / "requestor.py") configure_logging(log_dir) runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host(test_script_path, env=os.environ) as ( _cmd_task, cmd_monitor, _process_monitor, ): await cmd_monitor.wait_for_pattern(r"\[.+\] Renegotiating", timeout=50) await cmd_monitor.wait_for_pattern( r"agreement.terminate\(\): True", timeout=50) # assert not "Main timeout triggered :(" await cmd_monitor.wait_for_pattern(r"All done", timeout=50)
async def test_run_yacat( log_dir: Path, project_dir: Path, ) -> None: # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml( Path(__file__).parent / "assets" / "goth-config.yml") examples_dir = project_dir / "examples" configure_logging(log_dir) runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"npm run --prefix {examples_dir} ts:yacat -- -d " r"--mask ?a?a --hash \$P\$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0 " "--subnet-tag goth --number-of-providers 2", env=os.environ, ) as (_cmd_task, cmd_monitor, _process_monitor): # Add assertions to the command output monitor `cmd_monitor`: cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent) all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed) await cmd_monitor.wait_for_pattern( f".*Keyspace size computed. Keyspace size = {EXPECTED_KEYSPACE_SIZE}", timeout=120) logger.info("Keyspace found") await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ", timeout=20) logger.info("Received proposals") await all_sent.wait_for_result(timeout=120) logger.info("All tasks sent") await all_computed.wait_for_result(timeout=120) logger.info("All tasks computed") await cmd_monitor.wait_for_pattern(".*Password found: yo", timeout=60) logger.info("Password found, waiting for Executor shutdown") await cmd_monitor.wait_for_pattern(".*Executor has shut down", timeout=180) logger.info("Requestor script finished")
def log_dir() -> Path: base_dir = Path("/", "tmp", "goth-tests") date_str = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S%z") log_dir = base_dir / f"goth_{date_str}" log_dir.mkdir(parents=True) configure_logging(log_dir) return log_dir
async def test_run_blender(project_dir: Path, log_dir: Path, goth_config_path: Path, config_overrides: List[Override]) -> None: # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) blender_path = project_dir / "examples" / "blender" / "blender.py" configure_logging(log_dir) runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"{blender_path} --subnet-tag goth --min-cpu-threads 1", env=os.environ, ) as (_cmd_task, cmd_monitor, _process_monitor): # Add assertions to the command output monitor `cmd_monitor`: cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) all_sent = cmd_monitor.add_assertion(assert_all_tasks_started) all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed) await cmd_monitor.wait_for_pattern(".*Received proposals from 2 ", timeout=20) logger.info("Received proposals") await cmd_monitor.wait_for_pattern(".*Agreement proposed ", timeout=10) logger.info("Agreement proposed") await cmd_monitor.wait_for_pattern(".*Agreement confirmed ", timeout=10) logger.info("Agreement confirmed") await all_sent.wait_for_result(timeout=120) logger.info("All tasks sent") await all_computed.wait_for_result(timeout=120) logger.info("All tasks computed, waiting for Golem shutdown") await cmd_monitor.wait_for_pattern( f".*{SummaryLogger.GOLEM_SHUTDOWN_SUCCESSFUL_MESSAGE}", timeout=120) logger.info("Requestor script finished")
def start(args): """Start the test network using a configuration read from `args.config_file`.""" configuration = load_yaml(args.config_file) base_log_dir = args.log_dir or DEFAULT_LOG_DIR log_dir = make_logs_dir(Path(base_log_dir)) configure_logging(log_dir, args.log_level) loop = asyncio.get_event_loop() task = start_network(configuration, log_dir=log_dir) loop.run_until_complete(task)
async def test_run_blender( log_dir: Path, project_dir: Path, ) -> None: # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml( Path(__file__).parent / "assets" / "goth-config.yml") examples_dir = project_dir / "examples" configure_logging(log_dir) runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"npm run --prefix {examples_dir} ts:blender -- -d --subnet-tag goth", env=os.environ, ) as (_cmd_task, cmd_monitor, _process_monitor): # Add assertions to the command output monitor `cmd_monitor`: cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) all_sent = cmd_monitor.add_assertion(assert_all_tasks_sent) all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed) await cmd_monitor.wait_for_pattern(".*Agreement proposed ", timeout=20) logger.info("Agreement proposed") await cmd_monitor.wait_for_pattern(".*Agreement confirmed ", timeout=20) logger.info("Agreement confirmed") await all_sent.wait_for_result(timeout=120) logger.info("All tasks sent") await all_computed.wait_for_result(timeout=120) logger.info("All tasks computed, waiting for Executor shutdown") await cmd_monitor.wait_for_pattern(".*Executor has shut down", timeout=180) logger.info("Requestor script finished")
async def test_agreement_termination( log_dir: Path, goth_config_path: Path, config_overrides: List[Override], ) -> None: # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) test_script_path = str(Path(__file__).parent / "requestor.py") configure_logging(log_dir) runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host(test_script_path, env=os.environ) as ( _cmd_task, cmd_monitor, _process_monitor, ): cmd_monitor.add_assertion(assert_all_tasks_computed) # Wait for worker failure due to command error assertion = cmd_monitor.add_assertion(assert_command_error) agr_id = await assertion.wait_for_result(timeout=60) logger.info("Detected command error in activity for agreement %s", agr_id) # Make sure no new tasks are sent and the agreement is terminated assertion = cmd_monitor.add_assertion( partial(assert_agreement_cancelled, agr_id), name=f"assert_agreement_cancelled({agr_id})", ) await assertion.wait_for_result(timeout=10) # Wait for executor shutdown await cmd_monitor.wait_for_pattern("ShutdownFinished", timeout=60) logger.info("Requestor script finished")
async def test_multiactivity_agreement(project_dir: Path, log_dir: Path, config_overrides) -> None: configure_logging(log_dir) # Override the default test configuration to create only one provider node nodes = [ { "name": "requestor", "type": "Requestor" }, { "name": "provider-1", "type": "VM-Wasm-Provider", "use-proxy": True }, ] config_overrides.append(("nodes", nodes)) goth_config = goth.configuration.load_yaml( project_dir / "tests" / "goth" / "assets" / "goth-config.yml", config_overrides, ) runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) requestor_path = project_dir / "tests" / "goth" / "test_multiactivity_agreement" / "requestor.js" async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"node {requestor_path}", env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor): # Wait for agreement assertion = cmd_monitor.add_assertion(assert_agreement_created) agr_id = await assertion.wait_for_result(timeout=120) # Wait for multiple workers run for the agreement assertion = cmd_monitor.add_assertion( partial(assert_multiple_workers_run, agr_id), name=f"assert_multiple_workers_run({agr_id})", ) await assertion.wait_for_result(timeout=120)
async def test_async_task_generation( log_dir: Path, goth_config_path: Path, config_overrides: List[goth.configuration.Override], ) -> None: """Run the `requestor.py` and make sure that it's standard output is as expected.""" configure_logging(log_dir) # Override the default test configuration to create only one provider node nodes = [ { "name": "requestor", "type": "Requestor" }, { "name": "provider-1", "type": "VM-Wasm-Provider", "use-proxy": True }, ] config_overrides.append(("nodes", nodes)) goth_config = goth.configuration.load_yaml(goth_config_path, config_overrides) runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( str(Path(__file__).parent / "requestor.py"), env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor): # The requestor should print "task result: 3" once ... await cmd_monitor.wait_for_pattern("task result: 3", timeout=60) # ... then "task result: 2" twice ... for _ in range(3): await cmd_monitor.wait_for_pattern("task result: 2", timeout=10) # ... and "task result: 1" six times. for _ in range(6): await cmd_monitor.wait_for_pattern("task result: 1", timeout=10) await cmd_monitor.wait_for_pattern("all done!", timeout=10)
async def test_concurrent_executors( log_dir: Path, goth_config_path: Path, config_overrides: List[goth.configuration.Override], ) -> None: """Run the `requestor.py` and make sure that it's standard output is as expected.""" configure_logging(log_dir) goth_config = goth.configuration.load_yaml(goth_config_path, config_overrides) runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( str(Path(__file__).parent / "requestor.py"), env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor): # Wait for job ALEF summary await cmd_monitor.wait_for_pattern(".*ALEF.* Job finished", timeout=60) await cmd_monitor.wait_for_pattern( ".*ALEF.* Negotiated 2 agreements", timeout=5) await cmd_monitor.wait_for_pattern( ".*ALEF.* Provider .* computed 8 tasks", timeout=5) await cmd_monitor.wait_for_pattern( ".*ALEF.* Activity failed 1 time", timeout=5) # Wait for job BET summary await cmd_monitor.wait_for_pattern(".*BET.* Job finished", timeout=60) await cmd_monitor.wait_for_pattern( ".*BET.* Negotiated 1 agreement", timeout=5) await cmd_monitor.wait_for_pattern( ".*BET.* Provider .* computed 8 tasks", timeout=5) await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20)
async def test_agreement_termination( project_dir: Path, log_dir: Path, config_overrides, ) -> None: # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml( project_dir / "tests" / "goth" / "assets" / "goth-config.yml", config_overrides, ) test_script_path = project_dir / "tests" / "goth" / "test_agreement_termination" / "requestor.js" configure_logging(log_dir) runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host(f"node {test_script_path}", env=os.environ) as ( _cmd_task, cmd_monitor, _process_monitor, ): cmd_monitor.add_assertion(assert_all_tasks_computed) # Make sure no new tasks are sent and the agreement is terminated assertion = cmd_monitor.add_assertion(assert_agreement_cancelled) await assertion.wait_for_result(timeout=120) # Wait for executor shutdown await cmd_monitor.wait_for_pattern(".*Shutdown complete.*", timeout=120) logger.info("Requestor script finished")
async def test_run_custom_usage_counter( log_dir: Path, project_dir: Path, goth_config_path: Path, config_overrides: List[Override], ) -> None: configure_logging(log_dir) # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) requestor_path = project_dir / "examples" / "custom-usage-counter" / "custom_usage_counter.py" runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"{requestor_path} --running-time {RUNNING_TIME} --subnet-tag {SUBNET_TAG}", env=os.environ, ) as (_cmd_task, cmd_monitor, _process_monitor): cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) cmd_monitor.add_assertion(assert_correct_startup_and_shutdown) cmd_monitor.add_assertion(assert_counter_not_decremented) await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=300) logger.info(f"Requestor script finished")
async def test_run_scan( log_dir: Path, project_dir: Path, goth_config_path: Path, config_overrides: List[Override], ) -> None: configure_logging(log_dir) # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) requestor_path = project_dir / "examples" / "scan" / "scan.py" runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"{requestor_path} --subnet-tag {SUBNET_TAG} --scan-size 3", env=os.environ, ) as (_cmd_task, cmd_monitor, process_monitor): cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) # ensure this line is produced twice with a differing provider and task info: # Task finished by provider 'provider-N', task data: M providers = set() tasks = set() for i in range(2): output = await cmd_monitor.wait_for_pattern( ".*Task finished by provider", timeout=120) matches = re.match( ".*by provider 'provider-(\d)', task data: (\d)", output) providers.add(matches.group(1)) tasks.add(matches.group(2)) assert providers == {"1", "2"} assert tasks == {"0", "1"} logger.info( f"Scanner tasks completed for the two providers in the network." ) # ensure no more tasks are executed by the two providers logger.info("Waiting to see if another task gets started...") await asyncio.sleep(30) tasks_finished = [ e for e in cmd_monitor._events if re.match(".*Task finished by provider", e) ] assert len(tasks_finished) == 2 logger.info( f"As expected, no more tasks started. Issuing a break...") proc: asyncio.subprocess.Process = await process_monitor.get_process( ) proc.send_signal(signal.SIGINT) logger.info("SIGINT sent...") await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20) logger.info(f"Requestor script finished.")
async def test_run_webapp( log_dir: Path, project_dir: Path, goth_config_path: Path, config_overrides: List[Override], ) -> None: configure_logging(log_dir) # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) # disable the mitm proxy used to capture the requestor agent -> daemon API calls # because it doesn't support websockets which are needed by the VPN (and the Local HTTP Proxy) requestor = [c for c in goth_config.containers if c.name == "requestor"][0] requestor.use_proxy = False requestor_path = project_dir / "examples" / "webapp" / "webapp.py" runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"{requestor_path} --subnet-tag {SUBNET_TAG}", env=os.environ, ) as (_cmd_task, cmd_monitor, process_monitor): start_time = time.time() def elapsed_time(): return f"time: {(time.time() - start_time):.1f}" cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) logger.info("Waiting for the instances to start") # A longer timeout to account for downloading a VM image await cmd_monitor.wait_for_pattern("DB instance started.*", timeout=240) logger.info("Db instance started") await cmd_monitor.wait_for_pattern("Local HTTP server listening on.*", timeout=120) logger.info("HTTP instance started") requests.post(ONELINER_URL, data={"message": ONELINER_ENTRY}) r = requests.get(ONELINER_URL) assert r.status_code == 200 assert ONELINER_ENTRY in r.text logger.info("DB write confirmed :)") proc: asyncio.subprocess.Process = await process_monitor.get_process() proc.send_signal(signal.SIGINT) logger.info("Sent SIGINT...") for i in range(2): await cmd_monitor.wait_for_pattern(".*Service terminated.*", timeout=20) logger.info(f"The instances have been terminated ({elapsed_time()})") await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20) logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_run_ssh( log_dir: Path, project_dir: Path, config_overrides: List[Override], ssh_verify_connection: bool, ) -> None: websocat_check = pexpect.spawn("/usr/bin/which websocat") exit_code = websocat_check.wait() if exit_code != 0: raise ProcessLookupError( "websocat binary not found, please install it or check your PATH.") configure_logging(log_dir) # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml( Path(__file__).parent / "assets" / "goth-config.yml") requestor_path = project_dir / "examples" / "ssh" / "ssh.js" runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"node {requestor_path} --subnet-tag {SUBNET_TAG} --timeout 15", env=os.environ, ) as (_cmd_task, cmd_monitor, process_monitor): start_time = time.time() def elapsed_time(): return f"time: {(time.time() - start_time):.1f}" cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) await cmd_monitor.wait_for_pattern(".*Created network", timeout=20) logger.info(f"Network created") await cmd_monitor.wait_for_pattern(".*Agreement proposed ", timeout=20) logger.info("Agreement proposed") await cmd_monitor.wait_for_pattern(".*Agreement confirmed ", timeout=20) logger.info("Agreement confirmed") ssh_connections = [] # # A longer timeout to account for downloading a VM image for i in range(2): ssh_string = await cmd_monitor.wait_for_pattern( "ssh -o ProxyCommand", timeout=120) matches = re.match("ssh -o ProxyCommand=('.*') (root@.*)", ssh_string) # the default API port goes through a proxy that logs REST requests # but does not support websocket connections # hence, we're replacing it with a port that connects directly # to the daemon's port in the requestor's Docker container proxy_cmd = re.sub(":16(\\d\\d\\d)", ":6\\1", matches.group(1)) auth_str = matches.group(2) password = re.sub( "Password: "******"", await cmd_monitor.wait_for_pattern("Password:"******"Skipping SSH connection check. Use `--ssh-verify-connection` to perform it." ) else: for proxy_cmd, auth_str, password in ssh_connections: args = [ "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "ProxyCommand=" + proxy_cmd, auth_str, "uname -v", ] logger.debug("running ssh with: %s", args) ssh = pexpect.spawn(" ".join(args)) ssh.expect("[pP]assword:", timeout=5) ssh.sendline(password) ssh.expect("#1-Alpine SMP", timeout=5) ssh.expect(pexpect.EOF, timeout=5) logger.info("Connection to %s confirmed.", auth_str) logger.info("SSH connections confirmed.") for _ in range(2): await cmd_monitor.wait_for_pattern("Task .* completed", timeout=20) await cmd_monitor.wait_for_pattern(".*Computation finished", timeout=20) await cmd_monitor.wait_for_pattern(".*Removed network", timeout=20) logger.info(f"Network removed") await cmd_monitor.wait_for_pattern(".*Executor has shut down", timeout=20) logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_run_ssh( log_dir: Path, project_dir: Path, goth_config_path: Path, config_overrides: List[Override], ssh_verify_connection: bool, ) -> None: if ssh_verify_connection: websocat_check = pexpect.spawn("/usr/bin/which websocat") exit_code = websocat_check.wait() if exit_code != 0: raise ProcessLookupError( "websocat binary not found, please install it or check your PATH. " "You may also skip the connection check by omitting `--ssh-verify-connection`." ) configure_logging(log_dir) # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) # disable the mitm proxy used to capture the requestor agent -> daemon API calls # because it doesn't support websockets which are neeeded to enable VPN connections requestor = [c for c in goth_config.containers if c.name == "requestor"][0] requestor.use_proxy = False requestor_path = project_dir / "examples" / "ssh" / "ssh.py" runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( f"{requestor_path} --subnet-tag {SUBNET_TAG}", env=os.environ, ) as (_cmd_task, cmd_monitor, process_monitor): start_time = time.time() def elapsed_time(): return f"time: {(time.time() - start_time):.1f}" cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) ssh_connections = [] # A longer timeout to account for downloading a VM image for i in range(2): ssh_string = await cmd_monitor.wait_for_pattern( "ssh -o ProxyCommand", timeout=120) matches = re.match("ssh -o ProxyCommand=('.*') (root@.*)", ssh_string) proxy_cmd = matches.group(1) auth_str = matches.group(2) password = re.sub( "password: "******"", await cmd_monitor.wait_for_pattern("password:"******".*SshService running on provider.*SshService running on provider", timeout=10) if not ssh_verify_connection: logger.warning( "Skipping SSH connection check. Use `--ssh-verify-connection` to perform it." ) else: for proxy_cmd, auth_str, password in ssh_connections: args = [ "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "ProxyCommand=" + proxy_cmd, auth_str, "uname -v", ] logger.debug("running ssh with: %s", args) ssh = pexpect.spawn(" ".join(args)) ssh.expect("[pP]assword:", timeout=5) ssh.sendline(password) ssh.expect("#1-Alpine SMP", timeout=5) ssh.expect(pexpect.EOF, timeout=5) logger.info("Connection to %s confirmed.", auth_str) logger.info("SSH connections confirmed.") proc: asyncio.subprocess.Process = await process_monitor.get_process( ) proc.send_signal(signal.SIGINT) logger.info("Sent SIGINT...") for _ in range(2): await cmd_monitor.wait_for_pattern( ".*SshService terminated on provider", timeout=120) logger.info( f"The instances have been terminated ({elapsed_time()})") await cmd_monitor.wait_for_pattern(".*All jobs have finished", timeout=20) logger.info(f"Requestor script finished ({elapsed_time()})")
async def test_instance_restart( log_dir: Path, goth_config_path: Path, config_overrides: List[goth.configuration.Override], ) -> None: """Run the `requestor.py` and make sure that it's standard output is as expected.""" configure_logging(log_dir) # Override the default test configuration to create only one provider node nodes = [ { "name": "requestor", "type": "Requestor" }, { "name": "provider-1", "type": "VM-Wasm-Provider", "use-proxy": True }, ] config_overrides.append(("nodes", nodes)) goth_config = goth.configuration.load_yaml(goth_config_path, config_overrides) runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] async with requestor.run_command_on_host( str(Path(__file__).parent / "requestor.py"), env=os.environ) as (_cmd_task, cmd_monitor, _process_monitor): cmd_monitor.add_assertion(count_instances) # The first attempt to create an instance should fail await cmd_monitor.wait_for_pattern("STARTING 1$", timeout=60) await cmd_monitor.wait_for_pattern(".*CommandExecutionError", timeout=20) # The second one should successfully start and fail in `running` state await cmd_monitor.wait_for_pattern("STARTING 2$", timeout=20) await cmd_monitor.wait_for_pattern("RUNNING 2$", timeout=20) await cmd_monitor.wait_for_pattern(".*CommandExecutionError", timeout=20) await cmd_monitor.wait_for_pattern("STOPPING 2$", timeout=20) # The third instance should be started, but not running await cmd_monitor.wait_for_pattern("STARTING 3$", timeout=20) await cmd_monitor.wait_for_pattern("Cluster stopped$", timeout=60) assert instances_started == { 1, 2, 3 }, ("Expected to see instances 1, 2, 3 starting, saw instances " f"{', '.join(str(n) for n in instances_started)} instead") assert instances_running == { 2 }, ("Expected to see only instance 2 running, saw instances " f"{', '.join(str(n) for n in instances_running)} instead")
async def test_demand_resubscription(log_dir: Path, goth_config_path: Path, monkeypatch) -> None: """Test that checks that a demand is re-submitted after its previous submission expires.""" configure_logging(log_dir) # Override the default test configuration to create only one provider node nodes = [ { "name": "requestor", "type": "Requestor" }, { "name": "provider-1", "type": "VM-Wasm-Provider", "use-proxy": True }, ] goth_config = load_yaml(goth_config_path, [("nodes", nodes)]) vm_package = await vm.repo( image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae", min_mem_gib=0.5, min_storage_gib=2.0, ) runner = Runner(base_log_dir=log_dir, compose_config=goth_config.compose_config) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] env = dict(os.environ) env.update(requestor.get_agent_env_vars()) # Setup the environment for the requestor for key, val in env.items(): monkeypatch.setenv(key, val) monitor = EventMonitor() monitor.add_assertion(assert_demand_resubscribed) monitor.start() # The requestor enable_default_logger() async def worker(work_ctx, tasks): async for task in tasks: script = work_ctx.new_script() script.run("/bin/sleep", "5") yield script task.accept_result() async with Golem( budget=10.0, event_consumer=monitor.add_event_sync, ) as golem: task: Task # mypy needs this for some reason async for task in golem.execute_tasks( worker, [Task(data=n) for n in range(20)], vm_package, max_workers=1, timeout=timedelta(seconds=30), ): logger.info("Task %d computed", task.data) await monitor.stop() for a in monitor.failed: raise a.result()
async def test_run_yacat( log_dir: Path, project_dir: Path, goth_config_path: Path, config_overrides: List[Override], ) -> None: configure_logging(log_dir) # This is the default configuration with 2 wasm/VM providers goth_config = load_yaml(goth_config_path, config_overrides) yacat_path = project_dir / "examples" / "yacat" / "yacat.py" runner = Runner( base_log_dir=log_dir, compose_config=goth_config.compose_config, ) async with runner(goth_config.containers): requestor = runner.get_probes(probe_type=RequestorProbe)[0] logfile = f"hashcat-yapapi-{datetime.now().strftime('%Y-%m-%d_%H.%M.%S')}.log" async with requestor.run_command_on_host( f"{yacat_path} --mask ?a?a --hash $P$5ZDzPE45CigTC6EY4cXbyJSLj/pGee0 " f"--subnet-tag goth --chunk-size {CHUNK_SIZE} --max-workers {PROVIDER_COUNT} " f"--log-file {(log_dir / logfile).resolve()}", env=os.environ, ) as (_cmd_task, cmd_monitor, _process_monitor): # Add assertions to the command output monitor `cmd_monitor`: cmd_monitor.add_assertion(assert_no_errors) cmd_monitor.add_assertion(assert_all_invoices_accepted) all_sent = cmd_monitor.add_assertion(assert_all_tasks_started) all_computed = cmd_monitor.add_assertion(assert_all_tasks_computed) await cmd_monitor.wait_for_pattern(".*Received proposals", timeout=30) logger.info("Received proposals") await cmd_monitor.wait_for_pattern( f".*The keyspace size is {EXPECTED_KEYSPACE_SIZE}", timeout=120) logger.info("Keyspace found") await all_sent.wait_for_result(timeout=60) logger.info("All tasks sent") await all_computed.wait_for_result(timeout=120) logger.info("All tasks computed") await cmd_monitor.wait_for_pattern(".*Password found: yo", timeout=60) logger.info("Password found, waiting for Golem shutdown") await cmd_monitor.wait_for_pattern( f".*{SummaryLogger.GOLEM_SHUTDOWN_SUCCESSFUL_MESSAGE}", timeout=120) logger.info("Requestor script finished")