async def main(
scan_size: int, max_workers: int, subnet_tag, payment_driver=None, payment_network=None
):
payload = await vm.repo(image_hash=IMAGE_HASH)
async def worker(ctx: WorkContext, tasks):
assert ctx.provider_id not in scanned_nodes
async for task in tasks:
print(
f"{TEXT_COLOR_CYAN}"
f"Getting info for {ctx.provider_id} (aka {ctx.provider_name})"
f"{TEXT_COLOR_DEFAULT}",
)
script = ctx.new_script()
future_result = script.run("/bin/cat", "/proc/cpuinfo")
yield script
result = (await future_result).stdout or ""
cpu_model_match = re.search("^model name\\s+:\\s+(.*)$", result, flags=re.MULTILINE)
if cpu_model_match:
result = cpu_model_match.group(1)
else:
result = None
# add the node to the set so we don't end up signing another agreement with it
scanned_nodes.add(ctx.provider_id)
# and accept the result (pass the result to the loop in `main`)
task.accept_result((ctx.provider_id, ctx.provider_name, result))
# as we don't really want the engine to execute any more tasks on this node,
# we signal the parent generator to exit and through that
# also request termination of the worker and the agreement
#
# issuing a `break` here instead will usually not do what the user is expecting,
# as the parent generator would just exit cleanly without notifying the
# engine and there's nothing stopping the engine from re-launching the activity/worker
# on the same agreement
await tasks.aclose()
async with Golem(
budget=1,
strategy=ScanStrategy(),
subnet_tag=subnet_tag,
payment_driver=payment_driver,
payment_network=payment_network,
) as golem:
print_env_info(golem)
print(
f"{TEXT_COLOR_YELLOW}Scanning {pluralize(scan_size, 'node')}, "
f"using {pluralize(max_workers, 'concurrent worker')}.{TEXT_COLOR_DEFAULT}"
)
tasks: List[Task] = [Task(i) for i in range(scan_size)]
async for task in golem.execute_tasks(worker, tasks, payload, max_workers=max_workers):
print(f"{TEXT_COLOR_MAGENTA}{task.result}{TEXT_COLOR_DEFAULT}")
async def main(subnet_tag,
payment_driver=None,
payment_network=None,
num_instances=2):
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Golem(
budget=1.0,
subnet_tag=subnet_tag,
payment_driver=payment_driver,
payment_network=payment_network,
) as golem:
print_env_info(golem)
network = await golem.create_network("192.168.0.1/24")
async with network:
cluster = await golem.run_service(SshService,
network=network,
num_instances=num_instances)
instances = cluster.instances
while True:
print(instances)
try:
await asyncio.sleep(5)
except (KeyboardInterrupt, asyncio.CancelledError):
break
cluster.stop()
cnt = 0
while cnt < 3 and any(s.is_available for s in instances):
print(instances)
await asyncio.sleep(5)
cnt += 1
async def main(args):
package = await vm.repo(
image_hash="055911c811e56da4d75ffc928361a78ed13077933ffa8320fb1ec2db",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async with Golem(
budget=10.0,
subnet_tag=args.subnet_tag,
payment_driver=args.payment_driver,
payment_network=args.payment_network,
) as golem:
print_env_info(golem)
start_time = datetime.now()
completed = golem.execute_tasks(
compute_keyspace,
[Task(data="compute_keyspace")],
payload=package,
timeout=KEYSPACE_TIMEOUT_ALL_TASKS,
)
keyspace = 0
async for task in completed:
keyspace = task.result
print(
f"{TEXT_COLOR_CYAN}"
f"Task computed: keyspace size count. The keyspace size is {keyspace}"
f"{TEXT_COLOR_DEFAULT}"
)
data = (Task(data=c) for c in range(0, keyspace, args.chunk_size))
max_workers = args.max_workers or math.ceil(keyspace / args.chunk_size) // 2
completed = golem.execute_tasks(
perform_mask_attack,
data,
payload=package,
max_workers=max_workers,
timeout=MASK_ATTACK_TIMEOUT,
)
password = None
async for task in completed:
print(
f"{TEXT_COLOR_CYAN}Task computed: {task}, result: {task.result}{TEXT_COLOR_DEFAULT}"
)
result = _parse_result(task.result)
if result:
password = result
if password:
print(f"{TEXT_COLOR_GREEN}Password found: {password}{TEXT_COLOR_DEFAULT}")
else:
print(f"{TEXT_COLOR_RED}No password found{TEXT_COLOR_DEFAULT}")
print(f"{TEXT_COLOR_CYAN}Total time: {datetime.now() - start_time}{TEXT_COLOR_DEFAULT}")
async def main(subnet_tag, payment_driver, payment_network, port):
async with Golem(
budget=1.0,
subnet_tag=subnet_tag,
payment_driver=payment_driver,
payment_network=payment_network,
) as golem:
print_env_info(golem)
network = await golem.create_network("192.168.0.1/24")
async with network:
db_cluster = await golem.run_service(DbService, network=network)
db_instance = db_cluster.instances[0]
def still_starting(cluster):
return any(i.state in (ServiceState.pending,
ServiceState.starting)
for i in cluster.instances)
def raise_exception_if_still_starting(cluster):
if still_starting(cluster):
raise Exception(
f"Failed to start {cluster} instances "
f"after {STARTING_TIMEOUT.total_seconds()} seconds")
commissioning_time = datetime.now()
while (still_starting(db_cluster)
and datetime.now() < commissioning_time + STARTING_TIMEOUT):
print(db_cluster.instances)
await asyncio.sleep(5)
raise_exception_if_still_starting(db_cluster)
print(
f"{TEXT_COLOR_CYAN}DB instance started, spawning the web server{TEXT_COLOR_DEFAULT}"
)
web_cluster = await golem.run_service(
HttpService,
network=network,
instance_params=[{
"db_address": db_instance.network_node.ip
}],
)
# wait until all remote http instances are started
while (still_starting(web_cluster)
and datetime.now() < commissioning_time + STARTING_TIMEOUT):
print(web_cluster.instances + db_cluster.instances)
await asyncio.sleep(5)
raise_exception_if_still_starting(web_cluster)
# service instances started, start the local HTTP server
proxy = LocalHttpProxy(web_cluster, port)
await proxy.run()
print(
f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\nhttp://localhost:{port}{TEXT_COLOR_DEFAULT}"
)
# wait until Ctrl-C
while True:
print(web_cluster.instances + db_cluster.instances)
try:
await asyncio.sleep(10)
except (KeyboardInterrupt, asyncio.CancelledError):
break
# perform the shutdown of the local http server and the service cluster
await proxy.stop()
print(f"{TEXT_COLOR_CYAN}HTTP server stopped{TEXT_COLOR_DEFAULT}")
web_cluster.stop()
db_cluster.stop()
cnt = 0
while cnt < 3 and any(
s.is_available
for s in web_cluster.instances + db_cluster.instances):
print(web_cluster.instances + db_cluster.instances)
await asyncio.sleep(5)
cnt += 1
async def main(
subnet_tag,
running_time,
payment_driver=None,
payment_network=None,
num_instances=1,
show_usage=False,
):
async with Golem(
budget=1.0,
subnet_tag=subnet_tag,
payment_driver=payment_driver,
payment_network=payment_network,
) as golem:
print_env_info(golem)
commissioning_time = datetime.now()
# start the service
cluster = await golem.run_service(
SimpleService,
instance_params=[
{"instance_name": f"simple-service-{i+1}", "show_usage": show_usage}
for i in range(num_instances)
],
expiration=datetime.now(timezone.utc)
+ STARTING_TIMEOUT
+ EXPIRATION_MARGIN
+ timedelta(seconds=running_time),
)
print(f"{TEXT_COLOR_YELLOW}" f"Starting {cluster}..." f"{TEXT_COLOR_DEFAULT}")
def print_instances():
print(
f"instances: "
+ str(
[
f"{s.name}: {s.state.value}"
+ (f" on {s.provider_name}" if s.provider_id else "")
for s in cluster.instances
]
)
)
def still_starting():
return any(
i.state in (ServiceState.pending, ServiceState.starting) for i in cluster.instances
)
# wait until instances are started
while still_starting() and datetime.now() < commissioning_time + STARTING_TIMEOUT:
print_instances()
await asyncio.sleep(5)
if still_starting():
raise Exception(f"Failed to start instances before {STARTING_TIMEOUT} elapsed :( ...")
print(f"{TEXT_COLOR_YELLOW}All instances started :){TEXT_COLOR_DEFAULT}")
# allow the service to run for a short while
# (and allowing its requestor-end handlers to interact with it)
start_time = datetime.now()
while datetime.now() < start_time + timedelta(seconds=running_time):
print_instances()
await asyncio.sleep(5)
print(f"{TEXT_COLOR_YELLOW}Stopping {cluster}...{TEXT_COLOR_DEFAULT}")
cluster.stop()
# wait for instances to stop
cnt = 0
while cnt < 10 and any(s.is_available for s in cluster.instances):
print_instances()
await asyncio.sleep(5)
print_instances()
async def main(golem, show_usage, min_cpu_threads):
print_env_info(golem)
package = await vm.repo(
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
# only run on provider nodes that have more than 0.5gb of RAM available
min_mem_gib=0.5,
# only run on provider nodes that have more than 2gb of storage space available
min_storage_gib=2.0,
# only run on provider nodes which a certain number of CPU threads available
min_cpu_threads=min_cpu_threads,
)
async def worker(ctx: WorkContext, tasks):
script_dir = pathlib.Path(__file__).resolve().parent
scene_path = str(script_dir / "cubes.blend")
# Set timeout for the first script executed on the provider. Usually, 30 seconds
# should be more than enough for computing a single frame of the provided scene,
# however a provider may require more time for the first task if it needs to download
# the VM image first. Once downloaded, the VM image will be cached and other tasks that use
# that image will be computed faster.
script = ctx.new_script(timeout=timedelta(minutes=10))
script.upload_file(scene_path, "/golem/resource/scene.blend")
async for task in tasks:
frame = task.data
crops = [{"outfilebasename": "out", "borders_x": [0.0, 1.0], "borders_y": [0.0, 1.0]}]
script.upload_json(
{
"scene_file": "/golem/resource/scene.blend",
"resolution": (400, 300),
"use_compositing": False,
"crops": crops,
"samples": 100,
"frames": [frame],
"output_format": "PNG",
"RESOURCES_DIR": "/golem/resources",
"WORK_DIR": "/golem/work",
"OUTPUT_DIR": "/golem/output",
},
"/golem/work/params.json",
)
script.run("/golem/entrypoints/run-blender.sh")
output_file = f"output_{frame}.png"
script.download_file(f"/golem/output/out{frame:04d}.png", output_file)
try:
yield script
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(
f"{TEXT_COLOR_RED}"
f"Task {task} timed out on {ctx.provider_name}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}"
)
raise
# reinitialize the script which we send to the engine to compute subsequent frames
script = ctx.new_script(timeout=timedelta(minutes=1))
if show_usage:
raw_state = await ctx.get_raw_state()
usage = format_usage(await ctx.get_usage())
cost = await ctx.get_cost()
print(
f"{TEXT_COLOR_MAGENTA}"
f" --- {ctx.provider_name} STATE: {raw_state}\n"
f" --- {ctx.provider_name} USAGE: {usage}\n"
f" --- {ctx.provider_name} COST: {cost}"
f"{TEXT_COLOR_DEFAULT}"
)
# Iterator over the frame indices that we want to render
frames: range = range(0, 60, 10)
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
# TODO: make this dynamic, e.g. depending on the size of files to transfer
init_overhead = 3
# Providers will not accept work if the timeout is outside of the [5 min, 30min] range.
# We increase the lower bound to 6 min to account for the time needed for our demand to
# reach the providers.
min_timeout, max_timeout = 6, 30
timeout = timedelta(minutes=max(min(init_overhead + len(frames) * 2, max_timeout), min_timeout))
num_tasks = 0
start_time = datetime.now()
try:
await golem.start()
completed_tasks = golem.execute_tasks(
worker,
[Task(data=frame) for frame in frames],
payload=package,
max_workers=3,
timeout=timeout,
)
async for task in completed_tasks:
num_tasks += 1
print(
f"{TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}"
)
print(
f"{TEXT_COLOR_CYAN}"
f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}"
f"{TEXT_COLOR_DEFAULT}"
)
finally:
await golem.stop()
async def main(subnet_tag, payment_driver, payment_network, num_instances, port, running_time):
async with Golem(
budget=1.0,
subnet_tag=subnet_tag,
payment_driver=payment_driver,
payment_network=payment_network,
) as golem:
print_env_info(golem)
commissioning_time = datetime.now()
network = await golem.create_network("192.168.0.1/24")
cluster = await golem.run_service(
HttpService,
network=network,
num_instances=num_instances,
expiration=datetime.now(timezone.utc)
+ STARTING_TIMEOUT
+ EXPIRATION_MARGIN
+ timedelta(seconds=running_time),
)
instances = cluster.instances
def still_starting():
return any(i.state in (ServiceState.pending, ServiceState.starting) for i in instances)
# wait until all remote http instances are started
while still_starting() and datetime.now() < commissioning_time + STARTING_TIMEOUT:
print(f"instances: {instances}")
await asyncio.sleep(5)
if still_starting():
raise Exception(
f"Failed to start instances after {STARTING_TIMEOUT.total_seconds()} seconds"
)
# service instances started, start the local HTTP server
proxy = LocalHttpProxy(cluster, port)
await proxy.run()
print(
f"{TEXT_COLOR_CYAN}Local HTTP server listening on:\nhttp://localhost:{port}{TEXT_COLOR_DEFAULT}"
)
# wait until Ctrl-C
start_time = datetime.now()
while datetime.now() < start_time + timedelta(seconds=running_time):
print(instances)
try:
await asyncio.sleep(10)
except (KeyboardInterrupt, asyncio.CancelledError):
break
# perform the shutdown of the local http server and the service cluster
await proxy.stop()
print(f"{TEXT_COLOR_CYAN}HTTP server stopped{TEXT_COLOR_DEFAULT}")
cluster.stop()
cnt = 0
while cnt < 3 and any(s.is_available for s in instances):
print(instances)
await asyncio.sleep(5)
cnt += 1
await network.remove()