async def worker(ctx: WorkContext, tasks):
script_dir = pathlib.Path(__file__).resolve().parent
scene_path = str(script_dir / "cubes.blend")
# Set timeout for the first script executed on the provider. Usually, 30 seconds
# should be more than enough for computing a single frame of the provided scene,
# however a provider may require more time for the first task if it needs to download
# the VM image first. Once downloaded, the VM image will be cached and other tasks that use
# that image will be computed faster.
script = ctx.new_script(timeout=timedelta(minutes=10))
script.upload_file(scene_path, "/golem/resource/scene.blend")
async for task in tasks:
frame = task.data
crops = [{"outfilebasename": "out", "borders_x": [0.0, 1.0], "borders_y": [0.0, 1.0]}]
script.upload_json(
{
"scene_file": "/golem/resource/scene.blend",
"resolution": (400, 300),
"use_compositing": False,
"crops": crops,
"samples": 100,
"frames": [frame],
"output_format": "PNG",
"RESOURCES_DIR": "/golem/resources",
"WORK_DIR": "/golem/work",
"OUTPUT_DIR": "/golem/output",
},
"/golem/work/params.json",
)
script.run("/golem/entrypoints/run-blender.sh")
output_file = f"output_{frame}.png"
script.download_file(f"/golem/output/out{frame:04d}.png", output_file)
try:
yield script
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(
f"{TEXT_COLOR_RED}"
f"Task {task} timed out on {ctx.provider_name}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}"
)
raise
# reinitialize the script which we send to the engine to compute subsequent frames
script = ctx.new_script(timeout=timedelta(minutes=1))
if show_usage:
raw_state = await ctx.get_raw_state()
usage = format_usage(await ctx.get_usage())
cost = await ctx.get_cost()
print(
f"{TEXT_COLOR_MAGENTA}"
f" --- {ctx.provider_name} STATE: {raw_state}\n"
f" --- {ctx.provider_name} USAGE: {usage}\n"
f" --- {ctx.provider_name} COST: {cost}"
f"{TEXT_COLOR_DEFAULT}"
)
async def perform_mask_attack(ctx: WorkContext, tasks: AsyncIterable[Task]):
"""Worker script which performs a hashcat mask attack against a target hash.
This function is used as the `worker` parameter to `Golem#execute_tasks`.
It represents a sequence of commands to be executed on a remote provider node.
"""
async for task in tasks:
skip = task.data
limit = skip + args.chunk_size
output_name = f"yacat_{skip}.potfile"
worker_output_path = f"/golem/output/{output_name}"
script = ctx.new_script(timeout=MASK_ATTACK_TIMEOUT)
script.run(f"/bin/sh", "-c", _make_attack_command(skip, limit, worker_output_path))
try:
output_file = Path(gettempdir()) / output_name
script.download_file(worker_output_path, str(output_file))
yield script
with output_file.open() as f:
result = f.readline()
task.accept_result(result)
finally:
output_file.unlink()
async def worker(ctx: WorkContext, tasks):
"""A worker function for `Golem.execute_tasks()`.
The first call to this function will produce a worker
that sends an invalid `run` command to the provider.
This should cause `yield script` to fail with
`CommandExecutionError`.
The remaining calls will just send `sleep 5` to the
provider to simulate some work.
"""
nonlocal first_worker
should_fail = first_worker
first_worker = False
async for task in tasks:
script = ctx.new_script()
if should_fail:
# Send a command that will fail on the provider
script.run("xyz")
yield script
else:
# Simulate some work
script.run("/bin/sleep", "5")
yield script
task.accept_result()
async def worker(context: WorkContext, tasks: AsyncIterable[Task]):
async for task in tasks:
script = context.new_script()
future_result = script.run("/bin/sh", "-c", "date")
yield script
task.accept_result(result=await future_result)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.run("/golem/entrypoints/runold.sh")
output_file = "output.txt"
ctx.download_file("/golem/output/output.txt", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=18000))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(f"{utils.TEXT_COLOR_RED}"
f"Task timed out: {task}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
raise
async def worker_check_keyspace(ctx: WorkContext, tasks):
async for task in tasks:
keyspace_sh_filename = "keyspace.sh"
ctx.send_file(keyspace_sh_filename, "/golem/work/keyspace.sh")
ctx.run("/bin/sh", "/golem/work/keyspace.sh")
output_file = "keyspace.txt"
ctx.download_file("/golem/work/keyspace.txt", output_file)
yield ctx.commit()
task.accept_result()
async def worker(ctx: WorkContext, tasks):
script_dir = pathlib.Path(__file__).resolve().parent
async for task in tasks:
frame = task.data
ctx.run("/bin/sh", "-c", "./generate_data.sh")
output_file = f"output/{frame}.zip"
ctx.download_file(f"/golem/output/data.zip", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=1750))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(
f"{TEXT_COLOR_RED}"
f"Task {task} timed out on {ctx.provider_name}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}")
raise
async def worker(ctx: WorkContext, tasks):
scene_path = str(script_dir / "cubes.blend")
ctx.send_file(scene_path, "/golem/resource/scene.blend")
async for task in tasks:
frame = task.data
crops = [{
"outfilebasename": "out",
"borders_x": [0.0, 1.0],
"borders_y": [0.0, 1.0]
}]
ctx.send_json(
"/golem/work/params.json",
{
"scene_file": "/golem/resource/scene.blend",
"resolution": (400, 300),
"use_compositing": False,
"crops": crops,
"samples": 100,
"frames": [frame],
"output_format": "PNG",
"RESOURCES_DIR": "/golem/resources",
"WORK_DIR": "/golem/work",
"OUTPUT_DIR": "/golem/output",
},
)
ctx.run("/golem/entrypoints/run-blender.sh")
output_file = f"output_{frame}.png"
ctx.download_file(f"/golem/output/out{frame:04d}.png", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=120))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(f"{utils.TEXT_COLOR_RED}"
f"Task timed out: {task}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
raise
async def worker(ctx: WorkContext, tasks):
cwd = pathlib.Path.cwd()
async for task in tasks:
batch = task.data
try:
os.remove(batch.local_out(cwd))
except:
pass
for exe in batch.all_executables():
ctx.send_file(batch.local_exe(cwd, exe), batch.remote_exe(exe))
batch.make_local_sh(cwd)
ctx.send_file(batch.local_sh(cwd), batch.remote_sh())
ctx.run("/bin/bash", batch.remote_sh())
ctx.download_file(batch.remote_out(), batch.local_out(cwd))
try:
yield ctx.commit(timeout=timedelta(seconds=batch.timeout()))
task.accept_result(result=batch.local_out(cwd))
except BatchTimeoutError:
print(
f"{text_colors.RED}"
f"Task timed out: {task.data.id}, time: {task.running_time}"
f"{text_colors.DEFAULT}")
raise
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
output_file = f"output_{datetime.now()}_{random.random()}.txt"
script = ctx.new_script(timeout=timedelta(minutes=10))
script.run("/usr/bin/stress-ng", "--cpu", "1", "--timeout", "1")
script.run("/golem/task.sh", "-o", "1024", "-t", "5")
script.run("/golem/task.sh", "-f",
"/golem/output/output.txt,1048576")
script.download_file(f"/golem/output/output.txt", output_file)
script.run("/golem/task.sh", "-e", "1024", "-t", "5")
try:
yield script
task.accept_result(result=output_file)
except BatchTimeoutError:
print(
f"{TEXT_COLOR_RED}"
f"Task {task} timed out on {ctx.provider_name}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}")
raise
async def worker_find_password(ctx: WorkContext, tasks):
ctx.send_file("in.hash", "/golem/work/in.hash")
async for task in tasks:
skip = task.data
limit = skip + step
# Commands to be run on the provider
commands = (
"rm -f /golem/work/*.potfile ~/.hashcat/hashcat.potfile; "
f"touch /golem/work/hashcat_{skip}.potfile; "
f"hashcat -a 3 -m 400 /golem/work/in.hash {args.mask} --skip={skip} --limit={limit} --self-test-disable -o /golem/work/hashcat_{skip}.potfile || true"
)
ctx.run(f"/bin/sh", "-c", commands)
output_file = f"hashcat_{skip}.potfile"
ctx.download_file(f"/golem/work/hashcat_{skip}.potfile",
output_file)
yield ctx.commit()
task.accept_result(result=output_file)
async def compute_keyspace(context: WorkContext, tasks: AsyncIterable[Task]):
"""Worker script which computes the size of the keyspace for the mask attack.
This function is used as the `worker` parameter to `Golem#execute_tasks`.
It represents a sequence of commands to be executed on a remote provider node.
"""
async for task in tasks:
cmd = f"hashcat --keyspace " f"-a {HASHCAT_ATTACK_MODE} -m {args.hash_type} {args.mask}"
s = context.new_script(timeout=KEYSPACE_TIMEOUT)
s.run("/bin/bash", "-c", cmd)
try:
future_result = yield s
# each item is the result of a single command on the provider (including setup commands)
result: List[CommandExecuted] = await future_result
# we take the last item since it's the last command that was executed on the provider
cmd_result: CommandExecuted = result[-1]
keyspace = int(cmd_result.stdout)
task.accept_result(result=keyspace)
except CommandExecutionError as e:
raise RuntimeError(f"Failed to compute attack keyspace: {e}")
async def worker(ctx: WorkContext, tasks):
assert ctx.provider_id not in scanned_nodes
async for task in tasks:
print(
f"{TEXT_COLOR_CYAN}"
f"Getting info for {ctx.provider_id} (aka {ctx.provider_name})"
f"{TEXT_COLOR_DEFAULT}",
)
script = ctx.new_script()
future_result = script.run("/bin/cat", "/proc/cpuinfo")
yield script
result = (await future_result).stdout or ""
cpu_model_match = re.search("^model name\\s+:\\s+(.*)$", result, flags=re.MULTILINE)
if cpu_model_match:
result = cpu_model_match.group(1)
else:
result = None
# add the node to the set so we don't end up signing another agreement with it
scanned_nodes.add(ctx.provider_id)
# and accept the result (pass the result to the loop in `main`)
task.accept_result((ctx.provider_id, ctx.provider_name, result))
# as we don't really want the engine to execute any more tasks on this node,
# we signal the parent generator to exit and through that
# also request termination of the worker and the agreement
#
# issuing a `break` here instead will usually not do what the user is expecting,
# as the parent generator would just exit cleanly without notifying the
# engine and there's nothing stopping the engine from re-launching the activity/worker
# on the same agreement
await tasks.aclose()
async def worker_train_model(ctx: WorkContext, tasks):
async for task in tasks:
global_round = task.data['global_round']
node_id = task.data['node_id']
model_path = os.path.join(ROUND_WEIGHTS_FOLDER,
f'round_{global_round - 1}.h5')
ctx.send_file(model_path,
f"/golem/work/model_{global_round - 1}.h5")
specs = {
'start': task.data['start'],
'end': task.data['end'],
'batch_size': BATCH_SIZE,
'model_path': f'model_{global_round - 1}.h5',
'epochs': PROVIDER_EPOCHS,
'global_round': task.data['global_round'],
'node_number': task.data['node_id']
}
ctx.send_json(
"/golem/work/specs.json",
specs,
)
ctx.send_file('client.py', "/golem/work/client.py")
ctx.run("/bin/sh", "-c", "python3 client.py")
node_model_output = f'/golem/output/model_round_{global_round}_{node_id}.h5'
node_log_file = f'/golem/output/log_round_{global_round}_{node_id}.json'
ctx.download_file(
node_model_output,
os.path.join(WORKER_MODEL_WEIGHTS_FOLDER,
f'round_{global_round}_worker_{node_id}.h5'))
ctx.download_file(
node_log_file,
os.path.join(
WORKER_LOGS_FOLDER,
f'log_round_{global_round}_worker_{node_id}.json'))
yield ctx.commit(timeout=timedelta(minutes=7))
task.accept_result()
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
step_name = step['name']
commands = step['commands']
# prepair envs in string form of: "k1=v1 k2=v2 ... kn=vn "
envs = step.get('environment')
print(f"\033[36;1mSending the context zip file: {self.tar_fname}\033[0m")
ctx.send_file(self.tar_fname , "/golem/resource/context.zip")
# extracting tar file.
print(f"\033[36;1mExtracting the zip file: {self.tar_fname}\033[0m")
ctx.run("/bin/sh", "-c", "unzip /golem/resource/context.zip")
# run all commands one by one
for command in commands:
print(f"\033[36;1mRunning {command}\033[0m")
# set envs.
ctx.run("/bin/sh", "-c", f"{command} >> /golem/output/cmd.log 2>&1", env=envs)
log_fname = get_temp_log_file(step_name)
ctx.download_file(f"/golem/output/cmd.log", log_fname)
try:
yield ctx.commit(timeout=timedelta(minutes=30))
task.accept_result(result=log_fname)
except BatchTimeoutError:
print(f"Task timed out: {task}, time: {task.running_time}")
raise
ctx.log("no more task to run")