async def main(subnet_tag: str):
package = await vm.repo(
image_hash="7c63ebd87868e27eb99a687d1175e77450d7b3ff73fc86e6bdcef37b",
min_mem_gib=4.0,
min_storage_gib=8.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.run("/golem/entrypoints/runold.sh")
output_file = "output.txt"
ctx.download_file("/golem/output/output.txt", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=18000))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(f"{utils.TEXT_COLOR_RED}"
f"Task timed out: {task}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
raise
# Iterator over the frame indices that we want to render
frames: range = range(0, 1)
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
# TODO: make this dynamic, e.g. depending on the size of files to transfer
init_overhead = 3
# Providers will not accept work if the timeout is outside of the [5 min, 30min] range.
# We increase the lower bound to 6 min to account for the time needed for our demand to
# reach the providers.
min_timeout, max_timeout = 30, 150
timeout = timedelta(
minutes=max(min(init_overhead +
len(frames) * 2, max_timeout), min_timeout))
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=3,
budget=10.0,
timeout=timeout,
subnet_tag=subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executor:
async for task in executor.submit(
worker, [Task(data=frame) for frame in frames]):
print(
f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def main(subnet_tag: str, app: App, batches: List[Batch]):
package = await vm.repo(
# using existing image for 'blender' example
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=1.0,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
cwd = pathlib.Path.cwd()
async for task in tasks:
batch = task.data
try:
os.remove(batch.local_out(cwd))
except:
pass
for exe in batch.all_executables():
ctx.send_file(batch.local_exe(cwd, exe), batch.remote_exe(exe))
batch.make_local_sh(cwd)
ctx.send_file(batch.local_sh(cwd), batch.remote_sh())
ctx.run("/bin/bash", batch.remote_sh())
ctx.download_file(batch.remote_out(), batch.local_out(cwd))
try:
yield ctx.commit(timeout=timedelta(seconds=batch.timeout()))
task.accept_result(result=batch.local_out(cwd))
except BatchTimeoutError:
print(
f"{text_colors.RED}"
f"Task timed out: {task.data.id}, time: {task.running_time}"
f"{text_colors.DEFAULT}")
raise
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
timeout = timedelta(minutes=app.args.time_out)
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=len(batches),
budget=10.0,
timeout=timeout,
subnet_tag=subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executor:
async for task in executor.submit(
worker, [Task(data=batch) for batch in batches]):
print(
f"{text_colors.CYAN}"
f"Task computed: {task.data.id}, result: {task.result}, time: {task.running_time}"
f"{text_colors.DEFAULT}")
async def run_step(self, step, timeout=timedelta(minutes=10), budget=10, subnet_tag="community.3" ):
package = await vm.repo(
image_hash=step["image"],
min_mem_gib=1,
min_storage_gib=5.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
step_name = step['name']
commands = step['commands']
# prepair envs in string form of: "k1=v1 k2=v2 ... kn=vn "
envs = step.get('environment')
print(f"\033[36;1mSending the context zip file: {self.tar_fname}\033[0m")
ctx.send_file(self.tar_fname , "/golem/resource/context.zip")
# extracting tar file.
print(f"\033[36;1mExtracting the zip file: {self.tar_fname}\033[0m")
ctx.run("/bin/sh", "-c", "unzip /golem/resource/context.zip")
# run all commands one by one
for command in commands:
print(f"\033[36;1mRunning {command}\033[0m")
# set envs.
ctx.run("/bin/sh", "-c", f"{command} >> /golem/output/cmd.log 2>&1", env=envs)
log_fname = get_temp_log_file(step_name)
ctx.download_file(f"/golem/output/cmd.log", log_fname)
try:
yield ctx.commit(timeout=timedelta(minutes=30))
task.accept_result(result=log_fname)
except BatchTimeoutError:
print(f"Task timed out: {task}, time: {task.running_time}")
raise
ctx.log("no more task to run")
# By passing `event_emitter=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=1,
budget=budget,
timeout=timeout,
subnet_tag=subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executer:
async for task in executer.submit(worker, [Task(data=step)]):
print(f"\033[36;1mStep completed: {task}\033[0m")
# grab the logs
self.state[step['name']]['log'] = task.result
# notify about this task!
self.state[step['name']]['state'] = StepState.SUCCESS
self.post_progress(step['name'])
async def main(args):
package = await vm.repo(
image_hash="2c17589f1651baff9b82aa431850e296455777be265c2c5446c902e9",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker_check_keyspace(ctx: WorkContext, tasks):
async for task in tasks:
keyspace_sh_filename = "keyspace.sh"
ctx.send_file(keyspace_sh_filename, "/golem/work/keyspace.sh")
ctx.run("/bin/sh", "/golem/work/keyspace.sh")
output_file = "keyspace.txt"
ctx.download_file("/golem/work/keyspace.txt", output_file)
yield ctx.commit()
task.accept_result()
async def worker_find_password(ctx: WorkContext, tasks):
ctx.send_file("in.hash", "/golem/work/in.hash")
async for task in tasks:
skip = task.data
limit = skip + step
# Commands to be run on the provider
commands = (
"rm -f /golem/work/*.potfile ~/.hashcat/hashcat.potfile; "
f"touch /golem/work/hashcat_{skip}.potfile; "
f"hashcat -a 3 -m 400 /golem/work/in.hash {args.mask} --skip={skip} --limit={limit} --self-test-disable -o /golem/work/hashcat_{skip}.potfile || true"
)
ctx.run(f"/bin/sh", "-c", commands)
output_file = f"hashcat_{skip}.potfile"
ctx.download_file(f"/golem/work/hashcat_{skip}.potfile",
output_file)
yield ctx.commit()
task.accept_result(result=output_file)
# beginning of the main flow
write_hash(args.hash)
write_keyspace_check_script(args.mask)
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=args.number_of_providers,
budget=10.0,
# timeout should be keyspace / number of providers dependent
timeout=timedelta(minutes=25),
subnet_tag=args.subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executor:
keyspace_computed = False
# This is not a typical use of executor.submit as there is only one task, with no data:
async for _task in executor.submit(worker_check_keyspace,
[Task(data=None)]):
keyspace_computed = True
if not keyspace_computed:
# Assume the errors have been already reported and we may return quietly.
return
keyspace = read_keyspace()
print(
f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: keyspace size count. The keyspace size is {keyspace}"
f"{utils.TEXT_COLOR_DEFAULT}")
step = int(keyspace / args.number_of_providers) + 1
ranges = range(0, keyspace, step)
async for task in executor.submit(
worker_find_password, [Task(data=range) for range in ranges]):
print(f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}"
f"{utils.TEXT_COLOR_DEFAULT}")
password = read_password(ranges)
if password is None:
print(
f"{utils.TEXT_COLOR_RED}No password found{utils.TEXT_COLOR_DEFAULT}"
)
else:
print(f"{utils.TEXT_COLOR_GREEN}"
f"Password found: {password}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def main():
package = await vm.repo(
image_hash="c0317d4db8930afde1862f27973ee2f5b766c4d50a87409406e2e23f",
min_mem_gib=2,
min_storage_gib=2.5,
)
async def worker_train_model(ctx: WorkContext, tasks):
async for task in tasks:
global_round = task.data['global_round']
node_id = task.data['node_id']
model_path = os.path.join(ROUND_WEIGHTS_FOLDER,
f'round_{global_round - 1}.h5')
ctx.send_file(model_path,
f"/golem/work/model_{global_round - 1}.h5")
specs = {
'start': task.data['start'],
'end': task.data['end'],
'batch_size': BATCH_SIZE,
'model_path': f'model_{global_round - 1}.h5',
'epochs': PROVIDER_EPOCHS,
'global_round': task.data['global_round'],
'node_number': task.data['node_id']
}
ctx.send_json(
"/golem/work/specs.json",
specs,
)
ctx.send_file('client.py', "/golem/work/client.py")
ctx.run("/bin/sh", "-c", "python3 client.py")
node_model_output = f'/golem/output/model_round_{global_round}_{node_id}.h5'
node_log_file = f'/golem/output/log_round_{global_round}_{node_id}.json'
ctx.download_file(
node_model_output,
os.path.join(WORKER_MODEL_WEIGHTS_FOLDER,
f'round_{global_round}_worker_{node_id}.h5'))
ctx.download_file(
node_log_file,
os.path.join(
WORKER_LOGS_FOLDER,
f'log_round_{global_round}_worker_{node_id}.json'))
yield ctx.commit(timeout=timedelta(minutes=7))
task.accept_result()
print(f"{TEXT_COLOR_GREEN}"
f"Initialising your model."
f"{TEXT_COLOR_DEFAULT}")
model = get_compiled_model()
print(f"{TEXT_COLOR_GREEN}" f"Loading the data" f"{TEXT_COLOR_DEFAULT}")
training_dataset, testing_dataset, train_length, test_length = load_dataset(
BATCH_SIZE)
print(f"{TEXT_COLOR_GREEN}"
f"Initial model evaluation - "
f"{TEXT_COLOR_DEFAULT}")
eval_results = model.evaluate(testing_dataset)
print(f"{TEXT_COLOR_BLUE}"
f"ROUND 0 | Loss: {eval_results[0]} | Accuracy: {eval_results[1]}"
f"{TEXT_COLOR_DEFAULT}")
print(f"{TEXT_COLOR_MAGENTA}"
f"Saving Model Weights for round 0"
f"{TEXT_COLOR_DEFAULT}")
model.save(os.path.join(ROUND_WEIGHTS_FOLDER, 'round_0.h5'))
for global_round_number in range(1, GLOBAL_TRAINING_ROUNDS + 1):
print(f"{TEXT_COLOR_GREEN}"
f"Beginning Training Round {global_round_number}"
f"{TEXT_COLOR_DEFAULT}")
async with Executor(
package=package,
max_workers=NUM_PROVIDERS,
budget=20.0,
timeout=timedelta(minutes=29),
subnet_tag=SUBNET_TAG,
event_consumer=log_summary(log_event_repr),
) as executor:
# No problem if we miss a few samples
training_subset_steps = int(train_length / NUM_PROVIDERS)
executor_tasks = [
Task(
data={
'start': x,
'end': x + training_subset_steps,
'global_round': global_round_number,
'node_id': index + 1
}) for index, x in enumerate(
list(range(0, train_length, training_subset_steps)))
]
async for task in executor.submit(worker_train_model,
executor_tasks):
print(
f"{TEXT_COLOR_CYAN}"
f"Training round {global_round_number} completed on provider node {task.data['node_id']}"
f"{TEXT_COLOR_DEFAULT}")
all_worker_weights = get_client_model_weights(
WORKER_MODEL_WEIGHTS_FOLDER, global_round_number)
averaged_weights = federated_avg_weights(all_worker_weights)
model.set_weights(averaged_weights)
print(f"{TEXT_COLOR_GREEN}"
f"TRAINING ROUND {global_round_number} complete!"
f"{TEXT_COLOR_DEFAULT}")
eval_results = model.evaluate(testing_dataset)
print(
f"{TEXT_COLOR_BLUE}"
f"ROUND {global_round_number} | Loss: {eval_results[0]} | Accuracy: {eval_results[1]}"
f"{TEXT_COLOR_DEFAULT}")
print(f"{TEXT_COLOR_MAGENTA}"
f"Saving Model Weights for round {global_round_number}"
f"{TEXT_COLOR_DEFAULT}")
model.save(
os.path.join(ROUND_WEIGHTS_FOLDER,
f'round_{global_round_number}.h5'))
async def main(subnet_tag: str):
package = await vm.repo(
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
scene_path = str(script_dir / "cubes.blend")
ctx.send_file(scene_path, "/golem/resource/scene.blend")
async for task in tasks:
frame = task.data
crops = [{
"outfilebasename": "out",
"borders_x": [0.0, 1.0],
"borders_y": [0.0, 1.0]
}]
ctx.send_json(
"/golem/work/params.json",
{
"scene_file": "/golem/resource/scene.blend",
"resolution": (400, 300),
"use_compositing": False,
"crops": crops,
"samples": 100,
"frames": [frame],
"output_format": "PNG",
"RESOURCES_DIR": "/golem/resources",
"WORK_DIR": "/golem/work",
"OUTPUT_DIR": "/golem/output",
},
)
ctx.run("/golem/entrypoints/run-blender.sh")
output_file = f"output_{frame}.png"
ctx.download_file(f"/golem/output/out{frame:04d}.png", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=120))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(f"{utils.TEXT_COLOR_RED}"
f"Task timed out: {task}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
raise
# Iterator over the frame indices that we want to render
frames: range = range(0, 60, 10)
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
# TODO: make this dynamic, e.g. depending on the size of files to transfer
init_overhead = 3
# Providers will not accept work if the timeout is outside of the [5 min, 30min] range.
# We increase the lower bound to 6 min to account for the time needed for our demand to
# reach the providers.
min_timeout, max_timeout = 6, 30
timeout = timedelta(
minutes=max(min(init_overhead +
len(frames) * 2, max_timeout), min_timeout))
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=3,
budget=10.0,
timeout=timeout,
subnet_tag=subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executor:
async for task in executor.submit(
worker, [Task(data=frame) for frame in frames]):
print(
f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def main(subnet_tag, driver=None, network=None):
package = await vm.repo(
image_hash="2c5d9a80847eb147261c4e33df6e6955666ddd932ec40fd8b005f799",
min_mem_gib=1,
min_storage_gib=10.0,
)
async def worker(ctx: WorkContext, tasks):
script_dir = pathlib.Path(__file__).resolve().parent
async for task in tasks:
frame = task.data
ctx.run("/bin/sh", "-c", "./generate_data.sh")
output_file = f"output/{frame}.zip"
ctx.download_file(f"/golem/output/data.zip", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=1750))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(
f"{TEXT_COLOR_RED}"
f"Task {task} timed out on {ctx.provider_name}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}")
raise
# Iterator over the frame indices that we want to render
frames: range = range(
0,
60,
)
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
# TODO: make this dynamic, e.g. depending on the size of files to transfer
init_overhead = 3
# Providers will not accept work if the timeout is outside of the [5 min, 30min] range.
# We increase the lower bound to 6 min to account for the time needed for our demand to
# reach the providers.
min_timeout, max_timeout = 6, 30
timeout = timedelta(
minutes=max(min(init_overhead +
len(frames) * 2, max_timeout), min_timeout))
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=30,
budget=10.0,
timeout=timeout,
subnet_tag=subnet_tag,
driver=driver,
network=network,
event_consumer=log_summary(log_event_repr),
) as executor:
sys.stderr.write(
f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n"
f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, "
f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, "
f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n"
)
num_tasks = 0
start_time = datetime.now()
async for task in executor.submit(
worker, [Task(data=frame) for frame in frames]):
num_tasks += 1
print(
f"{TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}")
print(
f"{TEXT_COLOR_CYAN}"
f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}"
f"{TEXT_COLOR_DEFAULT}")