async def main(subnet_tag='testnet'):
package = await vm.repo(
image_hash='c33ebc0aee1109c6f9d1c02aa67b5800edd6ebe8be2bdc24b85bc4a8',
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.send_file('make.sh', '/golem/work/make.sh')
ctx.run('/bin/sh', '/golem/work/make.sh')
output_file = 'vim'
ctx.download_file('/golem/work/vim/src/vim', 'vim')
ctx.download_file('/golem/work/out.txt', 'log')
yield ctx.commit()
task.accept_task(result=output_file)
ctx.log(f'VIM compiled!')
tasks: range = range(0, 1, 1)
async with Engine(
package=package,
max_workers=1,
budget=10.0,
timeout=timedelta(minutes=10),
subnet_tag=subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
async for task in engine.map(worker, [Task(data=task) for task in tasks]):
print(f'\033[36;1mTask computed: {task}, result: {task.output}\033[0m')
async def main(subnet_tag='testnet'):
package = await vm.repo(
image_hash="746f7d86703c932d83809bf3ecc517450c5e23d693c9788e85b9682e",
min_mem_gib=1,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.send_file('fire.sh', '/golem/work/fire.sh')
ctx.run('/bin/sh', '/golem/work/fire.sh')
output_file = f"vim"
ctx.download_file('/golem/work/vim/src/vim', 'vim')
ctx.download_file('/golem/work/log.txt', 'log')
yield ctx.commit()
task.accept_task(result=output_file)
ctx.log(f'done!')
tasks: range = range(0, 1, 1)
async with Engine(
package=package,
max_workers=1,
budget=10.0,
timeout=timedelta(minutes=10),
subnet_tag=subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
async for task in engine.map(worker,
[Task(data=task) for task in tasks]):
print(
f'\033[36;1mTask computed: {task}, result: {task.output}\033[0m'
)
async def main(subnet_tag: str):
package = await vm.repo(
image_hash="7c63ebd87868e27eb99a687d1175e77450d7b3ff73fc86e6bdcef37b",
min_mem_gib=4.0,
min_storage_gib=8.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.run("/golem/entrypoints/runold.sh")
output_file = "output.txt"
ctx.download_file("/golem/output/output.txt", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=18000))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(f"{utils.TEXT_COLOR_RED}"
f"Task timed out: {task}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
raise
# Iterator over the frame indices that we want to render
frames: range = range(0, 1)
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
# TODO: make this dynamic, e.g. depending on the size of files to transfer
init_overhead = 3
# Providers will not accept work if the timeout is outside of the [5 min, 30min] range.
# We increase the lower bound to 6 min to account for the time needed for our demand to
# reach the providers.
min_timeout, max_timeout = 30, 150
timeout = timedelta(
minutes=max(min(init_overhead +
len(frames) * 2, max_timeout), min_timeout))
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=3,
budget=10.0,
timeout=timeout,
subnet_tag=subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executor:
async for task in executor.submit(
worker, [Task(data=frame) for frame in frames]):
print(
f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def main(duration,
time_delta,
num_trajectories=20,
num_nodes=4,
subnet_tag="devnet-alpha.2"):
package = await vm.repo(
image_hash="cea36374b451274ac584f747fcf876ac15865ee319ebb4e7eea94c23",
min_mem_gib=0.5,
min_storage_gib=5.0,
)
print(duration, time_delta, num_trajectories, num_nodes)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
node_no = task.data
ctx.send_file("lorenz.py", "/golem/work/task.py")
cmd = f"python3 /golem/work/task.py -d {duration} -n {node_no}/{num_nodes} -m {num_trajectories} -l {time_delta} "
print(f"\033[36;1mRunning {cmd}\033[0m")
ctx.run("sh", "-c", f"{cmd} >> /golem/output/log.txt 2>&1")
ctx.download_file(f"/golem/output/log.txt", "log.txt")
task_duration = duration / num_nodes
# frame start stop index
t0 = int(task_duration / time_delta) * (node_no - 1)
t1 = t0 + int(task_duration / time_delta)
for t in range(t0, t1):
ctx.download_file(f"/golem/output/frame_{t:04d}.png",
f"output/frame_{t:04d}.png")
yield ctx.commit()
task.accept_task()
ctx.log("no more task to run")
init_overhead: timedelta = timedelta(minutes=10)
# By passing `event_emitter=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Engine(
package=package,
max_workers=num_nodes,
budget=100.0,
timeout=init_overhead + timedelta(minutes=num_nodes * 2),
subnet_tag=subnet_tag,
event_emitter=log_summary(),
) as engine:
async for task in engine.map(
worker, [Task(data=i + 1) for i in range(num_nodes)]):
print(
f"\033[36;1mTask computed: {task}, result: {task.output}\033[0m"
)
print(f"\033[36;1mConverting ppng files to a gif animation!\033[0m")
i = "output/frame_%04d.png"
o = "output.gif"
subprocess.call(f"ffmpeg -i {i} {o}", shell=True)
async def main(subnet_tag: str, app: App, batches: List[Batch]):
package = await vm.repo(
# using existing image for 'blender' example
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=1.0,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
cwd = pathlib.Path.cwd()
async for task in tasks:
batch = task.data
try:
os.remove(batch.local_out(cwd))
except:
pass
for exe in batch.all_executables():
ctx.send_file(batch.local_exe(cwd, exe), batch.remote_exe(exe))
batch.make_local_sh(cwd)
ctx.send_file(batch.local_sh(cwd), batch.remote_sh())
ctx.run("/bin/bash", batch.remote_sh())
ctx.download_file(batch.remote_out(), batch.local_out(cwd))
try:
yield ctx.commit(timeout=timedelta(seconds=batch.timeout()))
task.accept_result(result=batch.local_out(cwd))
except BatchTimeoutError:
print(
f"{text_colors.RED}"
f"Task timed out: {task.data.id}, time: {task.running_time}"
f"{text_colors.DEFAULT}")
raise
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
timeout = timedelta(minutes=app.args.time_out)
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=len(batches),
budget=10.0,
timeout=timeout,
subnet_tag=subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executor:
async for task in executor.submit(
worker, [Task(data=batch) for batch in batches]):
print(
f"{text_colors.CYAN}"
f"Task computed: {task.data.id}, result: {task.result}, time: {task.running_time}"
f"{text_colors.DEFAULT}")
async def main(args):
package = await vm.repo(
image_hash="e34aaa2aac52cc98993b7d8fdbdcbafc9ba00aa2f38e7fa87796f7b7",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
(filename, offset, secondFrequency, smoothness) = task.data
full_filename = filename + ".png"
# Send these files so we don't have to update the Docker image
ctx.send_file('graphWavePair.py', F'{GOLEM_WORKDIR}graphWavePair.py')
ctx.run(F'chmod', 'u+x', F'{GOLEM_WORKDIR}graphWavePair.py')
ctx.run(F'/bin/sh', '-c', F'{GOLEM_WORKDIR}graphWavePair.py {filename} {offset} {secondFrequency} {smoothness}')
ctx.download_file(f'{GOLEM_WORKDIR+full_filename}', full_filename)
yield ctx.commit()
task.accept_task(result=full_filename)
async with Engine(
package=package,
max_workers=args.number_of_providers,
budget=10.0,
# timeout should be keyspace / number of providers dependent
timeout=timedelta(minutes=25),
subnet_tag=args.subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
# Prepare our parameters for 125 graphs
steps = np.arange(0.0, np.pi * 4, 0.1)
numSteps = len(steps)
inputs = []
for (count, step) in enumerate(steps):
filename = "graph-%04d" % (count,)
if (step < np.pi * 2):
distance = (count) / (numSteps / 2)
secondFrequency = interpolate(10, 16, distance)
smoothness = interpolate(10, 3, distance)
else:
distance = (count - numSteps / 2) / (numSteps / 2)
secondFrequency = interpolate(16, 10, distance)
smoothness = interpolate(3, 10, distance)
inputs.append((filename, step, secondFrequency, smoothness))
async for task in engine.map(worker, [Task(data=graphInput) for graphInput in inputs]):
print(
f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.output}"
f"{utils.TEXT_COLOR_DEFAULT}"
)
def __create_engine(self):
"""Creates yagna engine"""
return Engine(
package=self.package,
max_workers=self.max_workers,
budget=self.budget,
timeout=timedelta(minutes=25),
subnet_tag=self.subnet_tag,
# By passing `event_emitter=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
event_emitter=log_summary(log_event_repr),
)
async def main(args):
write_hash(args.hash)
package = await vm.repo(
image_hash='294668fd31f6535e65531fc4ea9d72e1377468f32ac8999f9cec89b3',
min_mem_gib=1,
min_storage_gib=2.0,
)
tasks: range = range(1, args.number_of_providers + 1)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.send_file('in.hash', '/golem/work/in.hash')
ctx.send_file('crack.sh', '/golem/work/crack.sh')
ctx.run('touch', '/golem/work/out.txt')
ctx.run('/bin/sh', '/golem/work/crack.sh', f'{task.data}', f'{args.number_of_providers}')
log_file = f'logs/log_{task.data}.txt'
ctx.download_file('/golem/work/log.txt', log_file)
output_file = f'outputs/out_{task.data}.txt'
ctx.download_file('/golem/work/out.txt', output_file)
yield ctx.commit()
task.accept_task(result=output_file)
password = None
async with Engine(
package=package,
max_workers=args.number_of_providers,
budget=10.0,
timeout=timedelta(minutes=10),
subnet_tag=args.subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
async for task in engine.map(worker, [Task(data=task) for task in tasks]):
print(f'\033[36;1mTask computed: {task}, result: {task.output}\033[0m')
password = read_password(task.data)
if password:
break
if password is None:
print(f"{utils.TEXT_COLOR_RED}No password found{utils.TEXT_COLOR_DEFAULT}")
else:
print(
f"{utils.TEXT_COLOR_GREEN}"
f"Password found: {password}"
f"{utils.TEXT_COLOR_DEFAULT}"
)
async def main(args):
package = await vm.repo(
image_hash='8af41741dc1a15fb85d519dce2b42e994ba2494acd4c71a96c5029d2',
min_mem_gib=1,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
output_file_name = f'{task.data}.png'
output_file_path = f'/golem/work/{output_file_name}'
ctx.send_file('plot.sh', '/golem/work/plot.sh')
command = (
'/bin/sh',
'/golem/work/plot.sh',
task.data,
data_file,
country_codes_file,
output_file_path,
args.parameter,
)
print(*command)
ctx.run(*command)
yield ctx.commit()
ctx.download_file(output_file_path, f'outputs/{output_file_name}')
task.accept_task(result=output_file_name)
async with Engine(
package=package,
max_workers=args.workers,
budget=10.0,
timeout=timedelta(minutes=10),
subnet_tag=args.subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
days = dates(args.start, args.end)
async for task in engine.map(worker,
[Task(data=date) for date in days]):
print(
f'\033[36;1mTask computed: {task}, result: {task.output}\033[0m'
)
gif_name = './covid.gif'
generate_gif(gif_name)
print(f'{utils.TEXT_COLOR_GREEN}'
f'gif generated: {gif_name}'
f'{utils.TEXT_COLOR_DEFAULT}')
async def run_step(self, step, timeout=timedelta(minutes=10), budget=10, subnet_tag="community.3" ):
package = await vm.repo(
image_hash=step["image"],
min_mem_gib=1,
min_storage_gib=5.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
step_name = step['name']
commands = step['commands']
# prepair envs in string form of: "k1=v1 k2=v2 ... kn=vn "
envs = step.get('environment')
print(f"\033[36;1mSending the context zip file: {self.tar_fname}\033[0m")
ctx.send_file(self.tar_fname , "/golem/resource/context.zip")
# extracting tar file.
print(f"\033[36;1mExtracting the zip file: {self.tar_fname}\033[0m")
ctx.run("/bin/sh", "-c", "unzip /golem/resource/context.zip")
# run all commands one by one
for command in commands:
print(f"\033[36;1mRunning {command}\033[0m")
# set envs.
ctx.run("/bin/sh", "-c", f"{command} >> /golem/output/cmd.log 2>&1", env=envs)
log_fname = get_temp_log_file(step_name)
ctx.download_file(f"/golem/output/cmd.log", log_fname)
try:
yield ctx.commit(timeout=timedelta(minutes=30))
task.accept_result(result=log_fname)
except BatchTimeoutError:
print(f"Task timed out: {task}, time: {task.running_time}")
raise
ctx.log("no more task to run")
# By passing `event_emitter=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=1,
budget=budget,
timeout=timeout,
subnet_tag=subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executer:
async for task in executer.submit(worker, [Task(data=step)]):
print(f"\033[36;1mStep completed: {task}\033[0m")
# grab the logs
self.state[step['name']]['log'] = task.result
# notify about this task!
self.state[step['name']]['state'] = StepState.SUCCESS
self.post_progress(step['name'])
async def main(args):
with open(".image.hash") as f:
hash = f.read()
package = await vm.repo(
image_hash=hash,
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
tasks_count = 0
async for task in tasks:
frames = [['0', '1', '2'], ['3', '4']][task.data]
framesStr = ' '.join(frames)
ctx.run(
"/bin/bash", "-c",
f"cd {GOLEM_WORKDIR} && /root/plot2.py {framesStr} > {GOLEM_WORKDIR}log.txt 2>&1"
)
for frame in frames:
ctx.download_file(f"{GOLEM_WORKDIR}log.txt",
f"log{task.data}.txt")
ctx.download_file(f"{GOLEM_WORKDIR}discontinuous{frame}.png",
f"discontinuous{frame}.png")
yield ctx.commit()
task.accept_task()
tasks_count += 1
# if tasks_count < 2:
# raise "Need at least 2 tasks!"
async with Engine(
package=package,
max_workers=args.number_of_providers,
budget=10.0,
# timeout should be keyspace / number of providers dependent
timeout=timedelta(minutes=25),
subnet_tag=args.subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
inputs = range(2)
async for task in engine.map(
worker, [Task(data=graphInput) for graphInput in inputs]):
print(f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.output}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def main():
package = await vm.repo(
image_hash="c0317d4db8930afde1862f27973ee2f5b766c4d50a87409406e2e23f",
min_mem_gib=2,
min_storage_gib=2.5,
)
async def worker_train_model(ctx: WorkContext, tasks):
async for task in tasks:
global_round = task.data['global_round']
node_id = task.data['node_id']
model_path = os.path.join(ROUND_WEIGHTS_FOLDER,
f'round_{global_round - 1}.h5')
ctx.send_file(model_path,
f"/golem/work/model_{global_round - 1}.h5")
specs = {
'start': task.data['start'],
'end': task.data['end'],
'batch_size': BATCH_SIZE,
'model_path': f'model_{global_round - 1}.h5',
'epochs': PROVIDER_EPOCHS,
'global_round': task.data['global_round'],
'node_number': task.data['node_id']
}
ctx.send_json(
"/golem/work/specs.json",
specs,
)
ctx.send_file('client.py', "/golem/work/client.py")
ctx.run("/bin/sh", "-c", "python3 client.py")
node_model_output = f'/golem/output/model_round_{global_round}_{node_id}.h5'
node_log_file = f'/golem/output/log_round_{global_round}_{node_id}.json'
ctx.download_file(
node_model_output,
os.path.join(WORKER_MODEL_WEIGHTS_FOLDER,
f'round_{global_round}_worker_{node_id}.h5'))
ctx.download_file(
node_log_file,
os.path.join(
WORKER_LOGS_FOLDER,
f'log_round_{global_round}_worker_{node_id}.json'))
yield ctx.commit(timeout=timedelta(minutes=7))
task.accept_result()
print(f"{TEXT_COLOR_GREEN}"
f"Initialising your model."
f"{TEXT_COLOR_DEFAULT}")
model = get_compiled_model()
print(f"{TEXT_COLOR_GREEN}" f"Loading the data" f"{TEXT_COLOR_DEFAULT}")
training_dataset, testing_dataset, train_length, test_length = load_dataset(
BATCH_SIZE)
print(f"{TEXT_COLOR_GREEN}"
f"Initial model evaluation - "
f"{TEXT_COLOR_DEFAULT}")
eval_results = model.evaluate(testing_dataset)
print(f"{TEXT_COLOR_BLUE}"
f"ROUND 0 | Loss: {eval_results[0]} | Accuracy: {eval_results[1]}"
f"{TEXT_COLOR_DEFAULT}")
print(f"{TEXT_COLOR_MAGENTA}"
f"Saving Model Weights for round 0"
f"{TEXT_COLOR_DEFAULT}")
model.save(os.path.join(ROUND_WEIGHTS_FOLDER, 'round_0.h5'))
for global_round_number in range(1, GLOBAL_TRAINING_ROUNDS + 1):
print(f"{TEXT_COLOR_GREEN}"
f"Beginning Training Round {global_round_number}"
f"{TEXT_COLOR_DEFAULT}")
async with Executor(
package=package,
max_workers=NUM_PROVIDERS,
budget=20.0,
timeout=timedelta(minutes=29),
subnet_tag=SUBNET_TAG,
event_consumer=log_summary(log_event_repr),
) as executor:
# No problem if we miss a few samples
training_subset_steps = int(train_length / NUM_PROVIDERS)
executor_tasks = [
Task(
data={
'start': x,
'end': x + training_subset_steps,
'global_round': global_round_number,
'node_id': index + 1
}) for index, x in enumerate(
list(range(0, train_length, training_subset_steps)))
]
async for task in executor.submit(worker_train_model,
executor_tasks):
print(
f"{TEXT_COLOR_CYAN}"
f"Training round {global_round_number} completed on provider node {task.data['node_id']}"
f"{TEXT_COLOR_DEFAULT}")
all_worker_weights = get_client_model_weights(
WORKER_MODEL_WEIGHTS_FOLDER, global_round_number)
averaged_weights = federated_avg_weights(all_worker_weights)
model.set_weights(averaged_weights)
print(f"{TEXT_COLOR_GREEN}"
f"TRAINING ROUND {global_round_number} complete!"
f"{TEXT_COLOR_DEFAULT}")
eval_results = model.evaluate(testing_dataset)
print(
f"{TEXT_COLOR_BLUE}"
f"ROUND {global_round_number} | Loss: {eval_results[0]} | Accuracy: {eval_results[1]}"
f"{TEXT_COLOR_DEFAULT}")
print(f"{TEXT_COLOR_MAGENTA}"
f"Saving Model Weights for round {global_round_number}"
f"{TEXT_COLOR_DEFAULT}")
model.save(
os.path.join(ROUND_WEIGHTS_FOLDER,
f'round_{global_round_number}.h5'))
async def main(subnet_tag: str):
package = await vm.repo(
image_hash="83b5ebab52f39e676173de32f56cf2648c136050b8fa1f31a791c467",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
ctx.send_file(
"./lens.py",
"/golem/work/lens.py",
)
async for task in tasks:
feed = task.data
ctx.send_json(
"/golem/work/params.json",
feed,
)
commands = (
"python3 /golem/work/lens.py >> /golem/output/task-log 2>&1;")
ctx.run("/bin/sh", "-c", commands)
frame_start = feed["start_frame"]
frame_end = feed["start_frame"] + len(feed["points"])
frames = range(frame_start, frame_end)
ctx.log(f"Downloading frames {frame_start}-{frame_end}...")
for frame in frames:
ctx.download_file(f"/golem/output/{frame}.png",
f"out/{frame + 100}.png")
output = f"task-log"
ctx.download_file(f"/golem/output/task-log", f"out/{output}")
yield ctx.commit()
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_task(result=output)
ctx.log("no more frames to render")
points = np.arange(0.001, 1.0, 0.005)
feeds = []
for i in range(len(points) // 10):
feed = {
"start_frame": 10 * i,
"points": [points[i] for i in range(10 * i, 10 * (i + 1))]
}
feeds.append(feed)
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Engine(
package=package,
max_workers=8,
budget=100.0,
timeout=timedelta(minutes=3) + timedelta(minutes=10),
subnet_tag=subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
async for task in engine.map(worker,
[Task(data=feed) for feed in feeds]):
print(f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.output}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def main(subnet_tag: str):
package = await vm.repo(
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
scene_path = str(script_dir / "cubes.blend")
ctx.send_file(scene_path, "/golem/resource/scene.blend")
async for task in tasks:
frame = task.data
crops = [{
"outfilebasename": "out",
"borders_x": [0.0, 1.0],
"borders_y": [0.0, 1.0]
}]
ctx.send_json(
"/golem/work/params.json",
{
"scene_file": "/golem/resource/scene.blend",
"resolution": (400, 300),
"use_compositing": False,
"crops": crops,
"samples": 100,
"frames": [frame],
"output_format": "PNG",
"RESOURCES_DIR": "/golem/resources",
"WORK_DIR": "/golem/work",
"OUTPUT_DIR": "/golem/output",
},
)
ctx.run("/golem/entrypoints/run-blender.sh")
output_file = f"output_{frame}.png"
ctx.download_file(f"/golem/output/out{frame:04d}.png", output_file)
yield ctx.commit()
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_task(result=output_file)
ctx.log("no more frames to render")
# iterator over the frame indices that we want to render
frames: range = range(0, 60, 10)
# TODO make this dynamic, e.g. depending on the size of files to transfer
# worst-case time overhead for initialization, e.g. negotiation, file transfer etc.
init_overhead: timedelta = timedelta(minutes=3)
# By passing `event_emitter=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Engine(
package=package,
max_workers=3,
budget=10.0,
timeout=init_overhead + timedelta(minutes=len(frames) * 2),
subnet_tag=subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
async for task in engine.map(worker,
[Task(data=frame) for frame in frames]):
print(f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.output}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def inner(*args, **kwargs):
# Firstly, we'll save the function body to file
tmpdir = tempfile.TemporaryDirectory()
module_path = PurePath(f"{tmpdir.name}/gfaas_module")
with open(module_path, "wb") as f:
marshal.dump(func.__code__, f)
if self.run_local:
import asyncio
fut = self.engine.submit(_local_remote_fn, module_path, *args)
res = await asyncio.wait_for(asyncio.wrap_future(fut),
self.timeout.seconds)
return res
else:
from yapapi.runner import Engine, Task, vm
from yapapi.runner.ctx import WorkContext
from yapapi.log import enable_default_logger, log_summary
# Save input args to files
saved_args = []
for i, arg in enumerate(args):
arg_path = PurePath(f"{tmpdir.name}/arg{i}")
with open(arg_path, "w") as f:
json.dump(arg, f)
saved_args.append(arg_path)
enable_default_logger()
package = await vm.repo(
image_hash=
"74e9cdb5a5aa2c73a54f9ebf109986801fe2d4f026ea7d9fbfcca221",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
out_path = PurePath(f"{tmpdir.name}/out")
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.send_file(module_path, "/golem/input/func")
remote_args = []
for (i, arg_path) in enumerate(saved_args):
remote_arg = f"/golem/input/arg{i}"
ctx.send_file(arg_path, remote_arg)
remote_args.append(remote_arg)
ctx.run("python", "/golem/runner.py",
"/golem/input/func", *remote_args)
ctx.download_file("/golem/output/out", out_path)
yield ctx.commit()
task.accept_task(result=out_path)
ctx.log("done")
init_overhead: timedelta = timedelta(minutes=3)
async with Engine(
package=package,
max_workers=1,
budget=self.budget,
timeout=init_overhead + self.timeout,
subnet_tag=self.subnet,
event_emitter=log_summary(),
) as engine:
async for progress in engine.map(worker,
[Task(data=None)]):
print(f"progress={progress}")
with open(out_path, "r") as f:
out = json.load(f)
return out
async def main(subnet_tag: str):
package = await vm.repo(
image_hash="9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker(ctx: WorkContext, tasks):
scene_path = str(script_dir / "cubes.blend")
ctx.send_file(scene_path, "/golem/resource/scene.blend")
async for task in tasks:
frame = task.data
crops = [{
"outfilebasename": "out",
"borders_x": [0.0, 1.0],
"borders_y": [0.0, 1.0]
}]
ctx.send_json(
"/golem/work/params.json",
{
"scene_file": "/golem/resource/scene.blend",
"resolution": (400, 300),
"use_compositing": False,
"crops": crops,
"samples": 100,
"frames": [frame],
"output_format": "PNG",
"RESOURCES_DIR": "/golem/resources",
"WORK_DIR": "/golem/work",
"OUTPUT_DIR": "/golem/output",
},
)
ctx.run("/golem/entrypoints/run-blender.sh")
output_file = f"output_{frame}.png"
ctx.download_file(f"/golem/output/out{frame:04d}.png", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=120))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(f"{utils.TEXT_COLOR_RED}"
f"Task timed out: {task}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
raise
# Iterator over the frame indices that we want to render
frames: range = range(0, 60, 10)
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
# TODO: make this dynamic, e.g. depending on the size of files to transfer
init_overhead = 3
# Providers will not accept work if the timeout is outside of the [5 min, 30min] range.
# We increase the lower bound to 6 min to account for the time needed for our demand to
# reach the providers.
min_timeout, max_timeout = 6, 30
timeout = timedelta(
minutes=max(min(init_overhead +
len(frames) * 2, max_timeout), min_timeout))
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=3,
budget=10.0,
timeout=timeout,
subnet_tag=subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executor:
async for task in executor.submit(
worker, [Task(data=frame) for frame in frames]):
print(
f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}, time: {task.running_time}"
f"{utils.TEXT_COLOR_DEFAULT}")
def _default_event_consumer() -> Callable[[events.Event], None]:
from yapapi.log import log_event_repr, log_summary
return log_summary(log_event_repr)
async def main(subnet_tag, driver=None, network=None):
package = await vm.repo(
image_hash="2c5d9a80847eb147261c4e33df6e6955666ddd932ec40fd8b005f799",
min_mem_gib=1,
min_storage_gib=10.0,
)
async def worker(ctx: WorkContext, tasks):
script_dir = pathlib.Path(__file__).resolve().parent
async for task in tasks:
frame = task.data
ctx.run("/bin/sh", "-c", "./generate_data.sh")
output_file = f"output/{frame}.zip"
ctx.download_file(f"/golem/output/data.zip", output_file)
try:
# Set timeout for executing the script on the provider. Two minutes is plenty
# of time for computing a single frame, for other tasks it may be not enough.
# If the timeout is exceeded, this worker instance will be shut down and all
# remaining tasks, including the current one, will be computed by other providers.
yield ctx.commit(timeout=timedelta(seconds=1750))
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_result(result=output_file)
except BatchTimeoutError:
print(
f"{TEXT_COLOR_RED}"
f"Task {task} timed out on {ctx.provider_name}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}")
raise
# Iterator over the frame indices that we want to render
frames: range = range(
0,
60,
)
# Worst-case overhead, in minutes, for initialization (negotiation, file transfer etc.)
# TODO: make this dynamic, e.g. depending on the size of files to transfer
init_overhead = 3
# Providers will not accept work if the timeout is outside of the [5 min, 30min] range.
# We increase the lower bound to 6 min to account for the time needed for our demand to
# reach the providers.
min_timeout, max_timeout = 6, 30
timeout = timedelta(
minutes=max(min(init_overhead +
len(frames) * 2, max_timeout), min_timeout))
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=30,
budget=10.0,
timeout=timeout,
subnet_tag=subnet_tag,
driver=driver,
network=network,
event_consumer=log_summary(log_event_repr),
) as executor:
sys.stderr.write(
f"yapapi version: {TEXT_COLOR_YELLOW}{yapapi_version}{TEXT_COLOR_DEFAULT}\n"
f"Using subnet: {TEXT_COLOR_YELLOW}{subnet_tag}{TEXT_COLOR_DEFAULT}, "
f"payment driver: {TEXT_COLOR_YELLOW}{executor.driver}{TEXT_COLOR_DEFAULT}, "
f"and network: {TEXT_COLOR_YELLOW}{executor.network}{TEXT_COLOR_DEFAULT}\n"
)
num_tasks = 0
start_time = datetime.now()
async for task in executor.submit(
worker, [Task(data=frame) for frame in frames]):
num_tasks += 1
print(
f"{TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}, time: {task.running_time}"
f"{TEXT_COLOR_DEFAULT}")
print(
f"{TEXT_COLOR_CYAN}"
f"{num_tasks} tasks computed, total time: {datetime.now() - start_time}"
f"{TEXT_COLOR_DEFAULT}")
async def main(args):
package = await vm.repo(
image_hash="2c17589f1651baff9b82aa431850e296455777be265c2c5446c902e9",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
async def worker_check_keyspace(ctx: WorkContext, tasks):
async for task in tasks:
keyspace_sh_filename = "keyspace.sh"
ctx.send_file(keyspace_sh_filename, "/golem/work/keyspace.sh")
ctx.run("/bin/sh", "/golem/work/keyspace.sh")
output_file = "keyspace.txt"
ctx.download_file("/golem/work/keyspace.txt", output_file)
yield ctx.commit()
task.accept_result()
async def worker_find_password(ctx: WorkContext, tasks):
ctx.send_file("in.hash", "/golem/work/in.hash")
async for task in tasks:
skip = task.data
limit = skip + step
# Commands to be run on the provider
commands = (
"rm -f /golem/work/*.potfile ~/.hashcat/hashcat.potfile; "
f"touch /golem/work/hashcat_{skip}.potfile; "
f"hashcat -a 3 -m 400 /golem/work/in.hash {args.mask} --skip={skip} --limit={limit} --self-test-disable -o /golem/work/hashcat_{skip}.potfile || true"
)
ctx.run(f"/bin/sh", "-c", commands)
output_file = f"hashcat_{skip}.potfile"
ctx.download_file(f"/golem/work/hashcat_{skip}.potfile",
output_file)
yield ctx.commit()
task.accept_result(result=output_file)
# beginning of the main flow
write_hash(args.hash)
write_keyspace_check_script(args.mask)
# By passing `event_consumer=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Executor(
package=package,
max_workers=args.number_of_providers,
budget=10.0,
# timeout should be keyspace / number of providers dependent
timeout=timedelta(minutes=25),
subnet_tag=args.subnet_tag,
event_consumer=log_summary(log_event_repr),
) as executor:
keyspace_computed = False
# This is not a typical use of executor.submit as there is only one task, with no data:
async for _task in executor.submit(worker_check_keyspace,
[Task(data=None)]):
keyspace_computed = True
if not keyspace_computed:
# Assume the errors have been already reported and we may return quietly.
return
keyspace = read_keyspace()
print(
f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: keyspace size count. The keyspace size is {keyspace}"
f"{utils.TEXT_COLOR_DEFAULT}")
step = int(keyspace / args.number_of_providers) + 1
ranges = range(0, keyspace, step)
async for task in executor.submit(
worker_find_password, [Task(data=range) for range in ranges]):
print(f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.result}"
f"{utils.TEXT_COLOR_DEFAULT}")
password = read_password(ranges)
if password is None:
print(
f"{utils.TEXT_COLOR_RED}No password found{utils.TEXT_COLOR_DEFAULT}"
)
else:
print(f"{utils.TEXT_COLOR_GREEN}"
f"Password found: {password}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def main(subnet_tag: str, presets: str, input_file: str, job: str):
package = await vm.repo(
image_hash="9a6b2409ccd96b3352b3fcdc6e1e568bdfa4d4e258589cfaff084786",
min_mem_gib=0.5,
min_storage_gib=2.0,
)
preset_list = list(filter(str.strip, presets.split(",")))
async def worker(ctx: WorkContext, tasks):
sent_file = "/golem/resource/input.file"
ctx.send_file(input_file, sent_file)
async for task in tasks:
preset = task.data
if "MKV" in preset or preset == "Roku 2160p60 4K HEVC Surround":
output_ext = ".mkv"
else:
output_ext = ".mp4"
subjob = f"{job}_{preset_list.index(preset)+1}"
update_status(subjob, "starting")
output_file = f"{subjob}{output_ext}"
ctx.log(f"job: {subjob}")
update_status(subjob, "job sent")
commands = (
'cd /golem/output; '
f'echo preset:{preset} > log.txt; '
f'echo output_file:{output_file} >> log.txt; '
f"HandBrakeCLI -i {sent_file} -o {output_file} --preset '{preset}' >> log.txt 2>&1; "
'ls -lah >> log.txt')
ctx.run("/bin/sh", "-c", commands)
#ctx.download_file(f"/golem/output/log.txt", "log.txt")
ctx.download_file(f"/golem/output/{output_file}",
f"./downloads/{output_file}")
yield ctx.commit()
update_status(subjob, "done")
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_task(result=output_file)
ctx.log("no more videos to convert!")
init_overhead: timedelta = timedelta(minutes=7)
# By passing `event_emitter=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Engine(
package=package,
max_workers=10,
budget=20.0,
timeout=init_overhead,
subnet_tag=subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
async for task in engine.map(
worker, [Task(data=preset) for preset in preset_list]):
print(f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.output}"
f"{utils.TEXT_COLOR_DEFAULT}")
async def main(subnet_tag: str, node_count: int, timeout_seconds: int,
password: str):
print(
f"Stats: Nodes = {node_count}, Timeout = {timeout_seconds}s, Password = {password}"
)
if node_count < 2:
raise Exception("Invalid node_count. There must be 2 or more nodes.")
else:
print("Running more than 2 nodes")
package = await vm.repo(
image_hash="a6722779ca397520db82f354684a46a13c473aca04e38b91ec32755e",
min_mem_gib=4,
min_storage_gib=3.0,
)
async def ripper(ctx: WorkContext, tasks):
async for task in tasks:
output_file = f"out_{str(task.data['node'])}.txt"
#ctx.send_file('ripper.sh', '/golem/entrypoints/ripper.sh') #in case you want to send your modify it bash uncomment
ctx.run('bin/sh', '/golem/entrypoints/ripper.sh',
str(task.data['node']), str(task.data['nodes']),
str(timeout_seconds), password)
ctx.log("task data is:")
ctx.log(task.data)
ctx.download_file("/golem/output/result.txt", output_file)
yield ctx.commit()
task.accept_task(result=output_file)
ctx.log("hmm.. it looks like we got some passwords")
# iterator over the frame indices that we want to render
nodes = [
Task(data={
'node': i + 1,
'nodes': node_count
}) for i in range(node_count)
]
init_overhead: timedelta = timedelta(minutes=10)
# By passing `event_emitter=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Engine(
package=package,
max_workers=node_count,
budget=30.0,
timeout=init_overhead + timedelta(minutes=node_count * 2),
subnet_tag=subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
async for task in engine.map(ripper, nodes):
print(f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.output}"
f"{utils.TEXT_COLOR_DEFAULT}")
# Process done, so remind the user of the parameters and show the results
print(
f"Parameters: Nodes = {node_count}, Timeout = {timeout_seconds}s, Password = {password}"
)
for i in range(node_count):
output_file = f"out_{str(i+1)}.txt"
with open(output_file) as f:
lines = f.readlines()
echoer_message = f"Ripper {i+1} did not find the password"
for line in lines:
if "?" in line:
echoer_message = f"{utils.TEXT_COLOR_GREEN}Worker {i+1} found the password: {line[2:].strip()}{utils.TEXT_COLOR_DEFAULT}"
break
print(echoer_message)
async def main(subnet_tag: str, node_count: int, timeout_seconds: int, password: str):
print(f"Parameters: Nodes = {node_count}, Timeout = {timeout_seconds}s, Password = {password}")
if node_count < 2:
raise Exception("Invalid node_count. There must be 2 or more nodes.")
package = await vm.repo(
image_hash="c6c34d6462daa7307ace83c08028461411a0e0133d4db053904a89df",
min_mem_gib=4,
min_storage_gib=3.0,
)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
output_file = f"out_{str(task.data['node'])}.txt"
ctx.run("/golem/entrypoints/crack.sh", str(task.data['node']), str(task.data['nodes']), str(timeout_seconds), password)
ctx.log("task data is:")
ctx.log(task.data)
ctx.download_file("/golem/output/result.txt", output_file)
yield ctx.commit()
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_task(result=output_file)
ctx.log("no more frames to render")
# iterator over the frame indices that we want to render
nodes = [Task(data={'node': i+1, 'nodes': node_count}) for i in range(node_count)]
init_overhead: timedelta = timedelta(minutes=10)
# By passing `event_emitter=log_summary()` we enable summary logging.
# See the documentation of the `yapapi.log` module on how to set
# the level of detail and format of the logged information.
async with Engine(
package=package,
max_workers=node_count,
budget=20.0,
timeout=init_overhead + timedelta(minutes=node_count * 2),
subnet_tag=subnet_tag,
event_emitter=log_summary(log_event_repr),
) as engine:
async for task in engine.map(worker, nodes):
print(
f"{utils.TEXT_COLOR_CYAN}"
f"Task computed: {task}, result: {task.output}"
f"{utils.TEXT_COLOR_DEFAULT}"
)
# Processing is done, so remind the user of the parameters and show the results
print(f"Parameters: Nodes = {node_count}, Timeout = {timeout_seconds}s, Password = {password}")
for i in range(node_count):
output_file = f"out_{str(i+1)}.txt"
with open(output_file) as f:
lines = f.readlines()
found_message = f"Worker {i+1} did not find the password"
for line in lines:
if "?" in line:
found_message = f"{utils.TEXT_COLOR_YELLOW}Worker {i+1} found the password: {line[2:].strip()}{utils.TEXT_COLOR_DEFAULT}"
break
print(found_message)
