async def worker(ctx: WorkContext, tasks):
async for task in tasks:
(filename, offset, secondFrequency, smoothness) = task.data
full_filename = filename + ".png"
# Send these files so we don't have to update the Docker image
ctx.send_file('graphWavePair.py', F'{GOLEM_WORKDIR}graphWavePair.py')
ctx.run(F'chmod', 'u+x', F'{GOLEM_WORKDIR}graphWavePair.py')
ctx.run(F'/bin/sh', '-c', F'{GOLEM_WORKDIR}graphWavePair.py {filename} {offset} {secondFrequency} {smoothness}')
ctx.download_file(f'{GOLEM_WORKDIR+full_filename}', full_filename)
yield ctx.commit()
task.accept_task(result=full_filename)
async def __worker_render_frame(self, ctx: WorkContext,
tasks: [GeomandelTask]):
"""Creates a set of instructions for each subtask"""
async for task in tasks:
# Send subtask data as JSON to the remote execution environment.
ctx.send_json(_TASK_INPUT_PATH, task.data.__dict__)
# Run python script loading the data from JSON and executing the geomandel binary to generate the image.
ctx.run(*_GEOMANDEL_COMMAND)
# Download the output file.
ctx.download_file(_TASK_OUTPUT_PATH, task.data.output_path)
# Return a sequence of commands to be executed when remote node agrees to process a task.
yield ctx.commit()
# TODO: actual verification
task.accept_task(result=task.data.output_path)
async def __transcode_remote(self, ctx: WorkContext,
tasks: [TranscodingTask]):
"""Creates a set of instructions for each subtask"""
async for task in tasks:
remote_output_path: str = f"/golem/work/output.{task.data.extension}"
# Send input video to remote node
ctx.send_file(task.data.input, _TASK_INPUT_REMOTE_PATH)
# Execute ffmpeg command.
ctx.run("/usr/bin/ffmpeg", "-i", _TASK_INPUT_REMOTE_PATH,
remote_output_path)
# Download the output file.
ctx.download_file(remote_output_path, task.data.output)
# Return a sequence of commands to be executed when remote node agrees to process a task.
yield ctx.commit()
task.accept_task()
async def worker(ctx: WorkContext, tasks):
tasks_count = 0
async for task in tasks:
frames = [['0', '1', '2'], ['3', '4']][task.data]
framesStr = ' '.join(frames)
ctx.run(
"/bin/bash", "-c",
f"cd {GOLEM_WORKDIR} && /root/plot2.py {framesStr} > {GOLEM_WORKDIR}log.txt 2>&1"
)
for frame in frames:
ctx.download_file(f"{GOLEM_WORKDIR}log.txt",
f"log{task.data}.txt")
ctx.download_file(f"{GOLEM_WORKDIR}discontinuous{frame}.png",
f"discontinuous{frame}.png")
yield ctx.commit()
task.accept_task()
tasks_count += 1
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
output_file_name = f'{task.data}.png'
output_file_path = f'/golem/work/{output_file_name}'
ctx.send_file('plot.sh', '/golem/work/plot.sh')
command = (
'/bin/sh',
'/golem/work/plot.sh',
task.data,
data_file,
country_codes_file,
output_file_path,
args.parameter,
)
print(*command)
ctx.run(*command)
yield ctx.commit()
ctx.download_file(output_file_path, f'outputs/{output_file_name}')
task.accept_task(result=output_file_name)
async def worker(ctx: WorkContext, tasks):
ctx.send_file("./program.py", "/golem/work/program.py")
async for task in tasks:
ctx.begin()
ctx.run("/golem/entrypoints/run.sh")
ctx.download_file("/golem/output/results.txt", "results.txt")
yield ctx.commit()
# TODO: Check if job results are valid
# and reject by: task.reject_task(msg = 'invalid file')
task.accept_task()
ctx.log("no more tasks!")
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.send_file(module_path, "/golem/input/func")
remote_args = []
for (i, arg_path) in enumerate(saved_args):
remote_arg = f"/golem/input/arg{i}"
ctx.send_file(arg_path, remote_arg)
remote_args.append(remote_arg)
ctx.run("python", "/golem/runner.py",
"/golem/input/func", *remote_args)
ctx.download_file("/golem/output/out", out_path)
yield ctx.commit()
task.accept_task(result=out_path)
ctx.log("done")
async def worker(ctx: WorkContext, tasks):
data_path = Path(DATA_PATH)
num_files = len(list(data_path.glob("*")))
async for task in tasks:
partition = task.data
start_index = partition * PARTITION_SLICES
# if it's the final partition, iterate all the way to the end
if partition == math.floor(num_files / PARTITION_SLICES):
end_index = num_files
else:
end_index = start_index + PARTITION_SLICES
tar_path = f"tmp/dicom_{partition}.tar.gz"
with tarfile.open(tar_path, mode="w:gz") as dicom_tar:
for i, dicom_path in enumerate(data_path.rglob("*.dcm")):
if start_index <= i < end_index:
dicom_tar.add(str(dicom_path))
ctx.send_file(tar_path, "/golem/resource/dicom.tar.gz")
ctx.begin()
# TODO send config info to container
ctx.run("/golem/entrypoints/run")
ctx.download_file(f"/golem/output/log.out",
f"output/log_{partition}.out")
# TODO download .npy file instead of .png
ctx.download_file(f"/golem/output/3d_clusters.png",
f"output/3d_clusters_{partition}.png")
yield ctx.commit()
# TODO: Check if job results are valid
# and reject by: task.reject_task(msg = 'invalid file')
task.accept_task()
ctx.log("no more partitions to search")
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
output_file = f"out_{str(task.data['node'])}.txt"
ctx.run("/golem/entrypoints/crack.sh", str(task.data['node']), str(task.data['nodes']), str(timeout_seconds), password)
ctx.log("task data is:")
ctx.log(task.data)
ctx.download_file("/golem/output/result.txt", output_file)
yield ctx.commit()
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_task(result=output_file)
ctx.log("no more frames to render")
async def worker(ctx: WorkContext, tasks):
ctx.send_file(
"./lens.py",
"/golem/work/lens.py",
)
async for task in tasks:
feed = task.data
ctx.send_json(
"/golem/work/params.json",
feed,
)
commands = (
"python3 /golem/work/lens.py >> /golem/output/task-log 2>&1;")
ctx.run("/bin/sh", "-c", commands)
frame_start = feed["start_frame"]
frame_end = feed["start_frame"] + len(feed["points"])
frames = range(frame_start, frame_end)
ctx.log(f"Downloading frames {frame_start}-{frame_end}...")
for frame in frames:
ctx.download_file(f"/golem/output/{frame}.png",
f"out/{frame + 100}.png")
output = f"task-log"
ctx.download_file(f"/golem/output/task-log", f"out/{output}")
yield ctx.commit()
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_task(result=output)
ctx.log("no more frames to render")
async def worker(ctx: WorkContext, tasks):
sent_file = "/golem/resource/input.file"
ctx.send_file(input_file, sent_file)
async for task in tasks:
preset = task.data
if "MKV" in preset or preset == "Roku 2160p60 4K HEVC Surround":
output_ext = ".mkv"
else:
output_ext = ".mp4"
subjob = f"{job}_{preset_list.index(preset)+1}"
update_status(subjob, "starting")
output_file = f"{subjob}{output_ext}"
ctx.log(f"job: {subjob}")
update_status(subjob, "job sent")
commands = (
'cd /golem/output; '
f'echo preset:{preset} > log.txt; '
f'echo output_file:{output_file} >> log.txt; '
f"HandBrakeCLI -i {sent_file} -o {output_file} --preset '{preset}' >> log.txt 2>&1; "
'ls -lah >> log.txt')
ctx.run("/bin/sh", "-c", commands)
#ctx.download_file(f"/golem/output/log.txt", "log.txt")
ctx.download_file(f"/golem/output/{output_file}",
f"./downloads/{output_file}")
yield ctx.commit()
update_status(subjob, "done")
# TODO: Check if job results are valid
# and reject by: task.reject_task(reason = 'invalid file')
task.accept_task(result=output_file)
ctx.log("no more videos to convert!")
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.send_file('in.hash', '/golem/work/in.hash')
ctx.send_file('crack.sh', '/golem/work/crack.sh')
ctx.run('touch', '/golem/work/out.txt')
ctx.run('/bin/sh', '/golem/work/crack.sh', f'{task.data}', f'{args.number_of_providers}')
log_file = f'logs/log_{task.data}.txt'
ctx.download_file('/golem/work/log.txt', log_file)
output_file = f'outputs/out_{task.data}.txt'
ctx.download_file('/golem/work/out.txt', output_file)
yield ctx.commit()
task.accept_task(result=output_file)
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.send_file('fire.sh', '/golem/work/fire.sh')
ctx.run('/bin/sh', '/golem/work/fire.sh')
output_file = f"vim"
ctx.download_file('/golem/work/vim/src/vim', 'vim')
ctx.download_file('/golem/work/log.txt', 'log')
yield ctx.commit()
task.accept_task(result=output_file)
ctx.log(f'done!')
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
node_no = task.data
ctx.send_file("lorenz.py", "/golem/work/task.py")
cmd = f"python3 /golem/work/task.py -d {duration} -n {node_no}/{num_nodes} -m {num_trajectories} -l {time_delta} "
print(f"\033[36;1mRunning {cmd}\033[0m")
ctx.run("sh", "-c", f"{cmd} >> /golem/output/log.txt 2>&1")
ctx.download_file(f"/golem/output/log.txt", "log.txt")
task_duration = duration / num_nodes
# frame start stop index
t0 = int(task_duration / time_delta) * (node_no - 1)
t1 = t0 + int(task_duration / time_delta)
for t in range(t0, t1):
ctx.download_file(f"/golem/output/frame_{t:04d}.png",
f"output/frame_{t:04d}.png")
yield ctx.commit()
task.accept_task()
ctx.log("no more task to run")
async def ripper(ctx: WorkContext, tasks):
async for task in tasks:
output_file = f"out_{str(task.data['node'])}.txt"
#ctx.send_file('ripper.sh', '/golem/entrypoints/ripper.sh') #in case you want to send your modify it bash uncomment
ctx.run('bin/sh', '/golem/entrypoints/ripper.sh',
str(task.data['node']), str(task.data['nodes']),
str(timeout_seconds), password)
ctx.log("task data is:")
ctx.log(task.data)
ctx.download_file("/golem/output/result.txt", output_file)
yield ctx.commit()
task.accept_task(result=output_file)
ctx.log("hmm.. it looks like we got some passwords")
async def worker(ctx: WorkContext, tasks):
ctx.send_file("./scene.blend", "/golem/resource/scene.blend")
async for task in tasks:
frame = task.data
ctx.begin()
crops = [{
"outfilebasename": "out",
"borders_x": [0.0, 1.0],
"borders_y": [0.0, 1.0]
}]
ctx.send_json(
"/golem/work/params.json",
{
"scene_file": "/golem/resource/scene.blend",
"resolution": (800, 600),
"use_compositing": False,
"crops": crops,
"samples": 100,
"frames": [frame],
"output_format": "PNG",
"RESOURCES_DIR": "/golem/resources",
"WORK_DIR": "/golem/work",
"OUTPUT_DIR": "/golem/output",
},
)
ctx.run("/golem/entrypoints/run-blender.sh")
ctx.download_file(f"/golem/output/out{frame:04d}.png",
f"output_{frame}.png")
yield ctx.commit()
# TODO: Check if job is valid
# and reject by: task.reject_task(msg = 'invalid file')
task.accept_task()
ctx.log("no more frame to render")
async def worker(ctx: WorkContext, tasks):
async for task in tasks:
ctx.send_file('make.sh', '/golem/work/make.sh')
ctx.run('/bin/sh', '/golem/work/make.sh')
output_file = 'vim'
ctx.download_file('/golem/work/vim/src/vim', 'vim')
ctx.download_file('/golem/work/out.txt', 'log')
yield ctx.commit()
task.accept_task(result=output_file)
ctx.log(f'VIM compiled!')