def test_find_task_script(self):
script_path = os.path.join(self.path, "resources", "scripts")
os.makedirs(script_path)
script = os.path.join(script_path, "bla")
open(script, "w").close()
task_file = os.path.join(self.path, "task", "testtask.py")
path = find_task_script(self.path, "bla")
self.assertTrue(os.path.isdir(os.path.dirname(path)))
self.assertEqual(os.path.basename(path), "bla")
with self.assertLogs(logger, level="ERROR"):
find_task_script(self.path, "notexisting")
def __init__(self, tag=BLENDER_DOCKER_TAG, image_id=None):
image = DockerImage(image_id=id) if image_id \
else DockerImage(self.BLENDER_DOCKER_IMAGE, tag=tag)
DockerEnvironment.__init__(self, [image])
self.short_description = "Blender (www.blender.org)"
self.main_program_file = find_task_script(self.APP_DIR, self.SCRIPT_NAME)
def test_dummytask_job(self):
app_dir = path.join(get_golem_path(), "apps", "dummy")
task_script = find_task_script(app_dir, "docker_dummytask.py")
with open(task_script) as f:
task_script_src = f.read()
os.mkdir(os.path.join(self.resources_dir, "data"))
os.mkdir(os.path.join(self.resources_dir, "code"))
# copy the resources to the resources dir
data_dir = path.join(get_golem_path(), "apps", "dummy", "test_data")
for f in os.listdir(data_dir):
task_file = path.join(data_dir, f)
if path.isfile(task_file) or path.isdir(task_file):
shutil.copy(task_file, path.join(self.resources_dir, "data",
f))
code_dir = path.join(get_golem_path(), "apps", "dummy", "resources",
"code_dir")
for f in os.listdir(code_dir):
task_file = path.join(code_dir, f)
if (path.isfile(task_file) or path.isdir(task_file)) \
and os.path.basename(task_file) != "__pycache__":
shutil.copy(task_file, path.join(self.resources_dir, "code",
f))
# this is the stuff that is available by "params" module
# in the docker job script
params = {
"data_files": ["in.data"],
"subtask_data": "00110011", # it is kept in string on purpose
"subtask_data_size":
8, # subtask_data_size is to double check the size,
# if we haven't kept subtask_data in string,
# we would lose leading zeros
"difficulty": 10,
"result_size": 256,
"result_file": "out.result",
}
with self._create_test_job(script=task_script_src,
params=params) as job:
job.start()
exit_code = job.wait()
self.assertEqual(exit_code, 0)
out_files = os.listdir(self.output_dir)
self.assertTrue(
any(f.endswith(".result") and "out" in f for f in out_files))
def test_blender_job(self):
app_dir = os.path.join(get_golem_path(), "apps", "blender")
task_script = find_task_script(app_dir, "docker_blendertask.py")
with open(task_script) as f:
task_script_src = f.read()
# prepare dummy crop script
from apps.blender.resources.scenefileeditor import generate_blender_crop_file
crop_script_contents = generate_blender_crop_file(
resolution=(800, 600),
borders_x=(0, 1),
borders_y=(0, 1),
use_compositing=True,
)
# copy the scene file to the resources dir
benchmarks_dir = path.join(get_golem_path(),
path.normpath("apps/blender/benchmark/"))
scene_files = glob.glob(path.join(benchmarks_dir, "**/*.blend"))
if len(scene_files) == 0:
self.fail("No .blend files available")
shutil.copy(scene_files[0], self.resources_dir)
params = {
"outfilebasename":
"out",
"scene_file":
DockerJob.RESOURCES_DIR + "/" + path.basename(scene_files[0]),
"script_src":
crop_script_contents,
"start_task":
42,
"end_task":
42,
"output_format":
"EXR",
"frames": [1],
}
with self._create_test_job(script=task_script_src,
params=params) as job:
job.start()
exit_code = job.wait()
self.assertEqual(exit_code, 0)
out_files = os.listdir(self.output_dir)
self.assertEqual(out_files, ['out_420001.exr'])
def test_luxrender_job(self):
app_dir = path.join(get_golem_path(), "apps", "lux")
task_script = find_task_script(app_dir, "docker_luxtask.py")
with open(task_script) as f:
task_script_src = f.read()
# read the scene file and copy the resources to the resources dir
lux_task_dir = path.join(get_golem_path(),
"apps", "lux", "benchmark", "test_task")
scene_src = None
for f in os.listdir(lux_task_dir):
task_file = path.join(lux_task_dir, f)
if path.isfile(task_file) and task_file.endswith(".lxs"):
if scene_src is not None:
self.fail("Multiple .lxs files found in {}"
.format(lux_task_dir))
with open(task_file, "r") as scene_file:
scene_src = scene_file.read()
elif path.isdir(task_file):
shutil.copytree(task_file, path.join(self.resources_dir, f))
if scene_src is None:
self.fail("No .lxs files found in {}".format(lux_task_dir))
params = {
"outfilebasename": "out",
"output_format": "png",
"scene_file_src": scene_src,
"start_task": 42,
"end_task": 42,
"frames": [1],
"scene_dir": "/golem/resources/",
"num_threads": 1
}
with self._create_test_job(script=task_script_src, params=params) as job:
job.start()
exit_code = job.wait()
self.assertEqual(exit_code, 0)
out_files = os.listdir(self.output_dir)
self.assertEqual(out_files, ['out42.png'])
def __get_merge_ctd(self, files):
script_file = dirmanager.find_task_script(APP_DIR,
"docker_luxmerge.py")
if script_file is None:
logger.error("Cannot find merger script")
return
with open(script_file) as f:
src_code = f.read()
ctd = ComputeTaskDef()
ctd.task_id = self.header.task_id
ctd.subtask_id = self.header.task_id
ctd.extra_data = {'output_flm': self.output_file, 'flm_files': files}
ctd.src_code = src_code
ctd.working_directory = "."
ctd.docker_images = self.header.docker_images
ctd.deadline = timeout_to_deadline(self.merge_timeout)
return ctd
def __init__(self,
tag=None,
image_id=None,
additional_images: List[DockerImage] = None):
if tag is None:
tag = self.DOCKER_TAG
image = DockerImage(image_id=image_id) if image_id \
else DockerImage(self.DOCKER_IMAGE, tag=tag)
Environment.__init__(self)
self.main_program_file = find_task_script(self.APP_DIR,
self.SCRIPT_NAME)
self.docker_images = [image]
if additional_images:
self.docker_images += additional_images
if self.SHORT_DESCRIPTION:
self.short_description = self.SHORT_DESCRIPTION
def test_blender_job(self):
app_dir = os.path.join(get_golem_path(), "apps", "blender")
task_script = find_task_script(app_dir, "docker_blendertask.py")
with open(task_script) as f:
task_script_src = f.read()
# prepare dummy crop script
crop_script_contents = generate_blender_crop_file(resolution=(800,
600),
borders_x=(0, 1),
borders_y=(0, 1),
use_compositing=True,
samples=5)
# copy the scene file to the resources dir
scene_file = pathlib.Path(get_golem_path())
scene_file /= "apps/blender/benchmark/test_task/cube.blend"
shutil.copy(str(scene_file), self.resources_dir)
dest_scene_file = pathlib.PurePosixPath(DockerJob.RESOURCES_DIR)
dest_scene_file /= scene_file.name
params = {
"outfilebasename": "out",
"scene_file": str(dest_scene_file),
"script_src": crop_script_contents,
"start_task": 42,
"end_task": 42,
"output_format": "EXR",
"frames": [1],
}
with self._create_test_job(script=task_script_src, params=params) \
as job:
job.start()
exit_code = job.wait(timeout=300)
self.assertEqual(exit_code, 0)
out_files = os.listdir(self.output_dir)
self.assertEqual(out_files, ['out_420001.exr'])
from golem.core import common
from golem.resource import dirmanager
import os
BLENDER_CROP_TEMPLATE_PATH = dirmanager.find_task_script(
os.path.join(common.get_golem_path(), 'apps', 'blender'),
"blendercrop.py.template")
if BLENDER_CROP_TEMPLATE_PATH is None:
raise IOError(
None, 'Template file not found: %s' %
os.path.join(common.get_golem_path(), 'apps', 'blender'))
def generate_blender_crop_file(resolution, borders_x, borders_y,
use_compositing, samples):
with open(BLENDER_CROP_TEMPLATE_PATH) as f:
contents = f.read()
contents %= {
'resolution_x': resolution[0],
'resolution_y': resolution[1],
'border_min_x': borders_x[0],
'border_max_x': borders_x[1],
'border_min_y': borders_y[0],
'border_max_y': borders_y[1],
'use_compositing': use_compositing,
'samples': samples
}
return contents