def start(self, verification_context: VerificationContext,
crop_count: int) -> Generator:
for i in range(0, crop_count):
if self.stopped:
break
crop = verification_context.get_crop_with_id(str(i))
if not crop:
raise Exception("Crop %s not found " % i)
left, top, right, bottom = crop.calculate_borders().to_tuple()
script_src = generate_blender_crop_file(
resolution=(verification_context.subtask_info['res_x'],
verification_context.subtask_info['res_y']),
borders_x=(left, right),
borders_y=(bottom, top),
use_compositing=False,
samples=verification_context.subtask_info['samples'])
task_definition = BlenderReferenceGenerator\
.generate_computational_task_definition(
verification_context.subtask_info,
script_src)
yield self.schedule_crop_job(verification_context, task_definition,
i)
if not self.stopped:
for i in range(0, crop_count):
verification_context.finished[i].callback(
(self.rendered_crops_results[i][0],
self.rendered_crops_results[i][1],
self.rendered_crops_results[i][2], i))
def test_crop_file_generation_dummy(self):
"""Test blender script generation with simplistic template."""
filepath = self.temp_file_name("tmpscene")
with open(filepath, 'w') as f:
f.write('''%(resolution_x)d
%(resolution_y)d
%(border_min_x).3f
%(border_max_x).3f
%(border_min_y).3f
%(border_max_y).3f
%(use_compositing)r
%(samples)d''')
# Unfortunatelly on windows you can't open tempfile second time
# that's why we are leaving with statement and using delete=False.
orig_path = scenefileeditor.BLENDER_CROP_TEMPLATE_PATH
scenefileeditor.BLENDER_CROP_TEMPLATE_PATH = filepath
try:
result = scenefileeditor.generate_blender_crop_file(
resolution=(1, 2),
borders_x=(3.01, 3.02),
borders_y=(4.01, 4.02),
use_compositing=True,
samples=5)
finally:
scenefileeditor.BLENDER_CROP_TEMPLATE_PATH = orig_path
expected = '''1
2
3.010
3.020
4.010
4.020
True
5'''
self.assertEqual(result, expected)
def query_extra_data_for_test_task(self):
scene_file = self._get_scene_file_rel_path()
script_src = generate_blender_crop_file(
resolution=BlenderRenderTask.BLENDER_MIN_BOX,
borders_x=(0.0, 1.0),
borders_y=(0.0, 1.0),
use_compositing=False,
samples=BlenderRenderTask.BLENDER_MIN_SAMPLE)
extra_data = {
"path_root": self.main_scene_dir,
"start_task": 1,
"end_task": 1,
"total_tasks": 1,
"outfilebasename": "testresult",
"scene_file": scene_file,
"script_src": script_src,
"frames": [1],
"output_format": "PNG"
}
hash = "{}".format(random.getrandbits(128))
dm = DirManager(self.root_path)
self.test_task_res_path = dm.get_task_test_dir(self.header.task_id)
logger.debug(self.test_task_res_path)
if not os.path.exists(self.test_task_res_path):
os.makedirs(self.test_task_res_path)
return self._new_compute_task_def(hash, extra_data, 0)
def test_crop_file_generation_full(self):
"""Mocks blender by providing bpy and tests wether generated script acted as expected."""
resolution = (1, 2)
borders_x = (3.01, 3.02)
borders_y = (4.01, 4.02)
use_compositing = True
expected_attributes = {
'resolution_x': resolution[0],
'resolution_y': resolution[1],
'border_min_x': borders_x[0],
'border_max_x': borders_x[1],
'border_min_y': borders_y[0],
'border_max_y': borders_y[1],
'use_compositing': use_compositing,
'tile_x': 0,
'tile_y': 0,
'resolution_percentage': 100,
'use_border': True,
'use_crop_to_border': True,
}
result = scenefileeditor.generate_blender_crop_file(
resolution=resolution,
borders_x=borders_x,
borders_y=borders_y,
use_compositing=use_compositing)
scene_m = mock.MagicMock()
scene_m.render = mock.NonCallableMock()
bpy_m = mock.MagicMock()
bpy_m.data.scenes = [scene_m]
bpy_m.ops.render.render.return_value = None
bpy_m.ops.file.report_missing_files.return_value = None
def hacked_import(*args, **kwargs):
if args[0] == 'bpy':
return bpy_m
return __import__(*args, **kwargs)
hacked_builtins = dict(__builtins__)
hacked_builtins['__import__'] = hacked_import
exec(result, {'__builtins__': hacked_builtins})
# test scene attributes
for name in expected_attributes:
expected = expected_attributes[name]
value = getattr(scene_m.render, name)
self.assertEqual(
value, expected,
'Value of scene.render.%s expected:%r got:%r' %
(name, expected, value))
# test calls
bpy_m.ops.render.render.assert_called_once_with()
bpy_m.ops.file.report_missing_files.assert_called_once_with()
def test_crop_file_generation_full(self):
"""Mocks blender by providing bpy and tests wether generated script
acted as expected."""
resolution = (1, 2)
borders_x = (3.01, 3.02)
borders_y = (4.01, 4.02)
use_compositing = True
samples = 5
expected_attributes = {
'resolution_x': resolution[0],
'resolution_y': resolution[1],
'border_min_x': borders_x[0],
'border_max_x': borders_x[1],
'border_min_y': borders_y[0],
'border_max_y': borders_y[1],
'use_compositing': use_compositing,
'tile_x': 0,
'tile_y': 0,
'resolution_percentage': 100,
'use_border': True,
'use_crop_to_border': True,
}
result = scenefileeditor.generate_blender_crop_file(
resolution=resolution,
borders_x=borders_x,
borders_y=borders_y,
use_compositing=use_compositing,
samples=samples)
scene_m = mock.MagicMock()
scene_m.render = mock.NonCallableMock()
bpy_m = mock.MagicMock()
bpy_m.context.scene = scene_m
bpy_m.ops.render.render.return_value = None
bpy_m.ops.file.report_missing_files.return_value = None
result = result.replace('import bpy', '')
globs = dict(globals())
globs['bpy'] = bpy_m
exec(result, globs)
# test scene attributes
for name in expected_attributes:
expected = expected_attributes[name]
value = getattr(scene_m.render, name)
self.assertEqual(
value, expected,
'Value of scene.render.%s expected:%r got:%r' %
(name, expected, value))
# test calls
bpy_m.ops.render.render.assert_not_called()
bpy_m.ops.file.report_missing_files.assert_called_once_with()
def test_blender_job(self):
app_dir = os.path.join(get_golem_path(), "apps", "blender")
task_script = find_task_script(app_dir, "docker_blendertask.py")
with open(task_script) as f:
task_script_src = f.read()
# prepare dummy crop script
from apps.blender.resources.scenefileeditor import generate_blender_crop_file
crop_script_contents = generate_blender_crop_file(
resolution=(800, 600),
borders_x=(0, 1),
borders_y=(0, 1),
use_compositing=True,
)
# copy the scene file to the resources dir
benchmarks_dir = path.join(get_golem_path(),
path.normpath("apps/blender/benchmark/"))
scene_files = glob.glob(path.join(benchmarks_dir, "**/*.blend"))
if len(scene_files) == 0:
self.fail("No .blend files available")
shutil.copy(scene_files[0], self.resources_dir)
params = {
"outfilebasename":
"out",
"scene_file":
DockerJob.RESOURCES_DIR + "/" + path.basename(scene_files[0]),
"script_src":
crop_script_contents,
"start_task":
42,
"end_task":
42,
"output_format":
"EXR",
"frames": [1],
}
with self._create_test_job(script=task_script_src,
params=params) as job:
job.start()
exit_code = job.wait()
self.assertEqual(exit_code, 0)
out_files = os.listdir(self.output_dir)
self.assertEqual(out_files, ['out_420001.exr'])
def extra_data(self):
return {
"path_root":
'',
"start_task":
1,
"end_task":
1,
"total_tasks":
1,
"outfilebasename":
'test task',
"scene_file":
'/golem/resources/wlochaty3.blend',
"script_src":
generate_blender_crop_file((320, 240), (0.0, 1.0), (0.0, 1.0),
False, 0),
"frames": [1],
"output_format":
'PNG',
}
def test_blender_job(self):
app_dir = os.path.join(get_golem_path(), "apps", "blender")
task_script = find_task_script(app_dir, "docker_blendertask.py")
with open(task_script) as f:
task_script_src = f.read()
# prepare dummy crop script
crop_script_contents = generate_blender_crop_file(resolution=(800,
600),
borders_x=(0, 1),
borders_y=(0, 1),
use_compositing=True,
samples=5)
# copy the scene file to the resources dir
scene_file = pathlib.Path(get_golem_path())
scene_file /= "apps/blender/benchmark/test_task/cube.blend"
shutil.copy(str(scene_file), self.resources_dir)
dest_scene_file = pathlib.PurePosixPath(DockerJob.RESOURCES_DIR)
dest_scene_file /= scene_file.name
params = {
"outfilebasename": "out",
"scene_file": str(dest_scene_file),
"script_src": crop_script_contents,
"start_task": 42,
"end_task": 42,
"output_format": "EXR",
"frames": [1],
}
with self._create_test_job(script=task_script_src, params=params) \
as job:
job.start()
exit_code = job.wait(timeout=300)
self.assertEqual(exit_code, 0)
out_files = os.listdir(self.output_dir)
self.assertEqual(out_files, ['out_420001.exr'])
def change_scope(self, subtask_id, start_box, tr_file, subtask_info):
extra_data, _ = super(BlenderVerificator,
self).change_scope(subtask_id, start_box,
tr_file, subtask_info)
min_x = start_box[0] / self.res_x
max_x = (start_box[0] + self.verification_options.box_size[0] +
1) / self.res_x
shift_y = (extra_data['start_task'] - 1) * (self.res_y /
extra_data['total_tasks'])
start_y = start_box[1] + shift_y
max_y = (self.res_y - start_y) / self.res_y
shift_y = start_y + self.verification_options.box_size[1] + 1
min_y = max((self.res_y - shift_y) / self.res_y, 0.0)
min_y = max(min_y, 0)
script_src = generate_blender_crop_file(
resolution=(self.res_x, self.res_y),
borders_x=(min_x, max_x),
borders_y=(min_y, max_y),
use_compositing=self.compositing)
extra_data['script_src'] = script_src
extra_data['output_format'] = self.output_format
return extra_data, (0, 0)
def query_extra_data_for_test_task(self):
scene_file = self._get_scene_file_rel_path()
if self.use_frames:
frames = [self.frames[0]]
if len(self.frames) > 1:
frames.append(max(self.frames))
else:
frames = [1]
script_src = generate_blender_crop_file(
resolution=(8, 8),
borders_x=(0.0, 1.0),
borders_y=(0.0, 1.0),
use_compositing=self.compositing)
extra_data = {
"path_root": self.main_scene_dir,
"start_task": 1,
"end_task": 1,
"total_tasks": self.total_tasks,
"outfilebasename": self.outfilebasename,
"scene_file": scene_file,
"script_src": script_src,
"frames": frames,
"output_format": self.output_format
}
hash = "{}".format(random.getrandbits(128))
self.test_task_res_path = get_test_task_path(self.root_path)
logger.debug(self.test_task_res_path)
if not os.path.exists(self.test_task_res_path):
os.makedirs(self.test_task_res_path)
return self._new_compute_task_def(hash, extra_data, None, 0)
def query_extra_data(self,
perf_index,
num_cores=0,
node_id=None,
node_name=None):
verdict = self._accept_client(node_id)
if verdict != AcceptClientVerdict.ACCEPTED:
should_wait = verdict == AcceptClientVerdict.SHOULD_WAIT
if should_wait:
logger.warning("Waiting for results from {}".format(node_name))
else:
logger.warning(
"Client {} banned from this task".format(node_name))
return self.ExtraData(should_wait=should_wait)
start_task, end_task = self._get_next_task()
scene_file = self._get_scene_file_rel_path()
if self.use_frames:
frames, parts = self._choose_frames(self.frames, start_task,
self.total_tasks)
else:
frames = [1]
parts = 1
if not self.use_frames:
min_y, max_y = self._get_min_max_y(start_task)
elif parts > 1:
min_y = (parts - self._count_part(start_task, parts)) * (1.0 /
parts)
max_y = (parts - self._count_part(start_task, parts) + 1) * (1.0 /
parts)
else:
min_y = 0.0
max_y = 1.0
script_src = generate_blender_crop_file(
resolution=(self.res_x, self.res_y),
borders_x=(0.0, 1.0),
borders_y=(min_y, max_y),
use_compositing=self.compositing)
extra_data = {
"path_root": self.main_scene_dir,
"start_task": start_task,
"end_task": end_task,
"total_tasks": self.total_tasks,
"outfilebasename": self.outfilebasename,
"scene_file": scene_file,
"script_src": script_src,
"frames": frames,
"output_format": self.output_format
}
hash = "{}".format(random.getrandbits(128))
self.subtasks_given[hash] = extra_data
self.subtasks_given[hash]['status'] = SubtaskStatus.starting
self.subtasks_given[hash]['perf'] = perf_index
self.subtasks_given[hash]['node_id'] = node_id
self.subtasks_given[hash]['parts'] = parts
if not self.use_frames:
self._update_task_preview()
else:
self._update_frame_task_preview()
ctd = self._new_compute_task_def(hash, extra_data, None, perf_index)
return self.ExtraData(ctd=ctd)
def query_extra_data(self, perf_index: float, num_cores: int = 0,
node_id: Optional[str] = None,
node_name: Optional[str] = None) \
-> FrameRenderingTask.ExtraData:
start_task, end_task = self._get_next_task()
scene_file = self._get_scene_file_rel_path()
if self.use_frames:
frames, parts = self._choose_frames(self.frames, start_task,
self.total_tasks)
else:
frames = self.frames or [1]
parts = 1
if not self.use_frames:
min_y, max_y = self._get_min_max_y(start_task)
elif parts > 1:
min_y = (parts - self._count_part(start_task, parts)) * (1.0 /
parts)
max_y = (parts - self._count_part(start_task, parts) + 1) * (1.0 /
parts)
else:
min_y = 0.0
max_y = 1.0
# Blender is using single precision math, we use numpy to emulate this.
# Send already converted values to blender.
min_y = numpy.float32(min_y)
max_y = numpy.float32(max_y)
script_src = generate_blender_crop_file(
resolution=(self.res_x, self.res_y),
borders_x=(0.0, 1.0),
borders_y=(min_y, max_y),
use_compositing=self.compositing,
samples=self.samples)
extra_data = {
"path_root": self.main_scene_dir,
"start_task": start_task,
"end_task": end_task,
"total_tasks": self.total_tasks,
"outfilebasename": self.outfilebasename,
"scene_file": scene_file,
"script_src": script_src,
"frames": frames,
"output_format": self.output_format,
}
subtask_id = self.create_subtask_id()
logger.debug(
'Created new subtask for task. '
'task_id=%s, subtask_id=%s, node_id=%s', self.header.task_id,
subtask_id, short_node_id(node_id or ''))
self.subtasks_given[subtask_id] = copy(extra_data)
self.subtasks_given[subtask_id]['subtask_id'] = subtask_id
self.subtasks_given[subtask_id]['status'] = SubtaskStatus.starting
self.subtasks_given[subtask_id]['node_id'] = node_id
self.subtasks_given[subtask_id]['parts'] = parts
self.subtasks_given[subtask_id]['res_x'] = self.res_x
self.subtasks_given[subtask_id]['res_y'] = self.res_y
self.subtasks_given[subtask_id]['samples'] = self.samples
self.subtasks_given[subtask_id]['use_frames'] = self.use_frames
self.subtasks_given[subtask_id]['all_frames'] = self.frames
self.subtasks_given[subtask_id]['crop_window'] = (0.0, 1.0, min_y,
max_y)
self.subtasks_given[subtask_id]['subtask_timeout'] = \
self.header.subtask_timeout
self.subtasks_given[subtask_id]['tmp_dir'] = self.tmp_dir
# FIXME issue #1955
part = self._count_part(start_task, parts)
for frame in frames:
frame_key = to_unicode(frame)
state = self.frames_state[frame_key]
state.status = TaskStatus.computing
state.started = state.started or time.time()
self.frames_subtasks[frame_key][part - 1] = subtask_id
if not self.use_frames:
self._update_task_preview()
else:
self._update_frame_task_preview()
ctd = self._new_compute_task_def(subtask_id,
extra_data,
perf_index=perf_index)
self.subtasks_given[subtask_id]['ctd'] = ctd
return self.ExtraData(ctd=ctd)
