def start(self, verification_context: VerificationContext, crop_count: int) -> Generator: for i in range(0, crop_count): if self.stopped: break crop = verification_context.get_crop_with_id(str(i)) if not crop: raise Exception("Crop %s not found " % i) left, top, right, bottom = crop.calculate_borders().to_tuple() script_src = generate_blender_crop_file( resolution=(verification_context.subtask_info['res_x'], verification_context.subtask_info['res_y']), borders_x=(left, right), borders_y=(bottom, top), use_compositing=False, samples=verification_context.subtask_info['samples']) task_definition = BlenderReferenceGenerator\ .generate_computational_task_definition( verification_context.subtask_info, script_src) yield self.schedule_crop_job(verification_context, task_definition, i) if not self.stopped: for i in range(0, crop_count): verification_context.finished[i].callback( (self.rendered_crops_results[i][0], self.rendered_crops_results[i][1], self.rendered_crops_results[i][2], i))
def test_crop_file_generation_dummy(self): """Test blender script generation with simplistic template.""" filepath = self.temp_file_name("tmpscene") with open(filepath, 'w') as f: f.write('''%(resolution_x)d %(resolution_y)d %(border_min_x).3f %(border_max_x).3f %(border_min_y).3f %(border_max_y).3f %(use_compositing)r %(samples)d''') # Unfortunatelly on windows you can't open tempfile second time # that's why we are leaving with statement and using delete=False. orig_path = scenefileeditor.BLENDER_CROP_TEMPLATE_PATH scenefileeditor.BLENDER_CROP_TEMPLATE_PATH = filepath try: result = scenefileeditor.generate_blender_crop_file( resolution=(1, 2), borders_x=(3.01, 3.02), borders_y=(4.01, 4.02), use_compositing=True, samples=5) finally: scenefileeditor.BLENDER_CROP_TEMPLATE_PATH = orig_path expected = '''1 2 3.010 3.020 4.010 4.020 True 5''' self.assertEqual(result, expected)
def query_extra_data_for_test_task(self): scene_file = self._get_scene_file_rel_path() script_src = generate_blender_crop_file( resolution=BlenderRenderTask.BLENDER_MIN_BOX, borders_x=(0.0, 1.0), borders_y=(0.0, 1.0), use_compositing=False, samples=BlenderRenderTask.BLENDER_MIN_SAMPLE) extra_data = { "path_root": self.main_scene_dir, "start_task": 1, "end_task": 1, "total_tasks": 1, "outfilebasename": "testresult", "scene_file": scene_file, "script_src": script_src, "frames": [1], "output_format": "PNG" } hash = "{}".format(random.getrandbits(128)) dm = DirManager(self.root_path) self.test_task_res_path = dm.get_task_test_dir(self.header.task_id) logger.debug(self.test_task_res_path) if not os.path.exists(self.test_task_res_path): os.makedirs(self.test_task_res_path) return self._new_compute_task_def(hash, extra_data, 0)
def test_crop_file_generation_full(self): """Mocks blender by providing bpy and tests wether generated script acted as expected.""" resolution = (1, 2) borders_x = (3.01, 3.02) borders_y = (4.01, 4.02) use_compositing = True expected_attributes = { 'resolution_x': resolution[0], 'resolution_y': resolution[1], 'border_min_x': borders_x[0], 'border_max_x': borders_x[1], 'border_min_y': borders_y[0], 'border_max_y': borders_y[1], 'use_compositing': use_compositing, 'tile_x': 0, 'tile_y': 0, 'resolution_percentage': 100, 'use_border': True, 'use_crop_to_border': True, } result = scenefileeditor.generate_blender_crop_file( resolution=resolution, borders_x=borders_x, borders_y=borders_y, use_compositing=use_compositing) scene_m = mock.MagicMock() scene_m.render = mock.NonCallableMock() bpy_m = mock.MagicMock() bpy_m.data.scenes = [scene_m] bpy_m.ops.render.render.return_value = None bpy_m.ops.file.report_missing_files.return_value = None def hacked_import(*args, **kwargs): if args[0] == 'bpy': return bpy_m return __import__(*args, **kwargs) hacked_builtins = dict(__builtins__) hacked_builtins['__import__'] = hacked_import exec(result, {'__builtins__': hacked_builtins}) # test scene attributes for name in expected_attributes: expected = expected_attributes[name] value = getattr(scene_m.render, name) self.assertEqual( value, expected, 'Value of scene.render.%s expected:%r got:%r' % (name, expected, value)) # test calls bpy_m.ops.render.render.assert_called_once_with() bpy_m.ops.file.report_missing_files.assert_called_once_with()
def test_crop_file_generation_full(self): """Mocks blender by providing bpy and tests wether generated script acted as expected.""" resolution = (1, 2) borders_x = (3.01, 3.02) borders_y = (4.01, 4.02) use_compositing = True samples = 5 expected_attributes = { 'resolution_x': resolution[0], 'resolution_y': resolution[1], 'border_min_x': borders_x[0], 'border_max_x': borders_x[1], 'border_min_y': borders_y[0], 'border_max_y': borders_y[1], 'use_compositing': use_compositing, 'tile_x': 0, 'tile_y': 0, 'resolution_percentage': 100, 'use_border': True, 'use_crop_to_border': True, } result = scenefileeditor.generate_blender_crop_file( resolution=resolution, borders_x=borders_x, borders_y=borders_y, use_compositing=use_compositing, samples=samples) scene_m = mock.MagicMock() scene_m.render = mock.NonCallableMock() bpy_m = mock.MagicMock() bpy_m.context.scene = scene_m bpy_m.ops.render.render.return_value = None bpy_m.ops.file.report_missing_files.return_value = None result = result.replace('import bpy', '') globs = dict(globals()) globs['bpy'] = bpy_m exec(result, globs) # test scene attributes for name in expected_attributes: expected = expected_attributes[name] value = getattr(scene_m.render, name) self.assertEqual( value, expected, 'Value of scene.render.%s expected:%r got:%r' % (name, expected, value)) # test calls bpy_m.ops.render.render.assert_not_called() bpy_m.ops.file.report_missing_files.assert_called_once_with()
def test_blender_job(self): app_dir = os.path.join(get_golem_path(), "apps", "blender") task_script = find_task_script(app_dir, "docker_blendertask.py") with open(task_script) as f: task_script_src = f.read() # prepare dummy crop script from apps.blender.resources.scenefileeditor import generate_blender_crop_file crop_script_contents = generate_blender_crop_file( resolution=(800, 600), borders_x=(0, 1), borders_y=(0, 1), use_compositing=True, ) # copy the scene file to the resources dir benchmarks_dir = path.join(get_golem_path(), path.normpath("apps/blender/benchmark/")) scene_files = glob.glob(path.join(benchmarks_dir, "**/*.blend")) if len(scene_files) == 0: self.fail("No .blend files available") shutil.copy(scene_files[0], self.resources_dir) params = { "outfilebasename": "out", "scene_file": DockerJob.RESOURCES_DIR + "/" + path.basename(scene_files[0]), "script_src": crop_script_contents, "start_task": 42, "end_task": 42, "output_format": "EXR", "frames": [1], } with self._create_test_job(script=task_script_src, params=params) as job: job.start() exit_code = job.wait() self.assertEqual(exit_code, 0) out_files = os.listdir(self.output_dir) self.assertEqual(out_files, ['out_420001.exr'])
def extra_data(self): return { "path_root": '', "start_task": 1, "end_task": 1, "total_tasks": 1, "outfilebasename": 'test task', "scene_file": '/golem/resources/wlochaty3.blend', "script_src": generate_blender_crop_file((320, 240), (0.0, 1.0), (0.0, 1.0), False, 0), "frames": [1], "output_format": 'PNG', }
def test_blender_job(self): app_dir = os.path.join(get_golem_path(), "apps", "blender") task_script = find_task_script(app_dir, "docker_blendertask.py") with open(task_script) as f: task_script_src = f.read() # prepare dummy crop script crop_script_contents = generate_blender_crop_file(resolution=(800, 600), borders_x=(0, 1), borders_y=(0, 1), use_compositing=True, samples=5) # copy the scene file to the resources dir scene_file = pathlib.Path(get_golem_path()) scene_file /= "apps/blender/benchmark/test_task/cube.blend" shutil.copy(str(scene_file), self.resources_dir) dest_scene_file = pathlib.PurePosixPath(DockerJob.RESOURCES_DIR) dest_scene_file /= scene_file.name params = { "outfilebasename": "out", "scene_file": str(dest_scene_file), "script_src": crop_script_contents, "start_task": 42, "end_task": 42, "output_format": "EXR", "frames": [1], } with self._create_test_job(script=task_script_src, params=params) \ as job: job.start() exit_code = job.wait(timeout=300) self.assertEqual(exit_code, 0) out_files = os.listdir(self.output_dir) self.assertEqual(out_files, ['out_420001.exr'])
def change_scope(self, subtask_id, start_box, tr_file, subtask_info): extra_data, _ = super(BlenderVerificator, self).change_scope(subtask_id, start_box, tr_file, subtask_info) min_x = start_box[0] / self.res_x max_x = (start_box[0] + self.verification_options.box_size[0] + 1) / self.res_x shift_y = (extra_data['start_task'] - 1) * (self.res_y / extra_data['total_tasks']) start_y = start_box[1] + shift_y max_y = (self.res_y - start_y) / self.res_y shift_y = start_y + self.verification_options.box_size[1] + 1 min_y = max((self.res_y - shift_y) / self.res_y, 0.0) min_y = max(min_y, 0) script_src = generate_blender_crop_file( resolution=(self.res_x, self.res_y), borders_x=(min_x, max_x), borders_y=(min_y, max_y), use_compositing=self.compositing) extra_data['script_src'] = script_src extra_data['output_format'] = self.output_format return extra_data, (0, 0)
def query_extra_data_for_test_task(self): scene_file = self._get_scene_file_rel_path() if self.use_frames: frames = [self.frames[0]] if len(self.frames) > 1: frames.append(max(self.frames)) else: frames = [1] script_src = generate_blender_crop_file( resolution=(8, 8), borders_x=(0.0, 1.0), borders_y=(0.0, 1.0), use_compositing=self.compositing) extra_data = { "path_root": self.main_scene_dir, "start_task": 1, "end_task": 1, "total_tasks": self.total_tasks, "outfilebasename": self.outfilebasename, "scene_file": scene_file, "script_src": script_src, "frames": frames, "output_format": self.output_format } hash = "{}".format(random.getrandbits(128)) self.test_task_res_path = get_test_task_path(self.root_path) logger.debug(self.test_task_res_path) if not os.path.exists(self.test_task_res_path): os.makedirs(self.test_task_res_path) return self._new_compute_task_def(hash, extra_data, None, 0)
def query_extra_data(self, perf_index, num_cores=0, node_id=None, node_name=None): verdict = self._accept_client(node_id) if verdict != AcceptClientVerdict.ACCEPTED: should_wait = verdict == AcceptClientVerdict.SHOULD_WAIT if should_wait: logger.warning("Waiting for results from {}".format(node_name)) else: logger.warning( "Client {} banned from this task".format(node_name)) return self.ExtraData(should_wait=should_wait) start_task, end_task = self._get_next_task() scene_file = self._get_scene_file_rel_path() if self.use_frames: frames, parts = self._choose_frames(self.frames, start_task, self.total_tasks) else: frames = [1] parts = 1 if not self.use_frames: min_y, max_y = self._get_min_max_y(start_task) elif parts > 1: min_y = (parts - self._count_part(start_task, parts)) * (1.0 / parts) max_y = (parts - self._count_part(start_task, parts) + 1) * (1.0 / parts) else: min_y = 0.0 max_y = 1.0 script_src = generate_blender_crop_file( resolution=(self.res_x, self.res_y), borders_x=(0.0, 1.0), borders_y=(min_y, max_y), use_compositing=self.compositing) extra_data = { "path_root": self.main_scene_dir, "start_task": start_task, "end_task": end_task, "total_tasks": self.total_tasks, "outfilebasename": self.outfilebasename, "scene_file": scene_file, "script_src": script_src, "frames": frames, "output_format": self.output_format } hash = "{}".format(random.getrandbits(128)) self.subtasks_given[hash] = extra_data self.subtasks_given[hash]['status'] = SubtaskStatus.starting self.subtasks_given[hash]['perf'] = perf_index self.subtasks_given[hash]['node_id'] = node_id self.subtasks_given[hash]['parts'] = parts if not self.use_frames: self._update_task_preview() else: self._update_frame_task_preview() ctd = self._new_compute_task_def(hash, extra_data, None, perf_index) return self.ExtraData(ctd=ctd)
def query_extra_data(self, perf_index: float, num_cores: int = 0, node_id: Optional[str] = None, node_name: Optional[str] = None) \ -> FrameRenderingTask.ExtraData: start_task, end_task = self._get_next_task() scene_file = self._get_scene_file_rel_path() if self.use_frames: frames, parts = self._choose_frames(self.frames, start_task, self.total_tasks) else: frames = self.frames or [1] parts = 1 if not self.use_frames: min_y, max_y = self._get_min_max_y(start_task) elif parts > 1: min_y = (parts - self._count_part(start_task, parts)) * (1.0 / parts) max_y = (parts - self._count_part(start_task, parts) + 1) * (1.0 / parts) else: min_y = 0.0 max_y = 1.0 # Blender is using single precision math, we use numpy to emulate this. # Send already converted values to blender. min_y = numpy.float32(min_y) max_y = numpy.float32(max_y) script_src = generate_blender_crop_file( resolution=(self.res_x, self.res_y), borders_x=(0.0, 1.0), borders_y=(min_y, max_y), use_compositing=self.compositing, samples=self.samples) extra_data = { "path_root": self.main_scene_dir, "start_task": start_task, "end_task": end_task, "total_tasks": self.total_tasks, "outfilebasename": self.outfilebasename, "scene_file": scene_file, "script_src": script_src, "frames": frames, "output_format": self.output_format, } subtask_id = self.create_subtask_id() logger.debug( 'Created new subtask for task. ' 'task_id=%s, subtask_id=%s, node_id=%s', self.header.task_id, subtask_id, short_node_id(node_id or '')) self.subtasks_given[subtask_id] = copy(extra_data) self.subtasks_given[subtask_id]['subtask_id'] = subtask_id self.subtasks_given[subtask_id]['status'] = SubtaskStatus.starting self.subtasks_given[subtask_id]['node_id'] = node_id self.subtasks_given[subtask_id]['parts'] = parts self.subtasks_given[subtask_id]['res_x'] = self.res_x self.subtasks_given[subtask_id]['res_y'] = self.res_y self.subtasks_given[subtask_id]['samples'] = self.samples self.subtasks_given[subtask_id]['use_frames'] = self.use_frames self.subtasks_given[subtask_id]['all_frames'] = self.frames self.subtasks_given[subtask_id]['crop_window'] = (0.0, 1.0, min_y, max_y) self.subtasks_given[subtask_id]['subtask_timeout'] = \ self.header.subtask_timeout self.subtasks_given[subtask_id]['tmp_dir'] = self.tmp_dir # FIXME issue #1955 part = self._count_part(start_task, parts) for frame in frames: frame_key = to_unicode(frame) state = self.frames_state[frame_key] state.status = TaskStatus.computing state.started = state.started or time.time() self.frames_subtasks[frame_key][part - 1] = subtask_id if not self.use_frames: self._update_task_preview() else: self._update_frame_task_preview() ctd = self._new_compute_task_def(subtask_id, extra_data, perf_index=perf_index) self.subtasks_given[subtask_id]['ctd'] = ctd return self.ExtraData(ctd=ctd)