def _make_render_tasks(self, job, parent_task_id: typing.Optional[bson.ObjectId]) \ -> typing.Tuple[typing.List[bson.ObjectId], typing.List[bson.ObjectId]]: """Creates the render tasks for this job. :returns: two lists of task IDs: (all tasks, parent tasks for next command) """ from flamenco.utils import iter_frame_range, frame_range_merge job_settings = job['settings'] parent_task_ids = [parent_task_id] if parent_task_id else None task_ids = [] for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']): frame_range = frame_range_merge(chunk_frames) frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True) task_cmds = [ commands.BlenderRender( blender_cmd=job_settings.get('blender_cmd', '{blender}'), filepath=job_settings['filepath'], format=job_settings.get('format'), render_output=str(self.render_dir / self.render_output.name), frames=frame_range_bstyle) ] name = 'blender-render-%s' % frame_range task_ids.append(self._create_task(job, task_cmds, name, 'blender-render', parents=parent_task_ids)) return task_ids, task_ids
def _make_progressive_render_tasks(self, job, name_fmt, parents, cycles_num_chunks: int, cycles_chunk_start: int, cycles_chunk_end: int, frame_chunk_size: int, task_priority: int): """Creates the render tasks for this job. :param parents: either a list of parents, one for each task, or a single parent used for all tasks. :returns: created task IDs, one render task per frame chunk. :rtype: list """ from bson import ObjectId from flamenco.utils import iter_frame_range, frame_range_merge job_settings = job['settings'] task_ids = [] frame_chunk_iter = iter_frame_range(job_settings['frames'], frame_chunk_size) for chunk_idx, chunk_frames in enumerate(frame_chunk_iter): frame_range = frame_range_merge(chunk_frames) frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True) name = name_fmt % frame_range render_output = self._render_output(cycles_chunk_start, cycles_chunk_end) task_cmds = [ commands.BlenderRenderProgressive( blender_cmd=job_settings['blender_cmd'], filepath=job_settings['filepath'], format=job_settings.get('format'), # Don't render to actual render output, but to an intermediate file. render_output=str(render_output), frames=frame_range_bstyle, cycles_num_chunks=cycles_num_chunks, cycles_chunk_start=cycles_chunk_start, cycles_chunk_end=cycles_chunk_end, ) ] if isinstance(parents, list): parent_task_id = parents[chunk_idx] else: parent_task_id = parents if not isinstance(parent_task_id, ObjectId): raise TypeError('parents should be list of ObjectIds or ObjectId, not %s' % parents) task_id = self._create_task( job, task_cmds, name, 'blender-render', parents=[parent_task_id], priority=task_priority) task_ids.append(task_id) return task_ids
def _make_render_tasks(self, job) -> typing.List[bson.ObjectId]: """Creates the render tasks for this job. :returns: the list of task IDs. """ from flamenco.utils import iter_frame_range, frame_range_merge job_settings = job['settings'] task_ids = [] for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']): frame_range = frame_range_merge(chunk_frames) frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True) task_cmds = [ commands.BlenderRender( blender_cmd=job_settings['blender_cmd'], filepath=job_settings['filepath'], format=job_settings.get('format'), render_output=str(self.render_dir / self.render_output.name), frames=frame_range_bstyle) ] name = 'blender-render-%s' % frame_range task_ids.append( self._create_task(job, task_cmds, name, 'blender-render')) return task_ids
def _make_render_tasks(self, job: dict, render_parent_tid: ObjectId) \ -> typing.Tuple[typing.List[ObjectId], typing.List[ObjectId]]: """Creates the render tasks for this job. :returns: the list of task IDs. """ from flamenco.utils import iter_frame_range, frame_range_merge job_settings = job['settings'] task_ids = [] parent_task_ids = [] for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']): frame_range = frame_range_merge(chunk_frames) frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True) first_frame = chunk_frames[0] last_frame = chunk_frames[-1] chunk_name = 'chunk-%05d-%05d' % (first_frame, last_frame) render_output = self.frames_dir / chunk_name / '######.jpg' # Export to frames. task_cmds = [ commands.BlenderRender( blender_cmd=job_settings.get('blender_cmd', '{blender}'), filepath=job_settings['filepath'], format='JPEG', render_output=str(render_output), frames=frame_range_bstyle, ) ] name = 'frame-chunk-%s' % frame_range render_task_id = self._create_task(job, task_cmds, name, 'blender-render', parents=[render_parent_tid]) task_ids.append(render_task_id) # Encode frames to video. file_extension = job_settings['output_file_extension'] task_cmds = [ commands.CreateVideo( input_files=str(render_output.with_name('*.jpg')), output_file=str(self.frames_dir / (chunk_name + file_extension)), fps=job_settings['fps'], ) ] name = 'video-chunk-%s' % frame_range encoding_task_id = self._create_task(job, task_cmds, name, 'video-encoding', parents=[render_task_id]) task_ids.append(encoding_task_id) parent_task_ids.append(encoding_task_id) return task_ids, parent_task_ids
def test_frame_range_merge(self): from flamenco.utils import frame_range_merge self.assertEqual('', frame_range_merge(None)) self.assertEqual('', frame_range_merge([])) self.assertEqual('1', frame_range_merge([1])) self.assertEqual('18,20,21', frame_range_merge([18, 20, 21])) self.assertEqual('18,20,21,23-25', frame_range_merge([18, 20, 21, 23, 24, 25])) self.assertEqual('1-3', frame_range_merge([1, 2, 3])) self.assertEqual('51,66-103', frame_range_merge([51] + list(range(66, 104)))) self.assertEqual( '0,531443,5315886,9999993414-9999993416', frame_range_merge( [0, 531443, 5315886, 9999993414, 9999993415, 9999993416]))
def _make_render_tasks(self, job, parent_task_id: typing.Optional[bson.ObjectId]) \ -> typing.Tuple[typing.List[bson.ObjectId], typing.List[bson.ObjectId]]: """Creates the render tasks for this job. :returns: two lists of task IDs: (all tasks, parent tasks for next command) """ from flamenco.utils import iter_frame_range, frame_range_merge job_settings = job['settings'] parent_task_ids = [parent_task_id] if parent_task_id else None task_ids = [] for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']): frame_range = frame_range_merge(chunk_frames) frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True) task_cmds = [ commands.BlenderRender( blender_cmd=job_settings.get('blender_cmd', '{blender}'), filepath=job_settings['filepath'], format=job_settings.get('format'), render_output=str(self.render_dir / self.render_output.name), frames=frame_range_bstyle) ] name = 'blender-render-%s' % frame_range task_ids.append( self._create_task(job, task_cmds, name, 'blender-render', parents=parent_task_ids)) return task_ids, task_ids
def _compile(self, job: dict): from flamenco.utils import iter_frame_range, frame_range_merge self._log.info('Compiling job %s', job['_id']) job_settings = job['settings'] task_count = 0 for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']): task_cmds = [ commands.Echo(message='Preparing to sleep'), commands.Sleep(time_in_seconds=job_settings['time_in_seconds']), ] name = 'sleep-%s' % frame_range_merge(chunk_frames) self._create_task(job, task_cmds, name, 'sleep') task_count += 1 self._log.info('Created %i tasks for job %s', task_count, job['_id'])
def test_frame_range_merge_blender_style(self): from flamenco.utils import frame_range_merge self.assertEqual('', frame_range_merge(None, blender_style=True)) self.assertEqual('', frame_range_merge([], blender_style=True)) self.assertEqual('1', frame_range_merge([1], blender_style=True)) self.assertEqual('18,20,21', frame_range_merge([18, 20, 21], blender_style=True)) self.assertEqual( '18,20,21,23..25', frame_range_merge([18, 20, 21, 23, 24, 25], blender_style=True)) self.assertEqual('1..3', frame_range_merge([1, 2, 3], blender_style=True)) self.assertEqual( '51,66..103', frame_range_merge([51] + list(range(66, 104)), blender_style=True)) self.assertEqual( '0,531443,5315886,9999993414..9999993416', frame_range_merge( [0, 531443, 5315886, 9999993414, 9999993415, 9999993416], blender_style=True))
def _compile(self, job): from flamenco.utils import iter_frame_range, frame_range_merge self._log.info('Compiling job %s', job['_id']) job_settings = job['settings'] task_count = 0 for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']): task_cmds = [ commands.Echo(message='Preparing to sleep'), commands.Sleep( time_in_seconds=job_settings['time_in_seconds']), ] name = 'sleep-%s' % frame_range_merge(chunk_frames) self._create_task(job, task_cmds, name, 'sleep') task_count += 1 self._log.info('Created %i tasks for job %s', task_count, job['_id'])
def _make_progressive_render_tasks(self, job, name_fmt, parents, cycles_num_chunks: int, cycles_chunk_start: int, cycles_chunk_end: int, frame_chunk_size: int, task_priority: int): """Creates the render tasks for this job. :param parents: either a list of parents, one for each task, or a single parent used for all tasks. :returns: created task IDs, one render task per frame chunk. :rtype: list """ from bson import ObjectId from flamenco.utils import iter_frame_range, frame_range_merge job_settings = job['settings'] task_ids = [] frame_chunk_iter = iter_frame_range(job_settings['frames'], frame_chunk_size) for chunk_idx, chunk_frames in enumerate(frame_chunk_iter): frame_range = frame_range_merge(chunk_frames) frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True) name = name_fmt % frame_range render_output = self._render_output(cycles_chunk_start, cycles_chunk_end) task_cmds = [ commands.BlenderRenderProgressive( blender_cmd=job_settings['blender_cmd'], filepath=job_settings['filepath'], format=job_settings.get('format'), # Don't render to actual render output, but to an intermediate file. render_output=str(render_output), frames=frame_range_bstyle, cycles_num_chunks=cycles_num_chunks, cycles_chunk_start=cycles_chunk_start, cycles_chunk_end=cycles_chunk_end, ) ] if isinstance(parents, list): parent_task_id = parents[chunk_idx] else: parent_task_id = parents if not isinstance(parent_task_id, ObjectId): raise TypeError( 'parents should be list of ObjectIds or ObjectId, not %s' % parents) task_id = self._create_task(job, task_cmds, name, 'blender-render', parents=[parent_task_id], priority=task_priority) task_ids.append(task_id) return task_ids
def _make_merge_tasks(self, job, name_fmt, cycles_chunk_idx, parents1, parents2, cycles_samples_to1, cycles_samples_from2, cycles_samples_to2, task_priority): """Creates merge tasks for each chunk, consisting of merges for each frame. :param cycles_chunk_idx: base-1 sample chunk index """ # Merging cannot happen unless we have at least two chunks assert cycles_chunk_idx >= 2 from flamenco.utils import frame_range_merge task_ids = [] weight1 = cycles_samples_to1 weight2 = cycles_samples_to2 - cycles_samples_from2 + 1 # Replace Blender formatting with Python formatting in render output path if cycles_chunk_idx == 2: # The first merge takes a render output as input1, subsequent ones take merge outputs. # Merging only happens from Cycles chunk 2 (it needs two inputs, hence 2 chunks). input1 = self._render_output(1, cycles_samples_to1) else: input1 = self._merge_output(cycles_samples_to1) input2 = self._render_output(cycles_samples_from2, cycles_samples_to2) output = self._merge_output(cycles_samples_to2) # Construct format strings from the paths. input1_fmt = str(input1).replace('######', '%06i.exr') input2_fmt = str(input2).replace('######', '%06i.exr') output_fmt = str(output).replace('######', '%06i.exr') final_dest_fmt = str(self.render_output).replace('######', '%06i.exr') for chunk_idx, chunk_frames in enumerate(self._iter_frame_chunks()): # Create a merge command for every frame in the chunk. task_cmds = [] for framenr in chunk_frames: intermediate = output_fmt % framenr task_cmds.append( commands.MergeProgressiveRenders( input1=input1_fmt % framenr, input2=input2_fmt % framenr, output=intermediate, weight1=weight1, weight2=weight2, )) task_cmds.append( commands.CopyFile( src=intermediate, dest=final_dest_fmt % framenr, )) name = name_fmt % frame_range_merge(chunk_frames) parent1 = parents1[chunk_idx] parent2 = parents2[chunk_idx] task_id = self._create_task(job, task_cmds, name, 'exr-merge', parents=[parent1, parent2], priority=task_priority) task_ids.append(task_id) return task_ids
def _make_render_tasks(self, job, render_parent_tid: ObjectId) \ -> typing.Tuple[typing.List[ObjectId], typing.List[ObjectId]]: """Creates the render tasks for this job. :returns: the list of task IDs. """ from flamenco.utils import iter_frame_range, frame_range_merge job_settings = job['settings'] task_ids = [] parent_task_ids = [] for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']): frame_range = frame_range_merge(chunk_frames) frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True) first_frame = chunk_frames[0] last_frame = chunk_frames[-1] chunk_name = 'chunk-%05d-%05d' % (first_frame, last_frame) render_output = self.frames_dir / chunk_name / '######.png' # Export to PNG frames. task_cmds = [ commands.BlenderRender( blender_cmd=job_settings.get('blender_cmd', '{blender}'), filepath=job_settings['filepath'], format='PNG', render_output=str(render_output), frames=frame_range_bstyle, ) ] name = 'frame-chunk-%s' % frame_range render_task_id = self._create_task(job, task_cmds, name, 'blender-render', parents=[render_parent_tid]) task_ids.append(render_task_id) # Encode PNG frames to video. file_extension = job_settings['output_file_extension'] task_cmds = [ commands.CreateVideo( input_files=str(render_output.with_name('*.png')), output_file=str(self.frames_dir / (chunk_name + file_extension)), fps=job_settings['fps'], ) ] name = 'video-chunk-%s' % frame_range encoding_task_id = self._create_task(job, task_cmds, name, 'video-encoding', parents=[render_task_id]) task_ids.append(encoding_task_id) parent_task_ids.append(encoding_task_id) return task_ids, parent_task_ids