def _make_render_tasks(self, job, parent_task_id: typing.Optional[bson.ObjectId]) \
-> typing.Tuple[typing.List[bson.ObjectId], typing.List[bson.ObjectId]]:
"""Creates the render tasks for this job.
:returns: two lists of task IDs: (all tasks, parent tasks for next command)
"""
from flamenco.utils import iter_frame_range, frame_range_merge
job_settings = job['settings']
parent_task_ids = [parent_task_id] if parent_task_id else None
task_ids = []
for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']):
frame_range = frame_range_merge(chunk_frames)
frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True)
task_cmds = [
commands.BlenderRender(
blender_cmd=job_settings.get('blender_cmd', '{blender}'),
filepath=job_settings['filepath'],
format=job_settings.get('format'),
render_output=str(self.render_dir / self.render_output.name),
frames=frame_range_bstyle)
]
name = 'blender-render-%s' % frame_range
task_ids.append(self._create_task(job, task_cmds, name, 'blender-render',
parents=parent_task_ids))
return task_ids, task_ids
def _make_progressive_render_tasks(self,
job, name_fmt, parents,
cycles_num_chunks: int,
cycles_chunk_start: int,
cycles_chunk_end: int,
frame_chunk_size: int,
task_priority: int):
"""Creates the render tasks for this job.
:param parents: either a list of parents, one for each task, or a
single parent used for all tasks.
:returns: created task IDs, one render task per frame chunk.
:rtype: list
"""
from bson import ObjectId
from flamenco.utils import iter_frame_range, frame_range_merge
job_settings = job['settings']
task_ids = []
frame_chunk_iter = iter_frame_range(job_settings['frames'], frame_chunk_size)
for chunk_idx, chunk_frames in enumerate(frame_chunk_iter):
frame_range = frame_range_merge(chunk_frames)
frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True)
name = name_fmt % frame_range
render_output = self._render_output(cycles_chunk_start, cycles_chunk_end)
task_cmds = [
commands.BlenderRenderProgressive(
blender_cmd=job_settings['blender_cmd'],
filepath=job_settings['filepath'],
format=job_settings.get('format'),
# Don't render to actual render output, but to an intermediate file.
render_output=str(render_output),
frames=frame_range_bstyle,
cycles_num_chunks=cycles_num_chunks,
cycles_chunk_start=cycles_chunk_start,
cycles_chunk_end=cycles_chunk_end,
)
]
if isinstance(parents, list):
parent_task_id = parents[chunk_idx]
else:
parent_task_id = parents
if not isinstance(parent_task_id, ObjectId):
raise TypeError('parents should be list of ObjectIds or ObjectId, not %s' % parents)
task_id = self._create_task(
job, task_cmds, name, 'blender-render',
parents=[parent_task_id],
priority=task_priority)
task_ids.append(task_id)
return task_ids
def _make_render_tasks(self, job) -> typing.List[bson.ObjectId]:
"""Creates the render tasks for this job.
:returns: the list of task IDs.
"""
from flamenco.utils import iter_frame_range, frame_range_merge
job_settings = job['settings']
task_ids = []
for chunk_frames in iter_frame_range(job_settings['frames'],
job_settings['chunk_size']):
frame_range = frame_range_merge(chunk_frames)
frame_range_bstyle = frame_range_merge(chunk_frames,
blender_style=True)
task_cmds = [
commands.BlenderRender(
blender_cmd=job_settings['blender_cmd'],
filepath=job_settings['filepath'],
format=job_settings.get('format'),
render_output=str(self.render_dir /
self.render_output.name),
frames=frame_range_bstyle)
]
name = 'blender-render-%s' % frame_range
task_ids.append(
self._create_task(job, task_cmds, name, 'blender-render'))
return task_ids
def _make_render_tasks(self, job: dict, render_parent_tid: ObjectId) \
-> typing.Tuple[typing.List[ObjectId], typing.List[ObjectId]]:
"""Creates the render tasks for this job.
:returns: the list of task IDs.
"""
from flamenco.utils import iter_frame_range, frame_range_merge
job_settings = job['settings']
task_ids = []
parent_task_ids = []
for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']):
frame_range = frame_range_merge(chunk_frames)
frame_range_bstyle = frame_range_merge(chunk_frames, blender_style=True)
first_frame = chunk_frames[0]
last_frame = chunk_frames[-1]
chunk_name = 'chunk-%05d-%05d' % (first_frame, last_frame)
render_output = self.frames_dir / chunk_name / '######.jpg'
# Export to frames.
task_cmds = [
commands.BlenderRender(
blender_cmd=job_settings.get('blender_cmd', '{blender}'),
filepath=job_settings['filepath'],
format='JPEG',
render_output=str(render_output),
frames=frame_range_bstyle,
)
]
name = 'frame-chunk-%s' % frame_range
render_task_id = self._create_task(job, task_cmds, name, 'blender-render',
parents=[render_parent_tid])
task_ids.append(render_task_id)
# Encode frames to video.
file_extension = job_settings['output_file_extension']
task_cmds = [
commands.CreateVideo(
input_files=str(render_output.with_name('*.jpg')),
output_file=str(self.frames_dir / (chunk_name + file_extension)),
fps=job_settings['fps'],
)
]
name = 'video-chunk-%s' % frame_range
encoding_task_id = self._create_task(job, task_cmds, name, 'video-encoding',
parents=[render_task_id])
task_ids.append(encoding_task_id)
parent_task_ids.append(encoding_task_id)
return task_ids, parent_task_ids
def test_frame_range_merge(self):
from flamenco.utils import frame_range_merge
self.assertEqual('', frame_range_merge(None))
self.assertEqual('', frame_range_merge([]))
self.assertEqual('1', frame_range_merge([1]))
self.assertEqual('18,20,21', frame_range_merge([18, 20, 21]))
self.assertEqual('18,20,21,23-25',
frame_range_merge([18, 20, 21, 23, 24, 25]))
self.assertEqual('1-3', frame_range_merge([1, 2, 3]))
self.assertEqual('51,66-103',
frame_range_merge([51] + list(range(66, 104))))
self.assertEqual(
'0,531443,5315886,9999993414-9999993416',
frame_range_merge(
[0, 531443, 5315886, 9999993414, 9999993415, 9999993416]))
def _make_render_tasks(self, job, parent_task_id: typing.Optional[bson.ObjectId]) \
-> typing.Tuple[typing.List[bson.ObjectId], typing.List[bson.ObjectId]]:
"""Creates the render tasks for this job.
:returns: two lists of task IDs: (all tasks, parent tasks for next command)
"""
from flamenco.utils import iter_frame_range, frame_range_merge
job_settings = job['settings']
parent_task_ids = [parent_task_id] if parent_task_id else None
task_ids = []
for chunk_frames in iter_frame_range(job_settings['frames'],
job_settings['chunk_size']):
frame_range = frame_range_merge(chunk_frames)
frame_range_bstyle = frame_range_merge(chunk_frames,
blender_style=True)
task_cmds = [
commands.BlenderRender(
blender_cmd=job_settings.get('blender_cmd', '{blender}'),
filepath=job_settings['filepath'],
format=job_settings.get('format'),
render_output=str(self.render_dir /
self.render_output.name),
frames=frame_range_bstyle)
]
name = 'blender-render-%s' % frame_range
task_ids.append(
self._create_task(job,
task_cmds,
name,
'blender-render',
parents=parent_task_ids))
return task_ids, task_ids
def _compile(self, job: dict):
from flamenco.utils import iter_frame_range, frame_range_merge
self._log.info('Compiling job %s', job['_id'])
job_settings = job['settings']
task_count = 0
for chunk_frames in iter_frame_range(job_settings['frames'], job_settings['chunk_size']):
task_cmds = [
commands.Echo(message='Preparing to sleep'),
commands.Sleep(time_in_seconds=job_settings['time_in_seconds']),
]
name = 'sleep-%s' % frame_range_merge(chunk_frames)
self._create_task(job, task_cmds, name, 'sleep')
task_count += 1
self._log.info('Created %i tasks for job %s', task_count, job['_id'])
def test_frame_range_merge_blender_style(self):
from flamenco.utils import frame_range_merge
self.assertEqual('', frame_range_merge(None, blender_style=True))
self.assertEqual('', frame_range_merge([], blender_style=True))
self.assertEqual('1', frame_range_merge([1], blender_style=True))
self.assertEqual('18,20,21',
frame_range_merge([18, 20, 21], blender_style=True))
self.assertEqual(
'18,20,21,23..25',
frame_range_merge([18, 20, 21, 23, 24, 25], blender_style=True))
self.assertEqual('1..3',
frame_range_merge([1, 2, 3], blender_style=True))
self.assertEqual(
'51,66..103',
frame_range_merge([51] + list(range(66, 104)), blender_style=True))
self.assertEqual(
'0,531443,5315886,9999993414..9999993416',
frame_range_merge(
[0, 531443, 5315886, 9999993414, 9999993415, 9999993416],
blender_style=True))
def _compile(self, job):
from flamenco.utils import iter_frame_range, frame_range_merge
self._log.info('Compiling job %s', job['_id'])
job_settings = job['settings']
task_count = 0
for chunk_frames in iter_frame_range(job_settings['frames'],
job_settings['chunk_size']):
task_cmds = [
commands.Echo(message='Preparing to sleep'),
commands.Sleep(
time_in_seconds=job_settings['time_in_seconds']),
]
name = 'sleep-%s' % frame_range_merge(chunk_frames)
self._create_task(job, task_cmds, name, 'sleep')
task_count += 1
self._log.info('Created %i tasks for job %s', task_count, job['_id'])
def _make_progressive_render_tasks(self, job, name_fmt, parents,
cycles_num_chunks: int,
cycles_chunk_start: int,
cycles_chunk_end: int,
frame_chunk_size: int,
task_priority: int):
"""Creates the render tasks for this job.
:param parents: either a list of parents, one for each task, or a
single parent used for all tasks.
:returns: created task IDs, one render task per frame chunk.
:rtype: list
"""
from bson import ObjectId
from flamenco.utils import iter_frame_range, frame_range_merge
job_settings = job['settings']
task_ids = []
frame_chunk_iter = iter_frame_range(job_settings['frames'],
frame_chunk_size)
for chunk_idx, chunk_frames in enumerate(frame_chunk_iter):
frame_range = frame_range_merge(chunk_frames)
frame_range_bstyle = frame_range_merge(chunk_frames,
blender_style=True)
name = name_fmt % frame_range
render_output = self._render_output(cycles_chunk_start,
cycles_chunk_end)
task_cmds = [
commands.BlenderRenderProgressive(
blender_cmd=job_settings['blender_cmd'],
filepath=job_settings['filepath'],
format=job_settings.get('format'),
# Don't render to actual render output, but to an intermediate file.
render_output=str(render_output),
frames=frame_range_bstyle,
cycles_num_chunks=cycles_num_chunks,
cycles_chunk_start=cycles_chunk_start,
cycles_chunk_end=cycles_chunk_end,
)
]
if isinstance(parents, list):
parent_task_id = parents[chunk_idx]
else:
parent_task_id = parents
if not isinstance(parent_task_id, ObjectId):
raise TypeError(
'parents should be list of ObjectIds or ObjectId, not %s' %
parents)
task_id = self._create_task(job,
task_cmds,
name,
'blender-render',
parents=[parent_task_id],
priority=task_priority)
task_ids.append(task_id)
return task_ids
def _make_merge_tasks(self, job, name_fmt, cycles_chunk_idx, parents1,
parents2, cycles_samples_to1, cycles_samples_from2,
cycles_samples_to2, task_priority):
"""Creates merge tasks for each chunk, consisting of merges for each frame.
:param cycles_chunk_idx: base-1 sample chunk index
"""
# Merging cannot happen unless we have at least two chunks
assert cycles_chunk_idx >= 2
from flamenco.utils import frame_range_merge
task_ids = []
weight1 = cycles_samples_to1
weight2 = cycles_samples_to2 - cycles_samples_from2 + 1
# Replace Blender formatting with Python formatting in render output path
if cycles_chunk_idx == 2:
# The first merge takes a render output as input1, subsequent ones take merge outputs.
# Merging only happens from Cycles chunk 2 (it needs two inputs, hence 2 chunks).
input1 = self._render_output(1, cycles_samples_to1)
else:
input1 = self._merge_output(cycles_samples_to1)
input2 = self._render_output(cycles_samples_from2, cycles_samples_to2)
output = self._merge_output(cycles_samples_to2)
# Construct format strings from the paths.
input1_fmt = str(input1).replace('######', '%06i.exr')
input2_fmt = str(input2).replace('######', '%06i.exr')
output_fmt = str(output).replace('######', '%06i.exr')
final_dest_fmt = str(self.render_output).replace('######', '%06i.exr')
for chunk_idx, chunk_frames in enumerate(self._iter_frame_chunks()):
# Create a merge command for every frame in the chunk.
task_cmds = []
for framenr in chunk_frames:
intermediate = output_fmt % framenr
task_cmds.append(
commands.MergeProgressiveRenders(
input1=input1_fmt % framenr,
input2=input2_fmt % framenr,
output=intermediate,
weight1=weight1,
weight2=weight2,
))
task_cmds.append(
commands.CopyFile(
src=intermediate,
dest=final_dest_fmt % framenr,
))
name = name_fmt % frame_range_merge(chunk_frames)
parent1 = parents1[chunk_idx]
parent2 = parents2[chunk_idx]
task_id = self._create_task(job,
task_cmds,
name,
'exr-merge',
parents=[parent1, parent2],
priority=task_priority)
task_ids.append(task_id)
return task_ids
def _make_render_tasks(self, job, render_parent_tid: ObjectId) \
-> typing.Tuple[typing.List[ObjectId], typing.List[ObjectId]]:
"""Creates the render tasks for this job.
:returns: the list of task IDs.
"""
from flamenco.utils import iter_frame_range, frame_range_merge
job_settings = job['settings']
task_ids = []
parent_task_ids = []
for chunk_frames in iter_frame_range(job_settings['frames'],
job_settings['chunk_size']):
frame_range = frame_range_merge(chunk_frames)
frame_range_bstyle = frame_range_merge(chunk_frames,
blender_style=True)
first_frame = chunk_frames[0]
last_frame = chunk_frames[-1]
chunk_name = 'chunk-%05d-%05d' % (first_frame, last_frame)
render_output = self.frames_dir / chunk_name / '######.png'
# Export to PNG frames.
task_cmds = [
commands.BlenderRender(
blender_cmd=job_settings.get('blender_cmd', '{blender}'),
filepath=job_settings['filepath'],
format='PNG',
render_output=str(render_output),
frames=frame_range_bstyle,
)
]
name = 'frame-chunk-%s' % frame_range
render_task_id = self._create_task(job,
task_cmds,
name,
'blender-render',
parents=[render_parent_tid])
task_ids.append(render_task_id)
# Encode PNG frames to video.
file_extension = job_settings['output_file_extension']
task_cmds = [
commands.CreateVideo(
input_files=str(render_output.with_name('*.png')),
output_file=str(self.frames_dir /
(chunk_name + file_extension)),
fps=job_settings['fps'],
)
]
name = 'video-chunk-%s' % frame_range
encoding_task_id = self._create_task(job,
task_cmds,
name,
'video-encoding',
parents=[render_task_id])
task_ids.append(encoding_task_id)
parent_task_ids.append(encoding_task_id)
return task_ids, parent_task_ids