def test_small_job(self, mock_datetime):
from flamenco.job_compilers import blender_render_progressive, commands
logging.basicConfig(level=logging.DEBUG)
job_doc = JobDocForTesting({
'_id': ObjectId(24 * 'f'),
'_created': self.created,
'settings': {
'frames': '1-5',
'chunk_size': 3,
'render_output': '/render/out/frames-######',
'fps': 5.3,
'format': 'OPEN_EXR',
'filepath': '/agent327/scenes/someshot/somefile.blend',
'blender_cmd': '/path/to/blender --enable-new-depsgraph',
'cycles_sample_count': 30,
# Effectively uncapped so that the number of tasks stays small.
# The actual capping is tested in test_chunk_generator() anyway.
'cycles_sample_cap': 30,
},
'job_type': 'blender-render-progressive',
})
task_manager = mock.Mock()
job_manager = mock.Mock()
# Create a stable 'now' for testing.
mock_now = datetime.datetime.now(tz=tz_util.utc)
mock_datetime.now.side_effect = [mock_now]
task_ids = [ObjectId() for _ in range(17)]
task_manager.api_create_task.side_effect = task_ids
compiler = blender_render_progressive.BlenderRenderProgressive(
task_manager=task_manager, job_manager=job_manager)
compiler._uncapped_chunk_count = 3 # Reduce to a testable number of tasks.
compiler.compile(job_doc)
task_manager.api_create_task.assert_has_calls([
# Pre-existing intermediate directory is destroyed.
mock.call( # task 0
job_doc,
[
commands.RemoveTree(
path='/render/out__intermediate-2018-07-06_115233')
],
'destroy-preexisting-intermediate',
status='under-construction',
task_type='file-management',
),
# First Cycles chunk goes into intermediate directory
mock.call( # task 1
job_doc,
[
commands.BlenderRenderProgressive(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath='/agent327/scenes/someshot/somefile.blend',
format='OPEN_EXR',
render_output=
'/render/out__intermediate-2018-07-06_115233/render-smpl-0001-0001-######',
frames='1..5',
cycles_num_chunks=30,
cycles_chunk_start=1,
cycles_chunk_end=1)
],
'render-s_1-1-f_1-5',
priority=0,
parents=[task_ids[0]],
status='under-construction',
task_type='blender-render',
),
mock.call( # task 2
job_doc,
[
commands.ExrSequenceToJpeg(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath='/agent327/scenes/someshot/somefile.blend',
exr_glob=
'/render/out__intermediate-2018-07-06_115233/render-smpl-0001-0001-*.exr',
output_pattern='preview-######',
)
],
'create-preview-images',
priority=1,
parents=[task_ids[1]],
status='under-construction',
task_type='blender-render',
),
mock.call( # task 3
job_doc,
[
commands.CreateVideo(
input_files=
'/render/out__intermediate-2018-07-06_115233/preview-*.jpg',
output_file=
'/render/out__intermediate-2018-07-06_115233/preview.mkv',
fps=5.3,
)
],
'create-preview-video',
priority=1,
parents=[task_ids[2]],
status='under-construction',
task_type='video-encoding',
),
# Second Cycles chunk renders to intermediate directory.
mock.call( # task 4
job_doc,
[
commands.BlenderRenderProgressive(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath='/agent327/scenes/someshot/somefile.blend',
format='OPEN_EXR',
render_output=
'/render/out__intermediate-2018-07-06_115233/render-smpl-0002-0010-######',
frames='1..5',
cycles_num_chunks=30,
cycles_chunk_start=2,
cycles_chunk_end=10)
],
'render-s_2-10-f_1-5',
priority=-10,
parents=[task_ids[0]],
status='under-construction',
task_type='blender-render',
),
# First merge pass, outputs to intermediate directory and copies to output dir
mock.call( # task 5
job_doc,
[
commands.MergeProgressiveRenderSequence(
blender_cmd='/path/to/blender --enable-new-depsgraph',
input1=
'/render/out__intermediate-2018-07-06_115233/render-smpl-0001-0001-000001.exr',
input2=
'/render/out__intermediate-2018-07-06_115233/render-smpl-0002-0010-000001.exr',
output=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0010-######',
weight1=1,
weight2=9,
frame_start=1,
frame_end=5,
),
],
'merge-to-s_10-f_1-5',
parents=[task_ids[1], task_ids[4]],
priority=1,
status='under-construction',
task_type='exr-merge',
),
mock.call( # task 6
job_doc,
[
commands.ExrSequenceToJpeg(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath='/agent327/scenes/someshot/somefile.blend',
exr_glob=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0010-*.exr',
output_pattern='preview-######',
)
],
'create-preview-images',
priority=1,
parents=[task_ids[2], task_ids[5]],
status='under-construction',
task_type='blender-render',
),
mock.call( # task 7
job_doc,
[
commands.CreateVideo(
input_files=
'/render/out__intermediate-2018-07-06_115233/preview-*.jpg',
output_file=
'/render/out__intermediate-2018-07-06_115233/preview.mkv',
fps=5.3,
)
],
'create-preview-video',
priority=1,
parents=[task_ids[3], task_ids[6]],
status='under-construction',
task_type='video-encoding',
),
# Third Cycles chunk renders to intermediate directory.
mock.call( # task 8
job_doc,
[
commands.BlenderRenderProgressive(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath='/agent327/scenes/someshot/somefile.blend',
format='OPEN_EXR',
render_output=
'/render/out__intermediate-2018-07-06_115233/render-smpl-0011-0030-######',
frames='1..3',
cycles_num_chunks=30,
cycles_chunk_start=11,
cycles_chunk_end=30)
],
'render-s_11-30-f_1-3',
priority=-20,
parents=[task_ids[0]],
status='under-construction',
task_type='blender-render',
),
mock.call( # task 9
job_doc,
[
commands.BlenderRenderProgressive(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath='/agent327/scenes/someshot/somefile.blend',
format='OPEN_EXR',
render_output=
'/render/out__intermediate-2018-07-06_115233/render-smpl-0011-0030-######',
frames='4,5',
cycles_num_chunks=30,
cycles_chunk_start=11,
cycles_chunk_end=30)
],
'render-s_11-30-f_4,5',
priority=-20,
parents=[task_ids[0]],
status='under-construction',
task_type='blender-render',
),
# Final merge pass. Could happen directly to the output directory, but to ensure the
# intermediate directory shows a complete picture (pun intended), we take a similar
# approach as earlier merge passes.
mock.call( # task 10
job_doc,
[
commands.MergeProgressiveRenderSequence(
blender_cmd='/path/to/blender --enable-new-depsgraph',
input1=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0010-000001.exr',
input2=
'/render/out__intermediate-2018-07-06_115233/render-smpl-0011-0030-000001.exr',
output=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0030-######',
weight1=10,
weight2=20,
frame_start=1,
frame_end=5,
),
],
'merge-to-s_30-f_1-5',
parents=[task_ids[5], task_ids[8], task_ids[9]],
priority=1,
status='under-construction',
task_type='exr-merge',
),
mock.call( # task 11
job_doc,
[
commands.ExrSequenceToJpeg(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath='/agent327/scenes/someshot/somefile.blend',
exr_glob=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0030-*.exr',
output_pattern='preview-######',
)
],
'create-preview-images',
priority=1,
parents=[task_ids[6], task_ids[10]],
status='under-construction',
task_type='blender-render',
),
mock.call( # task 12
job_doc,
[
commands.CreateVideo(
input_files=
'/render/out__intermediate-2018-07-06_115233/preview-*.jpg',
output_file=
'/render/out__intermediate-2018-07-06_115233/preview.mkv',
fps=5.3,
)
],
'create-preview-video',
priority=1,
parents=[task_ids[7], task_ids[11]],
status='under-construction',
task_type='video-encoding',
),
mock.call( # task 13
job_doc,
[
commands.MoveOutOfWay(src='/render/out'),
],
'move-outdir-out-of-way',
priority=1,
parents=[task_ids[10]],
status='under-construction',
task_type='file-management',
),
mock.call( # task 14
job_doc,
[
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0030-000001.exr',
dest='/render/out/frames-000001.exr',
),
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0030-000002.exr',
dest='/render/out/frames-000002.exr',
),
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0030-000003.exr',
dest='/render/out/frames-000003.exr',
),
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0030-000004.exr',
dest='/render/out/frames-000004.exr',
),
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/merge-smpl-0030-000005.exr',
dest='/render/out/frames-000005.exr',
),
],
'publish-exr-to-output',
priority=1,
parents=[task_ids[13]],
status='under-construction',
task_type='file-management',
),
mock.call( # task 15
job_doc,
[
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/preview-000001.jpg',
dest='/render/out/frames-000001.jpg',
),
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/preview-000002.jpg',
dest='/render/out/frames-000002.jpg',
),
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/preview-000003.jpg',
dest='/render/out/frames-000003.jpg',
),
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/preview-000004.jpg',
dest='/render/out/frames-000004.jpg',
),
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/preview-000005.jpg',
dest='/render/out/frames-000005.jpg',
),
],
'publish-jpeg-to-output',
priority=1,
parents=[task_ids[11], task_ids[13]],
status='under-construction',
task_type='file-management',
),
mock.call( # task 16
job_doc,
[
commands.CopyFile(
src=
'/render/out__intermediate-2018-07-06_115233/preview.mkv',
dest='/render/out/preview.mkv',
),
],
'publish-video-to-output',
priority=1,
parents=[task_ids[12], task_ids[13]],
status='under-construction',
task_type='file-management',
),
])
task_manager.api_set_task_status_for_job.assert_called_with(
job_doc['_id'], 'under-construction', 'queued', now=mock_now)
job_manager.api_set_job_status(job_doc['_id'],
'under-construction',
'queued',
now=mock_now)
def _test_for_extension(self, extension: str, mock_datetime):
from flamenco.job_compilers import blender_video_chunks, commands
job_doc = JobDocForTesting({
'_id': ObjectId(24 * 'f'),
'_created': self.created,
'settings': {
'frames': '100-250',
'fps': 24,
'chunk_size': 100,
'render_output': '/tmp/render/spring/export/FILENAME.MKV',
'filepath': '/spring/edit/sprloing.blend',
'output_file_extension': extension,
'images_or_video': 'video',
'extract_audio': True,
},
'job_type': 'blender-video-chunks',
})
task_manager = mock.Mock()
job_manager = mock.Mock()
# Create a stable 'now' for testing.
mock_datetime.now.side_effect = [self.mock_now, self.mock_now]
# We expect:
# - 1 move-out-of-way task
# - 2 frame rendering chunks of resp. 100 and 51 frames each
# - 2 video encoding chunks
# - 1 concat-videos task
# - 1 extract-audio task
# - 1 encode-audio task (because extracting only works to FLAC at the moment)
# - 1 mux-audio task
# - 1 move-to-final task
# so that's 10 tasks in total.
task_ids = [ObjectId() for _ in range(10)]
task_manager.api_create_task.side_effect = task_ids
compiler = blender_video_chunks.BlenderVideoChunks(
task_manager=task_manager, job_manager=job_manager)
compiler.compile(job_doc)
frames = '/tmp/render/spring/export/frames'
expected_final_output = f'/tmp/render/spring/export/' \
f'{self.mock_now:%Y_%m_%d}-sprloing{extension}'
task_manager.api_create_task.assert_has_calls([
mock.call( # 0
job_doc,
[commands.MoveOutOfWay(src=frames)],
'move-out-of-way',
status='under-construction',
task_type='file-management',
),
# Chunk (frames + video) tasks
mock.call( # 1
job_doc,
[commands.BlenderRender(
blender_cmd='{blender}',
filepath='/spring/edit/sprloing.blend',
render_output=f'{frames}/chunk-00100-00199/######.png',
format='PNG',
frames='100..199')],
'frame-chunk-100-199',
status='under-construction',
task_type='blender-render',
parents=[task_ids[0]],
),
mock.call( # 2
job_doc,
[commands.CreateVideo(
ffmpeg_cmd='{ffmpeg}',
input_files=f'{frames}/chunk-00100-00199/*.png',
output_file=f'{frames}/chunk-00100-00199{extension}',
fps=24)],
'video-chunk-100-199',
status='under-construction',
task_type='video-encoding',
parents=[task_ids[1]],
),
mock.call( # 3
job_doc,
[commands.BlenderRender(
blender_cmd='{blender}',
filepath='/spring/edit/sprloing.blend',
render_output=f'{frames}/chunk-00200-00250/######.png',
format='PNG',
frames='200..250')],
'frame-chunk-200-250',
status='under-construction',
task_type='blender-render',
parents=[task_ids[0]],
),
mock.call( # 4
job_doc,
[commands.CreateVideo(
ffmpeg_cmd='{ffmpeg}',
input_files=f'{frames}/chunk-00200-00250/*.png',
output_file=f'{frames}/chunk-00200-00250{extension}',
fps=24)],
'video-chunk-200-250',
status='under-construction',
task_type='video-encoding',
parents=[task_ids[3]],
),
# Extract & encode the audio
mock.call( # 5
job_doc,
[commands.BlenderRenderAudio(
blender_cmd='{blender}',
filepath='/spring/edit/sprloing.blend',
render_output=f'{frames}/audio.flac',
frame_start=100,
frame_end=250)],
'render-audio',
status='under-construction',
task_type='blender-render',
parents=[task_ids[0]],
),
mock.call( # 6
job_doc,
[commands.EncodeAudio(
ffmpeg_cmd='{ffmpeg}',
input_file=f'{frames}/audio.flac',
codec='aac',
bitrate='192k',
output_file=f'{frames}/audio.aac',
)],
'encode-audio',
status='under-construction',
task_type='video-encoding',
parents=[task_ids[5]],
),
# Create a video of the chunks.
mock.call( # 7
job_doc,
[commands.ConcatenateVideos(
ffmpeg_cmd='{ffmpeg}',
input_files=f'{frames}/chunk-*{extension}',
output_file=f'{frames}/video.mkv',
)],
'concatenate-videos',
status='under-construction',
task_type='video-encoding',
parents=[task_ids[2], task_ids[4]],
),
# Mux the audio into the video.
mock.call( # 8
job_doc,
[commands.MuxAudio(
ffmpeg_cmd='{ffmpeg}',
audio_file=f'{frames}/audio.aac',
video_file=f'{frames}/video.mkv',
output_file=f'{frames}/muxed.mkv',
)],
'mux-audio-video',
status='under-construction',
task_type='video-encoding',
parents=[task_ids[6], task_ids[7]],
),
# Move the file to its final place
mock.call( # 9
job_doc,
[commands.MoveWithCounter(
src=f'{frames}/muxed.mkv',
dest=expected_final_output,
)],
'move-with-counter',
status='under-construction',
task_type='file-management',
parents=[task_ids[8]],
),
])
task_manager.api_set_task_status_for_job.assert_called_with(
job_doc['_id'], 'under-construction', 'queued', now=self.mock_now)
job_manager.api_set_job_status(job_doc['_id'], 'under-construction', 'queued',
now=self.mock_now)
def test_create_video(self):
from flamenco.job_compilers import blender_render, commands
with self.app.app_context():
self.flamenco.db('managers').update_one({'_id': self.mngr_id}, {
'$set': {
'worker_task_types': ['blender-render', 'video-encoding']
}
})
job_doc = JobDocForTesting({
'_id': ObjectId(24 * 'f'),
'_created': self.created,
'manager': self.mngr_id,
'settings': {
'frames': '1-5',
'chunk_size': 3,
'render_output': '/render/out/frames-######',
'format': 'PNG',
'filepath':
'/agent327/scenes/someshot/somefile.flamenco.blend',
'blender_cmd': '/path/to/blender --enable-new-depsgraph',
# On top of pretty much the same settings as test_small_job(),
# we add those settings that trigger the creation of the
# create_video task.
'fps': 24,
'images_or_video': 'images',
'output_file_extension': '.png',
},
'job_type': 'blender-render',
})
task_manager = mock.Mock()
job_manager = mock.Mock()
# We expect:
# - 2 chunk of 3 resp 2 frames.
# - 1 create_video task.
# - 1 move-to-final task.
# so that's 4 tasks in total.
task_ids = [ObjectId() for _ in range(4)]
task_manager.api_create_task.side_effect = task_ids
compiler = blender_render.BlenderRender(task_manager=task_manager,
job_manager=job_manager)
with self.app.app_context():
compiler.compile(job_doc)
task_manager.api_create_task.assert_has_calls([
# Render tasks
mock.call(
job_doc,
[
commands.BlenderRender(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath=
'/agent327/scenes/someshot/somefile.flamenco.blend',
format='PNG',
render_output=
'/render/out__intermediate-2018-07-06_115233/frames-######',
frames='1..3')
],
'blender-render-1-3',
status='under-construction',
task_type='blender-render',
parents=None,
),
mock.call(
job_doc,
[
commands.BlenderRender(
blender_cmd='/path/to/blender --enable-new-depsgraph',
filepath=
'/agent327/scenes/someshot/somefile.flamenco.blend',
format='PNG',
render_output=
'/render/out__intermediate-2018-07-06_115233/frames-######',
frames='4,5')
],
'blender-render-4,5',
status='under-construction',
task_type='blender-render',
parents=None,
),
# Create a video of the final frames.
mock.call(
job_doc,
[
commands.CreateVideo(
input_files=
'/render/out__intermediate-2018-07-06_115233/*.png',
output_file=
'/render/out__intermediate-2018-07-06_115233/somefile-1-5.mkv',
fps=24,
ffmpeg_cmd='{ffmpeg}',
)
],
'create-video',
parents=task_ids[0:2],
status='under-construction',
task_type='video-encoding',
),
# Move to final location
mock.call(
job_doc,
[
commands.MoveToFinal(
src='/render/out__intermediate-2018-07-06_115233',
dest='/render/out')
],
'move-to-final',
parents=[task_ids[2]],
status='under-construction',
task_type='file-management',
),
])
task_manager.api_set_task_status_for_job.assert_called_with(
job_doc['_id'], 'under-construction', 'queued', now=mock.ANY)
job_manager.api_set_job_status(job_doc['_id'],
'under-construction',
'queued',
now=mock.ANY)