def startup(args: Args, chunk_queue: List[Chunk]):
"""
If resuming, open done file and get file properties from there
else get file properties and
"""
# TODO: move this out and pass in total frames and initial frames
done_path = args.temp / 'done.json'
if args.resume and done_path.exists():
log('Resuming...\n')
with open(done_path) as done_file:
data = json.load(done_file)
total = data['total']
done = len(data['done'])
initial = sum(data['done'].values())
log(f'Resumed with {done} encoded clips done\n\n')
else:
initial = 0
total = frame_probe_cv2(args.input)
if total < 1:
total = frame_probe(args.input)
d = {'total': total, 'done': {}}
with open(done_path, 'w') as done_file:
json.dump(d, done_file)
clips = len(chunk_queue)
args.workers = min(args.workers, clips)
print(f'\rQueue: {clips} Workers: {args.workers} Passes: {args.passes}\n'
f'Params: {" ".join(args.video_params)}')
counter = Manager().Counter(total, initial)
args.counter = counter
def _concatenate_mkvmerge(files, output, file_limit, cmd_limit, flip=False):
tmp_out = "{}.tmp{}.mkv".format(output, int(flip))
cmd = ["mkvmerge", "-o", tmp_out, files[0]]
remaining = []
for i, file in enumerate(files[1:]):
new_cmd = cmd + ['+{}'.format(file)]
if sum(len(s) for s in new_cmd) < cmd_limit \
and (file_limit == -1 or i < max(1, file_limit - 10)):
cmd = new_cmd
else:
remaining = files[i + 1:]
break
concat = subprocess.Popen(cmd, stdout=PIPE, universal_newlines=True)
message, _ = concat.communicate()
concat.wait()
if concat.returncode != 0:
log(message)
print(message)
raise Exception
if len(remaining) > 0:
return _concatenate_mkvmerge(
[tmp_out] + remaining, output, file_limit, cmd_limit, not flip)
else:
return tmp_out
def concatenate_mkvmerge(temp: Path, output):
"""
Uses mkvmerge to concatenate encoded segments into the final file
:param temp: the temp directory
:param output: the final output file
:return: None
"""
log('Concatenating\n')
encode_files = sorted((temp / 'encode').iterdir())
concat = ' +'.join(shlex.quote(f.as_posix()) for f in encode_files)
audio_file = temp / "audio.mkv"
audio = audio_file.as_posix() if audio_file.exists() else ''
cmd = f' mkvmerge {concat} {shlex.quote(audio)} -o {shlex.quote(output.as_posix())}'
concat = subprocess.Popen(cmd,
stdout=PIPE,
universal_newlines=True,
shell=True)
output, _ = concat.communicate()
concat.wait()
if concat.returncode != 0:
log(output)
print(output)
raise Exception
def startup(project: Project, chunk_queue: List[Chunk]):
"""
If resuming, open done file and get file properties from there
else get file properties and
"""
done_path = project.temp / 'done.json'
if project.resume and done_path.exists():
log('Resuming...\n')
with open(done_path) as done_file:
data = json.load(done_file)
set_total_frame_count(project,data['frames'])
done = len(data['done'])
initial = sum(data['done'].values())
log(f'Resumed with {done} encoded clips done\n\n')
else:
initial = 0
total = frame_probe_fast(project.input, project.is_vs)
d = {'frames': total, 'done': {}}
with open(done_path, 'w') as done_file:
json.dump(d, done_file)
clips = len(chunk_queue)
project.workers = min(project.workers, clips)
print(f'\rQueue: {clips} Workers: {project.workers} Passes: {project.passes}\n'
f'Params: {" ".join(project.video_params)}')
counter = Manager().Counter(get_total_frame_count(project), initial)
project.counter = counter
def create_video_queue_segment(args: Args, split_locations: List[int]) -> List[Chunk]:
"""
Create a list of chunks using segmented files
:param args: Args
:param split_locations: a list of frames to split on
:return: A list of chunks
"""
# segment into separate files
segment(args.input, args.temp, split_locations)
# get the names of all the split files
source_path = args.temp / 'split'
queue_files = [x for x in source_path.iterdir() if x.suffix == '.mkv']
queue_files.sort(key=lambda p: p.stem)
if len(queue_files) == 0:
er = 'Error: No files found in temp/split, probably splitting not working'
print(er)
log(er)
terminate()
chunk_queue = [create_chunk_from_segment(args, index, file) for index, file in enumerate(queue_files)]
return chunk_queue
def concatenate_mkvmerge(temp: Path, output):
"""
Uses mkvmerge to concatenate encoded segments into the final file
:param temp: the temp directory
:param output: the final output file
:return: None
"""
log('Concatenating\n')
output = shlex.quote(output.as_posix())
encode_files = sorted((temp / 'encode').iterdir(),
key=lambda x: int(x.stem)
if x.stem.isdigit() else x.stem)
encode_files = [shlex.quote(f.as_posix()) for f in encode_files]
if platform.system() == "Linux":
import resource
file_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
cmd_limit = os.sysconf(os.sysconf_names['SC_ARG_MAX'])
else:
file_limit = -1
cmd_limit = 32767
audio_file = temp / "audio.mkv"
audio = audio_file.as_posix() if audio_file.exists() else ''
if len(encode_files) > 1:
encode_files = [
_concatenate_mkvmerge(encode_files, output, file_limit, cmd_limit)
]
cmd = ['mkvmerge', '-o', output, encode_files[0]]
if audio:
cmd.append(audio)
concat = subprocess.Popen(cmd, stdout=PIPE, universal_newlines=True)
message, _ = concat.communicate()
concat.wait()
if concat.returncode != 0:
log(message)
print(message)
raise Exception
# remove temporary files used by recursive concat
if os.path.exists("{}.tmp0.mkv".format(output)):
os.remove("{}.tmp0.mkv".format(output))
if os.path.exists("{}.tmp1.mkv".format(output)):
os.remove("{}.tmp1.mkv".format(output))
def concatenate_ffmpeg(temp: Path, output: Path, encoder: str):
"""
Uses ffmpeg to concatenate encoded segments into the final file
:param temp: the temp directory
:param output: the final output file
:param encoder: the encoder
:return: None
"""
"""With FFMPEG concatenate encoded segments into final file."""
log('Concatenating\n')
with open(temp / "concat", 'w') as f:
encode_files = sorted((temp / 'encode').iterdir())
f.writelines(f'file {shlex.quote("file:"+str(file.absolute()))}\n'
for file in encode_files)
# Add the audio/subtitles/else file if one was extracted from the input
audio_file = temp / "audio.mkv"
if audio_file.exists():
audio = ('-i', audio_file.as_posix(), '-c', 'copy', '-map', '1')
else:
audio = ()
if encoder == 'x265':
cmd = [
'ffmpeg', '-y', '-fflags', '+genpts', '-hide_banner',
'-loglevel', 'error', '-f', 'concat', '-safe', '0', '-i',
(temp / "concat").as_posix(), *audio, '-c', 'copy', '-movflags',
'frag_keyframe+empty_moov', '-map', '0', '-f', 'mp4',
output.as_posix()
]
concat = subprocess.run(cmd, stdout=PIPE, stderr=STDOUT).stdout
else:
cmd = [
'ffmpeg', '-y', '-hide_banner',
'-loglevel', 'error', '-f', 'concat', '-safe', '0', '-i',
(temp / "concat").as_posix(), *audio, '-c', 'copy', '-map', '0',
output.as_posix()
]
concat = subprocess.run(cmd, stdout=PIPE, stderr=STDOUT).stdout
if len(concat) > 0:
log(concat.decode())
print(concat.decode())
raise Exception
def encode(chunk: Chunk, args: Args):
"""
Encodes a chunk.
:param chunk: The chunk to encode
:param args: The cli args
:return: None
"""
try:
st_time = time.time()
chunk_frames = chunk.frames
log(f'Enc: {chunk.name}, {chunk_frames} fr\n\n')
# Target Vmaf Mode
if args.vmaf_target:
target_vmaf_routine(args, chunk)
ENCODERS[args.encoder].on_before_chunk(args, chunk)
# skip first pass if reusing
start = 2 if args.reuse_first_pass and args.passes >= 2 else 1
# Run all passes for this chunk
for current_pass in range(start, args.passes + 1):
tqdm_bar(args, chunk, args.encoder, args.counter, chunk_frames,
args.passes, current_pass)
ENCODERS[args.encoder].on_after_chunk(args, chunk)
# get the number of encoded frames, if no check assume it worked and encoded same number of frames
encoded_frames = chunk_frames if args.no_check else frame_check_output(
chunk, chunk_frames)
# write this chunk as done if it encoded correctly
if encoded_frames == chunk_frames:
write_progress_file(Path(args.temp / 'done.json'), chunk,
encoded_frames)
enc_time = round(time.time() - st_time, 2)
log(f'Done: {chunk.name} Fr: {encoded_frames}\n'
f'Fps: {round(encoded_frames / enc_time, 4)} Time: {enc_time} sec.\n\n'
)
except Exception as e:
_, _, exc_tb = sys.exc_info()
print(f'Error in encoding loop {e}\nAt line {exc_tb.tb_lineno}')
def encode(chunk: Chunk, project: Project):
"""
Encodes a chunk.
:param chunk: The chunk to encode
:param project: The cli project
:return: None
"""
st_time = time.time()
chunk_frames = chunk.frames
log(f'Enc: {chunk.name}, {chunk_frames} fr\n\n')
# Target Quality Mode
if project.target_quality:
if project.target_quality_method == 'per_shot':
per_shot_target_quality_routine(project, chunk)
if project.target_quality_method == 'per_frame':
per_frame_target_quality_routine(project, chunk)
ENCODERS[project.encoder].on_before_chunk(project, chunk)
# skip first pass if reusing
start = 2 if project.reuse_first_pass and project.passes >= 2 else 1
# Run all passes for this chunk
for current_pass in range(start, project.passes + 1):
tqdm_bar(project, chunk, project.encoder, project.counter,
chunk_frames, project.passes, current_pass)
ENCODERS[project.encoder].on_after_chunk(project, chunk)
# get the number of encoded frames, if no check assume it worked and encoded same number of frames
encoded_frames = chunk_frames if project.no_check else frame_check_output(
chunk, chunk_frames)
# write this chunk as done if it encoded correctly
if encoded_frames == chunk_frames:
write_progress_file(Path(project.temp / 'done.json'), chunk,
encoded_frames)
enc_time = round(time.time() - st_time, 2)
log(f'Done: {chunk.name} Fr: {encoded_frames}\n'
f'Fps: {round(encoded_frames / enc_time, 4)} Time: {enc_time} sec.\n\n'
)
def concat_routine(project: Project):
"""
Runs the concatenation routine with project
:param project: the Project
:return: None
"""
try:
if project.encoder == 'vvc':
vvc_concat(project.temp, project.output_file.with_suffix('.h266'))
elif project.mkvmerge:
concatenate_mkvmerge(project.temp, project.output_file)
else:
concatenate_ffmpeg(project.temp, project.output_file, project.encoder)
except Exception as e:
_, _, exc_tb = sys.exc_info()
print(f'Concatenation failed, error\nAt line: {exc_tb.tb_lineno}\nError:{str(e)}')
log(f'Concatenation failed, aborting, error: {e}\n')
terminate()
def pyscene(video, threshold, min_scene_len):
"""
Running PySceneDetect detection on source video for segmenting.
Optimal threshold settings 15-50
"""
if not min_scene_len:
min_scene_len = 15
log(f'Starting PySceneDetect:\nThreshold: {threshold}, Min scene lenght: {min_scene_len}\n'
)
video_manager = VideoManager([str(video)])
scene_manager = SceneManager()
scene_manager.add_detector(
ContentDetector(threshold=threshold, min_scene_len=min_scene_len))
base_timecode = video_manager.get_base_timecode()
# Work on whole video
video_manager.set_duration()
# Set downscale factor to improve processing speed.
video_manager.set_downscale_factor()
# Start video_manager.
video_manager.start()
scene_manager.detect_scenes(frame_source=video_manager, show_progress=True)
# Obtain list of detected scenes.
scene_list = scene_manager.get_scene_list(base_timecode)
scenes = [int(scene[0].get_frames()) for scene in scene_list]
# Remove 0 from list
if scenes[0] == 0:
scenes.remove(0)
log(f'Found scenes: {len(scenes)}\n')
return scenes
def select_best_chunking_method(project: Project):
if not find_executable('vspipe'):
project.chunk_method = 'hybrid'
log('Set Chunking Method: Hybrid')
else:
try:
import vapoursynth
plugins = vapoursynth.get_core().get_plugins()
if 'systems.innocent.lsmas' in plugins:
log('Set Chunking Method: L-SMASH\n')
project.chunk_method = 'vs_lsmash'
elif 'com.vapoursynth.ffms2' in plugins:
log('Set Chunking Method: FFMS2\n')
project.chunk_method = 'vs_ffms2'
except:
log('Vapoursynth not installed but vspipe reachable\n' +
'Fallback to Hybrid\n')
project.chunk_method = 'hybrid'
def target_vmaf(chunk: Chunk, args: Args):
vmaf_cq = []
frames = chunk.frames
q_list = []
score = 0
# Make middle probe
middle_point = (args.min_q + args.max_q) // 2
q_list.append(middle_point)
last_q = middle_point
score = vmaf_probe(chunk, last_q, args)
vmaf_cq.append((score, last_q))
# Branch
if score < args.vmaf_target:
next_q = args.min_q
q_list.append(args.min_q)
else:
next_q = args.max_q
q_list.append(args.max_q)
# Edge case check
score = vmaf_probe(chunk, next_q, args)
vmaf_cq.append((score, next_q))
if next_q == args.min_q and score < args.vmaf_target:
log(f"Chunk: {chunk.name}, Fr: {frames}\n"
f"Q: {sorted([x[1] for x in vmaf_cq])}, Early Skip Low CQ\n"
f"Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n"
f"Target Q: {vmaf_cq[-1][1]} Vmaf: {vmaf_cq[-1][0]}\n\n")
return next_q
elif next_q == args.max_q and score > args.vmaf_target:
log(f"Chunk: {chunk.name}, Fr: {frames}\n"
f"Q: {sorted([x[1] for x in vmaf_cq])}, Early Skip High CQ\n"
f"Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n"
f"Target Q: {vmaf_cq[-1][1]} Vmaf: {vmaf_cq[-1][0]}\n\n")
return next_q
# VMAF search
for _ in range(args.vmaf_steps - 2):
new_point = weighted_search(vmaf_cq[-2][1], vmaf_cq[-2][0],
vmaf_cq[-1][1], vmaf_cq[-1][0],
args.vmaf_target)
if new_point in [x[1] for x in vmaf_cq]:
break
last_q = new_point
q_list.append(new_point)
score = vmaf_probe(chunk, new_point, args)
next_q = get_closest(q_list,
last_q,
positive=score >= args.vmaf_target)
vmaf_cq.append((score, new_point))
q, q_vmaf = get_target_q(vmaf_cq, args.vmaf_target)
log(f'Chunk: {chunk.name}, Fr: {frames}\n'
f'Q: {sorted([x[1] for x in vmaf_cq])}\n'
f'Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n'
f'Target Q: {q} Vmaf: {q_vmaf}\n\n')
# Plot Probes
if args.vmaf_plots and len(vmaf_cq) > 3:
plot_probes(args, vmaf_cq, chunk, frames)
return q
def on_before_chunk(self, project: Project, chunk: Chunk) -> None:
# vvc requires a yuv files as input, make it here
log(f'Creating yuv for chunk {chunk.name}\n')
Vvc.to_yuv(chunk)
log(f'Created yuv for chunk {chunk.name}\n')
super().on_before_chunk(project, chunk)
def pyscene(video, threshold, min_scene_len, is_vs, temp):
"""
Running PySceneDetect detection on source video for segmenting.
Optimal threshold settings 15-50
"""
if not min_scene_len:
min_scene_len = 15
log(f'Starting PySceneDetect:\nThreshold: {threshold}, Min scene length: {min_scene_len}\n Is Vapoursynth input: {is_vs}\n'
)
if is_vs:
# Handling vapoursynth, so we need to create a named pipe to feed to VideoManager.
# TODO: Do we clean this up after pyscenedetect has run, or leave it as part of the temp dir, where it will be cleaned up later?
if sys.platform == "linux":
vspipe_fifo = temp / 'vspipe.y4m'
mkfifo(vspipe_fifo)
else:
vspipe_fifo = None
vspipe_cmd = compose_vapoursynth_pipe(video, vspipe_fifo)
vspipe_process = Popen(vspipe_cmd)
# Get number of frames from Vapoursynth script to pass as duration to VideoManager.
# We need to pass the number of frames to the manager, otherwise it won't close the
# receiving end of the pipe, and will simply sit waiting after vspipe has finished sending
# the last frame.
frames = frame_probe(video)
video_manager = VideoManager([str(vspipe_fifo if is_vs else video)])
scene_manager = SceneManager()
scene_manager.add_detector(
ContentDetector(threshold=threshold, min_scene_len=min_scene_len))
base_timecode = video_manager.get_base_timecode()
video_manager.set_duration(duration=FrameTimecode(
frames, video_manager.get_framerate()) if is_vs else None)
# Set downscale factor to improve processing speed.
video_manager.set_downscale_factor()
# Start video_manager.
video_manager.start()
scene_manager.detect_scenes(frame_source=video_manager, show_progress=True)
# If fed using a vspipe process, ensure that vspipe has finished.
if is_vs:
vspipe_process.wait()
# Obtain list of detected scenes.
scene_list = scene_manager.get_scene_list(base_timecode)
scenes = [int(scene[0].get_frames()) for scene in scene_list]
# Remove 0 from list
if scenes[0] == 0:
scenes.remove(0)
log(f'Found scenes: {len(scenes)}\n')
return scenes
def target_vmaf(chunk: Chunk, args: Args):
vmaf_cq = []
frames = chunk.frames
q_list = []
score = 0
# Make middle probe
middle_point = (args.min_q + args.max_q) // 2
q_list.append(middle_point)
last_q = middle_point
score = vmaf_probe(chunk, last_q, args)
vmaf_cq.append((score, last_q))
if args.vmaf_steps < 3:
#Use Euler's method with known relation between cq and vmaf
vmaf_cq_deriv = -0.18
## Formula -ln(1-score/100) = vmaf_cq_deriv*last_q + constant
#constant = -ln(1-score/100) - vmaf_cq_deriv*last_q
## Formula -ln(1-args.vmaf_target/100) = vmaf_cq_deriv*cq + constant
#cq = (-ln(1-args.vmaf_target/100) - constant)/vmaf_cq_deriv
next_q = int(round(last_q + (transform_vmaf(args.vmaf_target)-transform_vmaf(score))/vmaf_cq_deriv))
#Clamp
if next_q < args.min_q:
next_q = args.min_q
if args.max_q < next_q:
next_q = args.max_q
#Single probe cq guess or exit to avoid divide by zero
if args.vmaf_steps == 1 or next_q == last_q:
return next_q
#Second probe at guessed value
score_2 = vmaf_probe(chunk, next_q, args)
#Calculate slope
vmaf_cq_deriv = (transform_vmaf(score_2)-transform_vmaf(score))/(next_q-last_q)
#Same deal different slope
next_q = int(round(next_q+(transform_vmaf(args.vmaf_target)-transform_vmaf(score_2))/vmaf_cq_deriv))
#Clamp
if next_q < args.min_q:
next_q = args.min_q
if args.max_q < next_q:
next_q = args.max_q
return next_q
# Initialize search boundary
vmaf_lower = score
vmaf_upper = score
vmaf_cq_lower = last_q
vmaf_cq_upper = last_q
# Branch
if score < args.vmaf_target:
next_q = args.min_q
q_list.append(args.min_q)
else:
next_q = args.max_q
q_list.append(args.max_q)
# Edge case check
score = vmaf_probe(chunk, next_q, args)
vmaf_cq.append((score, next_q))
if next_q == args.min_q and score < args.vmaf_target:
log(f"Chunk: {chunk.name}, Fr: {frames}\n"
f"Q: {sorted([x[1] for x in vmaf_cq])}, Early Skip Low CQ\n"
f"Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n"
f"Target Q: {vmaf_cq[-1][1]} Vmaf: {vmaf_cq[-1][0]}\n\n")
return next_q
elif next_q == args.max_q and score > args.vmaf_target:
log(f"Chunk: {chunk.name}, Fr: {frames}\n"
f"Q: {sorted([x[1] for x in vmaf_cq])}, Early Skip High CQ\n"
f"Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n"
f"Target Q: {vmaf_cq[-1][1]} Vmaf: {vmaf_cq[-1][0]}\n\n")
return next_q
# Set boundary
if score < args.vmaf_target:
vmaf_lower = score
vmaf_cq_lower = next_q
else:
vmaf_upper = score
vmaf_cq_upper = next_q
# VMAF search
for _ in range(args.vmaf_steps - 2):
new_point = weighted_search(vmaf_cq_lower, vmaf_lower, vmaf_cq_upper, vmaf_upper, args.vmaf_target)
if new_point in [x[1] for x in vmaf_cq]:
break
q_list.append(new_point)
score = vmaf_probe(chunk, new_point, args)
vmaf_cq.append((score, new_point))
# Update boundary
if score < args.vmaf_target:
vmaf_lower = score
vmaf_cq_lower = new_point
else:
vmaf_upper = score
vmaf_cq_upper = new_point
q, q_vmaf = get_target_q(vmaf_cq, args.vmaf_target)
log(f'Chunk: {chunk.name}, Fr: {frames}\n'
f'Q: {sorted([x[1] for x in vmaf_cq])}\n'
f'Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n'
f'Target Q: {q} Vmaf: {q_vmaf}\n\n')
# Plot Probes
if args.vmaf_plots and len(vmaf_cq) > 3:
plot_probes(args, vmaf_cq, chunk, frames)
return q
def aom_keyframes(video_path: Path, stat_file, min_scene_len, ffmpeg_pipe,
video_params):
"""[Get frame numbers for splits from aomenc 1 pass stat file]
"""
log(f'Started aom_keyframes scenedetection\nParams: {video_params}\n')
# Get CV2 fast framecount
video = cv2.VideoCapture(video_path.as_posix())
total = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
video.release()
if total < 1:
total = frame_probe(video_path)
f, e = compose_aomsplit_first_pass_command(video_path, stat_file,
ffmpeg_pipe, video_params)
tqdm_bar = tqdm(total=total,
initial=0,
dynamic_ncols=True,
unit="fr",
leave=True,
smoothing=0.2)
ffmpeg_pipe = subprocess.Popen(f, stdout=PIPE, stderr=STDOUT)
pipe = subprocess.Popen(e,
stdin=ffmpeg_pipe.stdout,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True)
encoder_history = deque(maxlen=20)
frame = 0
while True:
line = pipe.stdout.readline()
if len(line) == 0 and pipe.poll() is not None:
break
line = line.strip()
if line:
encoder_history.append(line)
match = re.search(r"frame.*?/([^ ]+?) ", line)
if match:
new = int(match.group(1))
if new > frame:
tqdm_bar.update(new - frame)
frame = new
if pipe.returncode != 0 and pipe.returncode != -2: # -2 is Ctrl+C for aom
enc_hist = '\n'.join(encoder_history)
er = f"\nAom first pass encountered an error: {pipe.returncode}\n{enc_hist}"
log(er)
print(er)
if not stat_file.exists():
terminate()
else:
# aom crashed, but created keyframes.log, so we will try to continue
print(
"WARNING: Aom first pass crashed, but created a first pass file. Keyframe splitting may not be accurate."
)
# aom kf-min-dist defaults to 0, but hardcoded to 3 in pass2_strategy.c test_candidate_kf. 0 matches default aom behavior
# https://aomedia.googlesource.com/aom/+/8ac928be918de0d502b7b492708d57ad4d817676/av1/av1_cx_iface.c#2816
# https://aomedia.googlesource.com/aom/+/ce97de2724d7ffdfdbe986a14d49366936187298/av1/encoder/pass2_strategy.c#1907
min_scene_len = 0 if min_scene_len is None else min_scene_len
keyframes = find_aom_keyframes(stat_file, min_scene_len)
return keyframes
