def set_target_quality(project):
"""
Av1an setup for target_quality
:param project: the Project
"""
if project.vmaf_path:
if not Path(project.vmaf_path).exists():
print(f"No model with this path: {Path(project.vmaf_path).as_posix()}")
terminate()
if project.probes < 4:
print('Target quality with less than 4 probes is experimental and not recommended')
terminate()
encoder = ENCODERS[project.encoder]
if project.encoder not in ('x265', 'svt_av1') and project.target_quality_method == 'per_frame':
print(f":: Per frame Target Quality is not supported for selected encoder\n:: Supported encoders: x265, svt_av1")
exit()
# setting range for q values
if project.min_q is None:
project.min_q, _ = encoder.default_q_range
assert project.min_q > 1
if project.max_q is None:
_, project.max_q = encoder.default_q_range
def create_video_queue_segment(args: Args, split_locations: List[int]) -> List[Chunk]:
"""
Create a list of chunks using segmented files
:param args: Args
:param split_locations: a list of frames to split on
:return: A list of chunks
"""
# segment into separate files
segment(args.input, args.temp, split_locations)
# get the names of all the split files
source_path = args.temp / 'split'
queue_files = [x for x in source_path.iterdir() if x.suffix == '.mkv']
queue_files.sort(key=lambda p: p.stem)
if len(queue_files) == 0:
er = 'Error: No files found in temp/split, probably splitting not working'
print(er)
log(er)
terminate()
chunk_queue = [create_chunk_from_segment(args, index, file) for index, file in enumerate(queue_files)]
return chunk_queue
def check_exes(project: Project):
"""
Checking required executables
:param project: the Project
"""
if not find_executable('ffmpeg'):
print('No ffmpeg')
terminate()
if project.chunk_method in ['vs_ffms2', 'vs_lsmash']:
if not find_executable('vspipe'):
print('vspipe executable not found')
terminate()
try:
import vapoursynth
plugins = vapoursynth.get_core().get_plugins()
if project.chunk_method == 'vs_lsmash' and "systems.innocent.lsmas" not in plugins:
print('lsmas is not installed')
terminate()
if project.chunk_method == 'vs_ffms2' and "com.vapoursynth.ffms2" not in plugins:
print('ffms2 is not installed')
terminate()
except ModuleNotFoundError:
print('Vapoursynth is not installed')
terminate()
def find_aom_keyframes(stat_file, key_freq_min):
keyframes_list = []
number_of_frames = round(os.stat(stat_file).st_size / 208) - 1
dict_list = []
try:
with open(stat_file, 'rb') as file:
frame_buf = file.read(208)
while len(frame_buf) > 0:
stats = struct.unpack('d' * 26, frame_buf)
p = dict(zip(fields, stats))
dict_list.append(p)
frame_buf = file.read(208)
except Exception as e:
print('Get exception:', e)
print('Recomended to switch to different method of scenedetection')
terminate()
# intentionally skipping 0th frame and last 16 frames
frame_count_so_far = 1
for i in range(1, number_of_frames - 16):
is_keyframe = False
# https://aomedia.googlesource.com/aom/+/ce97de2724d7ffdfdbe986a14d49366936187298/av1/encoder/pass2_strategy.c#2065
if frame_count_so_far >= key_freq_min:
is_keyframe = test_candidate_kf(dict_list, i, frame_count_so_far)
if is_keyframe:
keyframes_list.append(i)
frame_count_so_far = 0
frame_count_so_far += 1
return keyframes_list
def match_line(self, line: str):
"""Extract number of encoded frames from line.
:param line: one line of text output from the encoder
:return: match object from re.search matching the number of encoded frames"""
if 'error' in line.lower():
print('\n\nERROR IN ENCODING PROCESS\n\n', line)
terminate()
return re.search(r"Encoding frame\s+(\d+)", line)
def match_line(self, line: str):
"""Extract number of encoded frames from line.
:param line: one line of text output from the encoder
:return: match object from re.search matching the number of encoded frames"""
if 'fatal' in line.lower():
print('\n\nERROR IN ENCODING PROCESS\n\n', line)
terminate()
if 'Pass 2/2' in line or 'Pass 1/1' in line:
return re.search(r"frame.*?/([^ ]+?) ", line)
def encoding_loop(project: Project, chunk_queue: List[Chunk]):
"""Creating process pool for encoders, creating progress bar."""
if project.workers != 0:
with concurrent.futures.ThreadPoolExecutor(max_workers=project.workers) as executor:
future_cmd = {executor.submit(encode, cmd, project): cmd for cmd in chunk_queue}
for future in concurrent.futures.as_completed(future_cmd):
try:
future.result()
except Exception as exc:
_, _, exc_tb = sys.exc_info()
print(f'Encoding error {exc}\nAt line {exc_tb.tb_lineno}')
terminate()
project.counter.close()
def check_exes(args: Args):
"""
Checking required executables
:param args: the Args
"""
if not find_executable('ffmpeg'):
print('No ffmpeg')
terminate()
if args.chunk_method == 'vs_ffms2' and (not find_executable('vspipe')):
print('vspipe executable not found')
terminate()
def startup_check(project: Project):
"""
Performing essential checks at startup_check
Set constant values
"""
if sys.version_info < (3, 6):
print('Python 3.6+ required')
sys.exit()
if sys.platform == 'linux':
def restore_term():
os.system("stty sane")
atexit.register(restore_term)
if not project.chunk_method:
select_best_chunking_method(project)
# project.is_vs = is_vapoursynth(project.input)
if project.is_vs:
project.chunk_method = 'vs_ffms2'
check_exes(project)
set_target_quality(project)
if project.reuse_first_pass and project.encoder != 'aom' and project.split_method != 'aom_keyframes':
print('Reusing the first pass is only supported with \
the aom encoder and aom_keyframes split method.')
terminate()
setup_encoder(project)
# No check because vvc
if project.encoder == 'vvc':
project.no_check = True
if project.encoder == 'svt_vp9' and project.passes == 2:
print("Implicitly changing 2 pass svt-vp9 to 1 pass\n2 pass svt-vp9 isn't supported")
project.passes = 1
project.audio_params = shlex.split(project.audio_params)
project.ffmpeg = shlex.split(project.ffmpeg)
project.pix_format = ['-strict', '-1', '-pix_fmt', project.pix_format]
project.ffmpeg_pipe = [*project.ffmpeg, *project.pix_format,
'-bufsize', '50000K', '-f', 'yuv4mpegpipe', '-']
def startup_check(args: Args):
"""
Performing essential checks at startup_check
Set constant values
"""
if sys.version_info < (3, 6):
print('Python 3.6+ required')
sys.exit()
if sys.platform == 'linux':
def restore_term():
os.system("stty sane")
atexit.register(restore_term)
check_exes(args)
set_vmaf(args)
if args.reuse_first_pass and args.encoder != 'aom' and args.split_method != 'aom_keyframes':
print('Reusing the first pass is only supported with \
the aom encoder and aom_keyframes split method.')
terminate()
setup_encoder(args)
# No check because vvc
if args.encoder == 'vvc':
args.no_check = True
if args.encoder == 'svt_vp9' and args.passes == 2:
print(
"Implicitly changing 2 pass svt-vp9 to 1 pass\n2 pass svt-vp9 isn't supported"
)
args.passes = 1
args.audio_params = shlex.split(args.audio_params)
args.ffmpeg = shlex.split(args.ffmpeg)
args.pix_format = ['-strict', '-1', '-pix_fmt', args.pix_format]
args.ffmpeg_pipe = [
*args.ffmpeg, *args.pix_format, '-bufsize', '50000K', '-f',
'yuv4mpegpipe', '-'
]
def setup_encoder(args: Args):
"""
Settup encoder params and passes
:param args: the Args
"""
encoder = ENCODERS[args.encoder]
# validate encoder settings
settings_valid, error_msg = encoder.is_valid(args)
if not settings_valid:
print(error_msg)
terminate()
if args.passes is None:
args.passes = encoder.default_passes
args.video_params = encoder.default_args if args.video_params is None \
else shlex.split(args.video_params)
def concat_routine(project: Project):
"""
Runs the concatenation routine with project
:param project: the Project
:return: None
"""
try:
if project.encoder == 'vvc':
vvc_concat(project.temp, project.output_file.with_suffix('.h266'))
elif project.mkvmerge:
concatenate_mkvmerge(project.temp, project.output_file)
else:
concatenate_ffmpeg(project.temp, project.output_file, project.encoder)
except Exception as e:
_, _, exc_tb = sys.exc_info()
print(f'Concatenation failed, error\nAt line: {exc_tb.tb_lineno}\nError:{str(e)}')
log(f'Concatenation failed, aborting, error: {e}\n')
terminate()
def set_vmaf(args):
"""
Av1an setup for VMAF
:param args: the Args
"""
if args.vmaf_path:
if not Path(args.vmaf_path).exists():
print(f'No such model: {Path(args.vmaf_path).as_posix()}')
terminate()
if args.vmaf_steps < 4:
print('Target vmaf require more than 3 probes/steps')
terminate()
encoder = ENCODERS[args.encoder]
if args.min_q is None:
args.min_q, _ = encoder.default_q_range
if args.max_q is None:
_, args.max_q = encoder.default_q_range
def setup_encoder(project: Project):
"""
Setup encoder params and passes
:param project: the Project
"""
encoder = ENCODERS[project.encoder]
# validate encoder settings
settings_valid, error_msg = encoder.is_valid(project)
if not settings_valid:
print(error_msg)
terminate()
if project.passes is None:
project.passes = encoder.default_passes
project.video_params = encoder.default_args if project.video_params is None \
else shlex.split(project.video_params)
validate_inputs(project)
def set_target_quality(project):
"""
Av1an setup for VMAF
:param project: the Project
"""
if project.vmaf_path:
if not Path(project.vmaf_path).exists():
print(f"No model with this path: {Path(project.vmaf_path).as_posix()}")
terminate()
if project.probes < 4:
print('Target quality with less than 4 probes is experimental and not recommended')
terminate()
encoder = ENCODERS[project.encoder]
if project.min_q is None:
project.min_q, _ = encoder.default_q_range
if project.max_q is None:
_, project.max_q = encoder.default_q_range
def aom_keyframes(video_path: Path, stat_file, min_scene_len, ffmpeg_pipe,
video_params):
"""[Get frame numbers for splits from aomenc 1 pass stat file]
"""
log(f'Started aom_keyframes scenedetection\nParams: {video_params}\n')
# Get CV2 fast framecount
video = cv2.VideoCapture(video_path.as_posix())
total = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
video.release()
if total < 1:
total = frame_probe(video_path)
f, e = compose_aomsplit_first_pass_command(video_path, stat_file,
ffmpeg_pipe, video_params)
tqdm_bar = tqdm(total=total,
initial=0,
dynamic_ncols=True,
unit="fr",
leave=True,
smoothing=0.2)
ffmpeg_pipe = subprocess.Popen(f, stdout=PIPE, stderr=STDOUT)
pipe = subprocess.Popen(e,
stdin=ffmpeg_pipe.stdout,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True)
encoder_history = deque(maxlen=20)
frame = 0
while True:
line = pipe.stdout.readline()
if len(line) == 0 and pipe.poll() is not None:
break
line = line.strip()
if line:
encoder_history.append(line)
match = re.search(r"frame.*?/([^ ]+?) ", line)
if match:
new = int(match.group(1))
if new > frame:
tqdm_bar.update(new - frame)
frame = new
if pipe.returncode != 0 and pipe.returncode != -2: # -2 is Ctrl+C for aom
enc_hist = '\n'.join(encoder_history)
er = f"\nAom first pass encountered an error: {pipe.returncode}\n{enc_hist}"
log(er)
print(er)
if not stat_file.exists():
terminate()
else:
# aom crashed, but created keyframes.log, so we will try to continue
print(
"WARNING: Aom first pass crashed, but created a first pass file. Keyframe splitting may not be accurate."
)
# aom kf-min-dist defaults to 0, but hardcoded to 3 in pass2_strategy.c test_candidate_kf. 0 matches default aom behavior
# https://aomedia.googlesource.com/aom/+/8ac928be918de0d502b7b492708d57ad4d817676/av1/av1_cx_iface.c#2816
# https://aomedia.googlesource.com/aom/+/ce97de2724d7ffdfdbe986a14d49366936187298/av1/encoder/pass2_strategy.c#1907
min_scene_len = 0 if min_scene_len is None else min_scene_len
keyframes = find_aom_keyframes(stat_file, min_scene_len)
return keyframes
