def get_video_queue(temp: Path, resume):
"""Returns sorted list of all videos that need to be encoded. Big first."""
source_path = temp / 'split'
queue = [x for x in source_path.iterdir() if x.suffix == '.mkv']
done_file = temp / 'done.json'
if resume and done_file.exists():
try:
with open(done_file) as f:
data = json.load(f)
data = data['done'].keys()
queue = [x for x in queue if x.name not in data]
except Exception as e:
_, _, exc_tb = sys.exc_info()
print(f'Error at resuming {e}\nAt line {exc_tb.tb_lineno}')
queue = sorted(queue, key=lambda x: -x.stat().st_size)
if len(queue) == 0:
print(
'Error: No files found in .temp/split, probably splitting not working'
)
terminate()
return queue
def svt_av1_encode(inputs, passes, pipe, params):
"""SVT-AV1 encoding command composition."""
encoder = 'SvtAv1EncApp'
commands = []
if not params:
print('-w -h -fps is required parameters for svt_av1 encoder')
terminate()
if passes == 1:
commands = [(
f'-i {file[0]} {pipe} ' +
f' {encoder} -i stdin {params} -b {file[1].with_suffix(".ivf")} -',
(file[0], file[1].with_suffix('.ivf'))) for file in inputs]
if passes == 2:
p2i = '-input-stat-file '
p2o = '-output-stat-file '
commands = [(
f'-i {file[0]} {pipe} {encoder} -i stdin {params} {p2o} '
f'{file[0].with_suffix(".stat")} -b {file[0]}.bk - ',
f'-i {file[0]} {pipe} '
f'{encoder} -i stdin {params} {p2i} {file[0].with_suffix(".stat")} -b '
f'{file[1].with_suffix(".ivf")} - ', (file[0],
file[1].with_suffix('.ivf')))
for file in inputs]
return commands
def get_video_queue(temp: Path, resume):
"""
Compose and returns sorted list of all files that need to be encoded. Big first.
:param temp: Path to temp folder
:param resume: Flag on should already encoded chunks be discarded from queue
:return: Sorted big first list of chunks to encode
"""
source_path = temp / 'split'
queue = [x for x in source_path.iterdir() if x.suffix == '.mkv']
done_file = temp / 'done.json'
if resume and done_file.exists():
try:
with open(done_file) as f:
data = json.load(f)
data = data['done'].keys()
queue = [x for x in queue if x.name not in data]
except Exception as e:
_, _, exc_tb = sys.exc_info()
print(f'Error at resuming {e}\nAt line {exc_tb.tb_lineno}')
queue = sorted(queue, key=lambda x: -x.stat().st_size)
if len(queue) == 0:
er = 'Error: No files found in .temp/split, probably splitting not working'
print(er)
log(er)
terminate()
return queue
def call_vmaf(source: Path, encoded: Path, model=None, return_file=False):
if model:
mod = f":model_path={model}"
else:
mod = ''
# For vmaf calculation both source and encoded segment scaled to 1080
# for proper vmaf calculation
fl = source.with_name(encoded.stem).with_suffix('.xml').as_posix()
cmd = f'ffmpeg -loglevel error -hide_banner -r 60 -i {source.as_posix()} -r 60 -i {encoded.as_posix()} ' \
f'-filter_complex "[0:v]scale=1920:1080:flags=spline:force_original_aspect_ratio=decrease[scaled1];' \
f'[1:v]scale=1920:1080:flags=spline:force_original_aspect_ratio=decrease[scaled2];' \
f'[scaled2][scaled1]libvmaf=log_path={fl}{mod}" -f null - '
c = subprocess.run(cmd, shell=True, stdout=PIPE, stderr=STDOUT)
call = c.stdout
# print(c.stdout.decode())
if 'error' in call.decode().lower():
print('\n\nERROR IN VMAF CALCULATION\n\n', call.decode())
terminate()
if return_file:
return fl
call = call.decode().strip()
vmf = call.split()[-1]
try:
vmf = float(vmf)
except ValueError:
vmf = 0
return vmf
def tqdm_bar(i, encoder, counter, frame_probe_source, passes):
try:
encoder_history = deque(maxlen=20)
f, e = i.split('|')
f = " ffmpeg -y -hide_banner -loglevel error " + f
f, e = f.split(), e.split()
frame = 0
ffmpeg_pipe = subprocess.Popen(f, stdout=PIPE, stderr=STDOUT)
pipe = subprocess.Popen(e,
stdin=ffmpeg_pipe.stdout,
stdout=PIPE,
stderr=STDOUT,
universal_newlines=True)
while True:
line = pipe.stdout.readline().strip()
if line:
encoder_history.append(line)
if len(line) == 0 and pipe.poll() is not None:
break
if len(line) == 0:
continue
if encoder in ('aom', 'vpx', 'rav1e'):
match = None
if encoder in ('aom', 'vpx'):
if 'fatal' in line.lower():
print('\n\nERROR IN ENCODING PROCESS\n\n', line)
terminate()
if 'Pass 2/2' in line or 'Pass 1/1' in line:
match = re.search(r"frame.*?\/([^ ]+?) ", line)
elif encoder == 'rav1e':
if 'error' in line.lower():
print('\n\nERROR IN ENCODING PROCESS\n\n', line)
terminate()
match = re.search(r"encoded.*? ([^ ]+?) ", line)
if match:
new = int(match.group(1))
if new > frame:
counter.update(new - frame)
frame = new
if encoder == 'svt_av1':
counter.update(frame_probe_source // passes)
if pipe.returncode != 0 and pipe.returncode != -2: # -2 is Ctrl+C for aom
print(f"\nEncoder encountered an error: {pipe.returncode}")
print('\n'.join(encoder_history))
except Exception as e:
_, _, exc_tb = sys.exc_info()
print(f'Error at encode {e}\nAt line {exc_tb.tb_lineno}')
def compose_encoding_queue(files, temp, encoder, params, pipe, passes):
"""
Composing encoding queue with split videos.
:param files: List of files that need to be encoded
:param temp: Path of temp folder
:param encoder: Name of encoder to compose for
:param params: Encoding parameters
:param pipe: FFmpeg pipe
:passes: Number of passes
"""
assert params is not None # params needs to be set with at least get_default_params_for_encoder before this func
encoders = {
'svt_av1': 'SvtAv1EncApp',
'rav1e': 'rav1e',
'aom': 'aomenc',
'vpx': 'vpxenc',
'x265': 'x265'
}
enc_exe = encoders.get(encoder)
inputs = [(temp / "split" / file.name, temp / "encode" / file.name, file)
for file in files]
if encoder in ('aom', 'vpx'):
queue = aom_vpx_encode(inputs, enc_exe, passes, pipe, params)
elif encoder == 'rav1e':
queue = rav1e_encode(inputs, passes, pipe, params)
elif encoder == 'svt_av1':
queue = svt_av1_encode(inputs, passes, pipe, params)
elif encoder == 'x265':
queue = x265_encode(inputs, passes, pipe, params)
# Catch Error
if len(queue) == 0:
er = 'Error in making command queue'
log(er)
terminate()
log(f'Encoding Queue Composed\n'
f'Encoder: {encoder.upper()} Queue Size: {len(queue)} Passes: {passes}\n'
f'Params: {params}\n\n')
return queue
def compose_encoding_queue(files, temp, encoder, params, pipe, passes):
"""Composing encoding queue with split videos."""
encoders = {
'svt_av1': 'SvtAv1EncApp',
'rav1e': 'rav1e',
'aom': 'aomenc',
'vpx': 'vpxenc'
}
enc_exe = encoders.get(encoder)
inputs = [(temp / "split" / file.name, temp / "encode" / file.name, file)
for file in files]
if encoder in ('aom', 'vpx'):
if not params:
if enc_exe == 'vpxenc':
params = '--codec=vp9 --threads=4 --cpu-used=0 --end-usage=q --cq-level=30'
if enc_exe == 'aomenc':
params = '--threads=4 --cpu-used=6 --end-usage=q --cq-level=30'
queue = aom_vpx_encode(inputs, enc_exe, passes, pipe, params)
elif encoder == 'rav1e':
if not params:
params = ' --tiles 8 --speed 6 --quantizer 100'
queue = rav1e_encode(inputs, passes, pipe, params)
elif encoder == 'svt_av1':
if not params:
print('-w -h -fps is required parameters for svt_av1 encoder')
terminate()
queue = svt_av1_encode(inputs, passes, pipe, params)
# Catch Error
if len(queue) == 0:
print('Error in making command queue')
terminate()
return queue, params