def startup(project: Project, chunk_queue: List[Chunk]):
"""
If resuming, open done file and get file properties from there
else get file properties and
"""
done_path = project.temp / 'done.json'
if project.resume and done_path.exists():
log('Resuming...\n')
with open(done_path) as done_file:
data = json.load(done_file)
set_total_frame_count(project,data['frames'])
done = len(data['done'])
initial = sum(data['done'].values())
log(f'Resumed with {done} encoded clips done\n\n')
else:
initial = 0
total = frame_probe_fast(project.input, project.is_vs)
d = {'frames': total, 'done': {}}
with open(done_path, 'w') as done_file:
json.dump(d, done_file)
clips = len(chunk_queue)
project.workers = min(project.workers, clips)
print(f'\rQueue: {clips} Workers: {project.workers} Passes: {project.passes}\n'
f'Params: {" ".join(project.video_params)}')
counter = Manager().Counter(get_total_frame_count(project), initial)
project.counter = counter
def get_project(self):
"""
Create and return project object with all parameters
"""
if not self.parsed:
self.parse()
self.project = Project(self.parsed)
if self.project.config:
self.save_load_project_file()
return self.project
def outputs_filenames(project: Project):
"""
Set output filename
:param project: the Project
"""
suffix = '.mkv'
project.output_file = Path(project.output_file).with_suffix(suffix) if project.output_file \
else Path(f'{project.input.stem}_{project.encoder}{suffix}')
def select_best_chunking_method(project: Project):
if not find_executable('vspipe'):
project.chunk_method = 'hybrid'
log('Set Chunking Method: Hybrid')
else:
try:
import vapoursynth
plugins = vapoursynth.get_core().get_plugins()
if 'systems.innocent.lsmas' in plugins:
log('Set Chunking Method: L-SMASH\n')
project.chunk_method = 'vs_lsmash'
elif 'com.vapoursynth.ffms2' in plugins:
log('Set Chunking Method: FFMS2\n')
project.chunk_method = 'vs_ffms2'
except:
log('Vapoursynth not installed but vspipe reachable\n' +
'Fallback to Hybrid\n')
project.chunk_method = 'hybrid'
def setup_encoder(project: Project):
"""
Setup encoder params and passes
:param project: the Project
"""
encoder = ENCODERS[project.encoder]
# validate encoder settings
settings_valid, error_msg = encoder.is_valid(project)
if not settings_valid:
print(error_msg)
terminate()
if project.passes is None:
project.passes = encoder.default_passes
project.video_params = encoder.default_args if project.video_params is None \
else shlex.split(project.video_params)
validate_inputs(project)
def startup_check(project: Project):
"""
Performing essential checks at startup_check
Set constant values
"""
if sys.version_info < (3, 6):
print('Python 3.6+ required')
sys.exit()
if sys.platform == 'linux':
def restore_term():
os.system("stty sane")
atexit.register(restore_term)
if not project.chunk_method:
select_best_chunking_method(project)
# project.is_vs = is_vapoursynth(project.input)
if project.is_vs:
project.chunk_method = 'vs_ffms2'
check_exes(project)
set_target_quality(project)
if project.reuse_first_pass and project.encoder != 'aom' and project.split_method != 'aom_keyframes':
print('Reusing the first pass is only supported with \
the aom encoder and aom_keyframes split method.')
terminate()
setup_encoder(project)
# No check because vvc
if project.encoder == 'vvc':
project.no_check = True
if project.encoder == 'svt_vp9' and project.passes == 2:
print("Implicitly changing 2 pass svt-vp9 to 1 pass\n2 pass svt-vp9 isn't supported")
project.passes = 1
project.audio_params = shlex.split(project.audio_params)
project.ffmpeg = shlex.split(project.ffmpeg)
project.pix_format = ['-strict', '-1', '-pix_fmt', project.pix_format]
project.ffmpeg_pipe = [*project.ffmpeg, *project.pix_format,
'-bufsize', '50000K', '-f', 'yuv4mpegpipe', '-']
def calc_split_locations(project: Project) -> List[int]:
"""
Determines a list of frame numbers to split on with pyscenedetect or aom keyframes
:param project: the Project
:return: A list of frame numbers
"""
# inherit video params from aom encode unless we are using a different encoder, then use defaults
aom_keyframes_params = project.video_params if (
project.encoder == 'aom') else AOM_KEYFRAMES_DEFAULT_PARAMS
sc = []
# Splitting using PySceneDetect
if project.split_method == 'pyscene':
log(f'Starting scene detection Threshold: {project.threshold}, Min_scene_length: {project.min_scene_len}\n'
)
try:
sc = pyscene(project.input, project.threshold,
project.min_scene_len, project.is_vs, project.temp)
except Exception as e:
log(f'Error in PySceneDetect: {e}\n')
print(f'Error in PySceneDetect{e}\n')
terminate()
# Splitting based on aom keyframe placement
elif project.split_method == 'aom_keyframes':
stat_file = project.temp / 'keyframes.log'
sc = aom_keyframes(project.input, stat_file, project.min_scene_len,
project.ffmpeg_pipe, aom_keyframes_params,
project.is_vs)
else:
print(
f'No valid split option: {project.split_method}\nValid options: "pyscene", "aom_keyframes"'
)
terminate()
# Write scenes to file
if project.scenes:
write_scenes_to_file(sc, project.get_frames(), project.scenes)
return sc
def encode_file(project: Project):
"""
Encodes a single video file on the local machine.
:param project: The project for this encode
:return: None
"""
setup(project)
set_log(project.logging, project.temp)
# find split locations
split_locations = split_routine(project, project.resume)
# create a chunk queue
chunk_queue = load_or_gen_chunk_queue(project, project.resume,
split_locations)
# things that need to be done only the first time
if not project.resume:
extract_audio(project.input, project.temp, project.audio_params)
if project.reuse_first_pass:
segment_first_pass(project.temp, split_locations)
# do encoding loop
project.workers = determine_resources(project.encoder, project.workers)
startup(project, chunk_queue)
encoding_loop(project, chunk_queue)
# concat
concat_routine(project)
if project.vmaf or project.vmaf_plots:
plot_vmaf(project.input, project.output_file, project,
project.vmaf_path, project.vmaf_res)
# Delete temp folders
if not project.keep:
shutil.rmtree(project.temp)
def split_routine(project: Project, resuming: bool) -> List[int]:
"""
Performs the split routine. Runs pyscenedetect/aom keyframes and adds in extra splits if needed
:param project: the Project
:param resuming: if the encode is being resumed
:return: A list of frames to split on
"""
scene_file = project.temp / 'scenes.txt'
# if resuming, we already have the split file, so just read that and return
if resuming:
scenes, frames = read_scenes_from_file(scene_file)
project.set_frames(frames)
return scenes
# Run scenedetection or skip
if project.scenes == '0':
log('Skipping scene detection\n')
scenes = []
# Read saved scenes:
elif project.scenes and Path(project.scenes).exists():
log('Using Saved Scenes\n')
scenes, frames = read_scenes_from_file(Path(project.scenes))
project.set_frames(frames)
else:
# determines split frames with pyscenedetect or aom keyframes
scenes = calc_split_locations(project)
if project.scenes and Path(project.scenes).exists():
write_scenes_to_file(scenes, project.get_frames(),
Path(project.scenes))
# Write internal scenes
write_scenes_to_file(scenes, project.get_frames(), scene_file)
# Applying extra splits
if project.extra_split:
scenes = extra_splits(project, scenes)
# write scenes for resuming later if needed
return scenes
class Args:
"""
Class responsible for arg parsing
Creation of original project file
#TODO: validating/creating difference of different args
"""
def __init__(self):
self.parser = self.arg_parsing()
self.defaults = self.get_defaults()
self.parsed = None
self.project = None
def get_project(self):
"""
Create and return project object with all parameters
"""
if not self.parsed:
self.parse()
self.project = Project(self.parsed)
if self.project.config:
self.save_load_project_file()
return self.project
def get_defaults(self) -> dict:
"""
Get dictionary of default values specified in arg_parsing()
"""
return vars(self.parser.parse_args([]))
def get_difference(self) -> dict:
"""
Return difference of defaults and new
"""
return dict([x for x in self.parsed.items() if x not in self.defaults.items()])
def parse(self):
"""
Parse command line parameters provided by user
"""
self.parsed = vars(self.parser.parse_args())
if not self.parsed['input']:
self.parser.print_help()
sys.exit()
def save_load_project_file(self):
"""
Saves current/Loads given project file, loads saved project first and when overwrites only unique values from current parse
"""
cfg_path = Path(self.project.config)
if cfg_path.exists():
new = self.get_difference()
self.project.load_project_from_file(self.project.config)
self.project.load_project(new)
else:
self.project.save_project_to_file(self.project.config)
def arg_parsing(self):
"""Command line parsing and setting default variables"""
parser = argparse.ArgumentParser()
# Input/Output/Temp
io_group = parser.add_argument_group('Input and Output')
io_group.add_argument('--input', '-i', nargs='+', type=Path, help='Input File')
io_group.add_argument('--temp', type=Path, default=None, help='Set temp folder path')
io_group.add_argument('--output_file', '-o', type=Path, default=None, help='Specify output file')
io_group.add_argument('--mkvmerge', help='Use mkvmerge instead of ffmpeg to concatenate', action='store_true')
io_group.add_argument('--logging', '-log', type=str, default=None, help='Enable logging')
io_group.add_argument('--resume', '-r', help='Resuming previous session', action='store_true')
io_group.add_argument('--keep', help='Keep temporally folder after encode', action='store_true')
io_group.add_argument('--config', '-c', type=str, default=None, help="Path to config file, create if doesn't exists")
# Splitting
split_group = parser.add_argument_group('Splitting')
split_group.add_argument('--chunk_method', '-cm', type=str, default=None, help='Method for creating chunks',
choices=['select', 'vs_ffms2', 'vs_lsmash', 'hybrid'])
split_group.add_argument('--scenes', '-s', type=str, default=None, help='File location for scenes')
split_group.add_argument('--split_method', type=str, default='pyscene', help='Specify splitting method',
choices=['pyscene', 'aom_keyframes'])
split_group.add_argument('--extra_split', '-xs', type=int, default=240,
help='Number of frames after which make split')
# PySceneDetect split
split_group.add_argument('--threshold', '-tr', type=float, default=35, help='PySceneDetect Threshold')
split_group.add_argument('--min_scene_len', type=int, default=60, help='Minimum number of frames in a split')
# AOM Keyframe split
split_group.add_argument('--reuse_first_pass', help='Reuse the first pass from aom_keyframes split on the chunks',
action='store_true')
# Encoding
encode_group = parser.add_argument_group('Encoding')
encode_group.add_argument('--passes', '-p', type=int, default=None, help='Specify encoding passes', choices=[1, 2])
encode_group.add_argument('--video_params', '-v', type=str, default=None, help='encoding settings')
encode_group.add_argument('--encoder', '-enc', type=str, default='aom', help='Choosing encoder',
choices=['aom', 'svt_av1', 'svt_vp9', 'rav1e', 'vpx', 'x265', 'x264', 'vvc'])
encode_group.add_argument('--workers', '-w', type=int, default=0, help='Number of workers')
encode_group.add_argument('--no_check', '-n', help='Do not check encodings', action='store_true')
encode_group.add_argument('--force', help="Force encoding if input args seen as invalid", action='store_true')
# VVC
encode_group.add_argument('--vvc_conf', type=Path, default=None, help='Path to VVC confing file')
# FFmpeg params
ffmpeg_group = parser.add_argument_group('FFmpeg')
ffmpeg_group.add_argument('--ffmpeg', '-ff', type=str, default='', help='FFmpeg commands')
ffmpeg_group.add_argument('--audio_params', '-a', type=str, default='-c:a copy', help='FFmpeg audio settings')
ffmpeg_group.add_argument('--pix_format', '-fmt', type=str, default='yuv420p10le', help='FFmpeg pixel format')
# Vmaf
vmaf_group = parser.add_argument_group('VMAF')
vmaf_group.add_argument('--vmaf', help='Calculating vmaf after encode', action='store_true')
vmaf_group.add_argument('--vmaf_path', type=Path, default=None, help='Path to vmaf models')
vmaf_group.add_argument('--vmaf_res', type=str, default="1920x1080", help='Resolution used in vmaf calculation')
vmaf_group.add_argument('--n_threads', type=int, default=None, help='Threads for vmaf calculation')
# Target Quality
tq_group = parser.add_argument_group('Target Quality')
tq_group.add_argument('--target_quality', type=float, help='Value to target')
tq_group.add_argument('--target_quality_method', type=str, default='per_shot',
help='Method selection for target quality', choices=['per_frame', 'per_shot'])
tq_group.add_argument('--probes', type=int, default=4, help='Number of probes to make for target_quality')
tq_group.add_argument('--min_q', type=int, default=None, help='Min q for target_quality')
tq_group.add_argument('--max_q', type=int, default=None, help='Max q for target_quality')
tq_group.add_argument('--vmaf_plots', help='Make plots of probes in temp folder', action='store_true')
tq_group.add_argument('--probing_rate', type=int, default=4, help='Framerate for probes, 0 - original')
tq_group.add_argument('--vmaf_filter', type=str, default=None,
help='Filter applied to source at vmaf calcualation, use if you crop source')
# Misc
misc_group = parser.add_argument_group('Misc')
misc_group.add_argument('--version', action='version', version=f'Av1an version: {4.7}')
# Initialize project with initial values
return parser
def arg_parsing():
"""Command line parsing and setting default variables"""
parser = argparse.ArgumentParser()
# Input/Output/Temp
io_group = parser.add_argument_group('Input and Output')
io_group.add_argument('--input',
'-i',
nargs='+',
type=Path,
help='Input File')
io_group.add_argument('--temp',
type=Path,
default=None,
help='Set temp folder path')
io_group.add_argument('--output_file',
'-o',
type=Path,
default=None,
help='Specify output file')
io_group.add_argument('--mkvmerge',
help='Use mkvmerge instead of ffmpeg to concatenate',
action='store_true')
io_group.add_argument('--logging',
'-log',
type=str,
default=None,
help='Enable logging')
io_group.add_argument('--resume',
'-r',
help='Resuming previous session',
action='store_true')
io_group.add_argument('--keep',
help='Keep temporally folder after encode',
action='store_true')
io_group.add_argument('--force',
help="Force encoding if input args seen as invalid",
action='store_true')
# Splitting
split_group = parser.add_argument_group('Splitting')
split_group.add_argument(
'--chunk_method',
'-cm',
type=str,
default=None,
help='Method for creating chunks',
choices=['select', 'vs_ffms2', 'vs_lsmash', 'hybrid'])
split_group.add_argument('--scenes',
'-s',
type=str,
default=None,
help='File location for scenes')
split_group.add_argument('--split_method',
type=str,
default='pyscene',
help='Specify splitting method',
choices=['pyscene', 'aom_keyframes'])
split_group.add_argument('--extra_split',
'-xs',
type=int,
default=240,
help='Number of frames after which make split')
# PySceneDetect split
split_group.add_argument('--threshold',
'-tr',
type=float,
default=35,
help='PySceneDetect Threshold')
split_group.add_argument('--min_scene_len',
type=int,
default=60,
help='Minimum number of frames in a split')
# AOM Keyframe split
split_group.add_argument(
'--reuse_first_pass',
help='Reuse the first pass from aom_keyframes split on the chunks',
action='store_true')
# Encoding
encode_group = parser.add_argument_group('Encoding')
encode_group.add_argument('--passes',
'-p',
type=int,
default=None,
help='Specify encoding passes',
choices=[1, 2])
encode_group.add_argument('--video_params',
'-v',
type=str,
default=None,
help='encoding settings')
encode_group.add_argument('--encoder',
'-enc',
type=str,
default='aom',
help='Choosing encoder',
choices=[
'aom', 'svt_av1', 'svt_vp9', 'rav1e', 'vpx',
'x265', 'x264', 'vvc'
])
encode_group.add_argument('--workers',
'-w',
type=int,
default=0,
help='Number of workers')
encode_group.add_argument('--no_check',
'-n',
help='Do not check encodings',
action='store_true')
# VVC
encode_group.add_argument('--vvc_conf',
type=Path,
default=None,
help='Path to VVC confing file')
# FFmpeg params
ffmpeg_group = parser.add_argument_group('FFmpeg')
ffmpeg_group.add_argument('--ffmpeg',
'-ff',
type=str,
default='',
help='FFmpeg commands')
ffmpeg_group.add_argument('--audio_params',
'-a',
type=str,
default='-c:a copy',
help='FFmpeg audio settings')
ffmpeg_group.add_argument('--pix_format',
'-fmt',
type=str,
default='yuv420p10le',
help='FFmpeg pixel format')
# Vmaf
vmaf_group = parser.add_argument_group('VMAF')
vmaf_group.add_argument('--vmaf',
help='Calculating vmaf after encode',
action='store_true')
vmaf_group.add_argument('--vmaf_path',
type=Path,
default=None,
help='Path to vmaf models')
vmaf_group.add_argument('--vmaf_res',
type=str,
default="1920x1080",
help='Resolution used in vmaf calculation')
vmaf_group.add_argument('--n_threads',
type=int,
default=None,
help='Threads for vmaf calculation')
# Target Quality
tq_group = parser.add_argument_group('Target Quality')
tq_group.add_argument('--target_quality',
type=float,
help='Value of Vmaf to target')
tq_group.add_argument('--target_quality_method',
type=str,
default='per_frame',
help='Method selection for target quality')
tq_group.add_argument('--probes',
type=int,
default=4,
help='Number of probes to make for target_quality')
tq_group.add_argument('--min_q',
type=int,
default=None,
help='Min q for target_quality')
tq_group.add_argument('--max_q',
type=int,
default=None,
help='Max q for target_quality')
tq_group.add_argument('--vmaf_plots',
help='Make plots of probes in temp folder',
action='store_true')
tq_group.add_argument('--probing_rate',
type=int,
default=4,
help='Framerate for probes, 0 - original')
tq_group.add_argument(
'--vmaf_filter',
type=str,
default=None,
help=
'Filter applied to source at vmaf calcualation, use if you crop source'
)
# Misc
misc_group = parser.add_argument_group('Misc')
misc_group.add_argument('--version',
action='version',
version=f'Av1an version: {4}')
# Initialize project with initial values
proj = Project(vars(parser.parse_args()))
if not proj.input:
parser.print_help()
exit()
return proj