def startup(args: Args, chunk_queue: List[Chunk]): """ If resuming, open done file and get file properties from there else get file properties and """ # TODO: move this out and pass in total frames and initial frames done_path = args.temp / 'done.json' if args.resume and done_path.exists(): log('Resuming...\n') with open(done_path) as done_file: data = json.load(done_file) total = data['total'] done = len(data['done']) initial = sum(data['done'].values()) log(f'Resumed with {done} encoded clips done\n\n') else: initial = 0 total = frame_probe_cv2(args.input) if total < 1: total = frame_probe(args.input) d = {'total': total, 'done': {}} with open(done_path, 'w') as done_file: json.dump(d, done_file) clips = len(chunk_queue) args.workers = min(args.workers, clips) print(f'\rQueue: {clips} Workers: {args.workers} Passes: {args.passes}\n' f'Params: {" ".join(args.video_params)}') counter = Manager().Counter(total, initial) args.counter = counter
def _concatenate_mkvmerge(files, output, file_limit, cmd_limit, flip=False): tmp_out = "{}.tmp{}.mkv".format(output, int(flip)) cmd = ["mkvmerge", "-o", tmp_out, files[0]] remaining = [] for i, file in enumerate(files[1:]): new_cmd = cmd + ['+{}'.format(file)] if sum(len(s) for s in new_cmd) < cmd_limit \ and (file_limit == -1 or i < max(1, file_limit - 10)): cmd = new_cmd else: remaining = files[i + 1:] break concat = subprocess.Popen(cmd, stdout=PIPE, universal_newlines=True) message, _ = concat.communicate() concat.wait() if concat.returncode != 0: log(message) print(message) raise Exception if len(remaining) > 0: return _concatenate_mkvmerge( [tmp_out] + remaining, output, file_limit, cmd_limit, not flip) else: return tmp_out
def concatenate_mkvmerge(temp: Path, output): """ Uses mkvmerge to concatenate encoded segments into the final file :param temp: the temp directory :param output: the final output file :return: None """ log('Concatenating\n') encode_files = sorted((temp / 'encode').iterdir()) concat = ' +'.join(shlex.quote(f.as_posix()) for f in encode_files) audio_file = temp / "audio.mkv" audio = audio_file.as_posix() if audio_file.exists() else '' cmd = f' mkvmerge {concat} {shlex.quote(audio)} -o {shlex.quote(output.as_posix())}' concat = subprocess.Popen(cmd, stdout=PIPE, universal_newlines=True, shell=True) output, _ = concat.communicate() concat.wait() if concat.returncode != 0: log(output) print(output) raise Exception
def startup(project: Project, chunk_queue: List[Chunk]): """ If resuming, open done file and get file properties from there else get file properties and """ done_path = project.temp / 'done.json' if project.resume and done_path.exists(): log('Resuming...\n') with open(done_path) as done_file: data = json.load(done_file) set_total_frame_count(project,data['frames']) done = len(data['done']) initial = sum(data['done'].values()) log(f'Resumed with {done} encoded clips done\n\n') else: initial = 0 total = frame_probe_fast(project.input, project.is_vs) d = {'frames': total, 'done': {}} with open(done_path, 'w') as done_file: json.dump(d, done_file) clips = len(chunk_queue) project.workers = min(project.workers, clips) print(f'\rQueue: {clips} Workers: {project.workers} Passes: {project.passes}\n' f'Params: {" ".join(project.video_params)}') counter = Manager().Counter(get_total_frame_count(project), initial) project.counter = counter
def create_video_queue_segment(args: Args, split_locations: List[int]) -> List[Chunk]: """ Create a list of chunks using segmented files :param args: Args :param split_locations: a list of frames to split on :return: A list of chunks """ # segment into separate files segment(args.input, args.temp, split_locations) # get the names of all the split files source_path = args.temp / 'split' queue_files = [x for x in source_path.iterdir() if x.suffix == '.mkv'] queue_files.sort(key=lambda p: p.stem) if len(queue_files) == 0: er = 'Error: No files found in temp/split, probably splitting not working' print(er) log(er) terminate() chunk_queue = [create_chunk_from_segment(args, index, file) for index, file in enumerate(queue_files)] return chunk_queue
def concatenate_mkvmerge(temp: Path, output): """ Uses mkvmerge to concatenate encoded segments into the final file :param temp: the temp directory :param output: the final output file :return: None """ log('Concatenating\n') output = shlex.quote(output.as_posix()) encode_files = sorted((temp / 'encode').iterdir(), key=lambda x: int(x.stem) if x.stem.isdigit() else x.stem) encode_files = [shlex.quote(f.as_posix()) for f in encode_files] if platform.system() == "Linux": import resource file_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) cmd_limit = os.sysconf(os.sysconf_names['SC_ARG_MAX']) else: file_limit = -1 cmd_limit = 32767 audio_file = temp / "audio.mkv" audio = audio_file.as_posix() if audio_file.exists() else '' if len(encode_files) > 1: encode_files = [ _concatenate_mkvmerge(encode_files, output, file_limit, cmd_limit) ] cmd = ['mkvmerge', '-o', output, encode_files[0]] if audio: cmd.append(audio) concat = subprocess.Popen(cmd, stdout=PIPE, universal_newlines=True) message, _ = concat.communicate() concat.wait() if concat.returncode != 0: log(message) print(message) raise Exception # remove temporary files used by recursive concat if os.path.exists("{}.tmp0.mkv".format(output)): os.remove("{}.tmp0.mkv".format(output)) if os.path.exists("{}.tmp1.mkv".format(output)): os.remove("{}.tmp1.mkv".format(output))
def concatenate_ffmpeg(temp: Path, output: Path, encoder: str): """ Uses ffmpeg to concatenate encoded segments into the final file :param temp: the temp directory :param output: the final output file :param encoder: the encoder :return: None """ """With FFMPEG concatenate encoded segments into final file.""" log('Concatenating\n') with open(temp / "concat", 'w') as f: encode_files = sorted((temp / 'encode').iterdir()) f.writelines(f'file {shlex.quote("file:"+str(file.absolute()))}\n' for file in encode_files) # Add the audio/subtitles/else file if one was extracted from the input audio_file = temp / "audio.mkv" if audio_file.exists(): audio = ('-i', audio_file.as_posix(), '-c', 'copy', '-map', '1') else: audio = () if encoder == 'x265': cmd = [ 'ffmpeg', '-y', '-fflags', '+genpts', '-hide_banner', '-loglevel', 'error', '-f', 'concat', '-safe', '0', '-i', (temp / "concat").as_posix(), *audio, '-c', 'copy', '-movflags', 'frag_keyframe+empty_moov', '-map', '0', '-f', 'mp4', output.as_posix() ] concat = subprocess.run(cmd, stdout=PIPE, stderr=STDOUT).stdout else: cmd = [ 'ffmpeg', '-y', '-hide_banner', '-loglevel', 'error', '-f', 'concat', '-safe', '0', '-i', (temp / "concat").as_posix(), *audio, '-c', 'copy', '-map', '0', output.as_posix() ] concat = subprocess.run(cmd, stdout=PIPE, stderr=STDOUT).stdout if len(concat) > 0: log(concat.decode()) print(concat.decode()) raise Exception
def encode(chunk: Chunk, args: Args): """ Encodes a chunk. :param chunk: The chunk to encode :param args: The cli args :return: None """ try: st_time = time.time() chunk_frames = chunk.frames log(f'Enc: {chunk.name}, {chunk_frames} fr\n\n') # Target Vmaf Mode if args.vmaf_target: target_vmaf_routine(args, chunk) ENCODERS[args.encoder].on_before_chunk(args, chunk) # skip first pass if reusing start = 2 if args.reuse_first_pass and args.passes >= 2 else 1 # Run all passes for this chunk for current_pass in range(start, args.passes + 1): tqdm_bar(args, chunk, args.encoder, args.counter, chunk_frames, args.passes, current_pass) ENCODERS[args.encoder].on_after_chunk(args, chunk) # get the number of encoded frames, if no check assume it worked and encoded same number of frames encoded_frames = chunk_frames if args.no_check else frame_check_output( chunk, chunk_frames) # write this chunk as done if it encoded correctly if encoded_frames == chunk_frames: write_progress_file(Path(args.temp / 'done.json'), chunk, encoded_frames) enc_time = round(time.time() - st_time, 2) log(f'Done: {chunk.name} Fr: {encoded_frames}\n' f'Fps: {round(encoded_frames / enc_time, 4)} Time: {enc_time} sec.\n\n' ) except Exception as e: _, _, exc_tb = sys.exc_info() print(f'Error in encoding loop {e}\nAt line {exc_tb.tb_lineno}')
def encode(chunk: Chunk, project: Project): """ Encodes a chunk. :param chunk: The chunk to encode :param project: The cli project :return: None """ st_time = time.time() chunk_frames = chunk.frames log(f'Enc: {chunk.name}, {chunk_frames} fr\n\n') # Target Quality Mode if project.target_quality: if project.target_quality_method == 'per_shot': per_shot_target_quality_routine(project, chunk) if project.target_quality_method == 'per_frame': per_frame_target_quality_routine(project, chunk) ENCODERS[project.encoder].on_before_chunk(project, chunk) # skip first pass if reusing start = 2 if project.reuse_first_pass and project.passes >= 2 else 1 # Run all passes for this chunk for current_pass in range(start, project.passes + 1): tqdm_bar(project, chunk, project.encoder, project.counter, chunk_frames, project.passes, current_pass) ENCODERS[project.encoder].on_after_chunk(project, chunk) # get the number of encoded frames, if no check assume it worked and encoded same number of frames encoded_frames = chunk_frames if project.no_check else frame_check_output( chunk, chunk_frames) # write this chunk as done if it encoded correctly if encoded_frames == chunk_frames: write_progress_file(Path(project.temp / 'done.json'), chunk, encoded_frames) enc_time = round(time.time() - st_time, 2) log(f'Done: {chunk.name} Fr: {encoded_frames}\n' f'Fps: {round(encoded_frames / enc_time, 4)} Time: {enc_time} sec.\n\n' )
def concat_routine(project: Project): """ Runs the concatenation routine with project :param project: the Project :return: None """ try: if project.encoder == 'vvc': vvc_concat(project.temp, project.output_file.with_suffix('.h266')) elif project.mkvmerge: concatenate_mkvmerge(project.temp, project.output_file) else: concatenate_ffmpeg(project.temp, project.output_file, project.encoder) except Exception as e: _, _, exc_tb = sys.exc_info() print(f'Concatenation failed, error\nAt line: {exc_tb.tb_lineno}\nError:{str(e)}') log(f'Concatenation failed, aborting, error: {e}\n') terminate()
def pyscene(video, threshold, min_scene_len): """ Running PySceneDetect detection on source video for segmenting. Optimal threshold settings 15-50 """ if not min_scene_len: min_scene_len = 15 log(f'Starting PySceneDetect:\nThreshold: {threshold}, Min scene lenght: {min_scene_len}\n' ) video_manager = VideoManager([str(video)]) scene_manager = SceneManager() scene_manager.add_detector( ContentDetector(threshold=threshold, min_scene_len=min_scene_len)) base_timecode = video_manager.get_base_timecode() # Work on whole video video_manager.set_duration() # Set downscale factor to improve processing speed. video_manager.set_downscale_factor() # Start video_manager. video_manager.start() scene_manager.detect_scenes(frame_source=video_manager, show_progress=True) # Obtain list of detected scenes. scene_list = scene_manager.get_scene_list(base_timecode) scenes = [int(scene[0].get_frames()) for scene in scene_list] # Remove 0 from list if scenes[0] == 0: scenes.remove(0) log(f'Found scenes: {len(scenes)}\n') return scenes
def select_best_chunking_method(project: Project): if not find_executable('vspipe'): project.chunk_method = 'hybrid' log('Set Chunking Method: Hybrid') else: try: import vapoursynth plugins = vapoursynth.get_core().get_plugins() if 'systems.innocent.lsmas' in plugins: log('Set Chunking Method: L-SMASH\n') project.chunk_method = 'vs_lsmash' elif 'com.vapoursynth.ffms2' in plugins: log('Set Chunking Method: FFMS2\n') project.chunk_method = 'vs_ffms2' except: log('Vapoursynth not installed but vspipe reachable\n' + 'Fallback to Hybrid\n') project.chunk_method = 'hybrid'
def target_vmaf(chunk: Chunk, args: Args): vmaf_cq = [] frames = chunk.frames q_list = [] score = 0 # Make middle probe middle_point = (args.min_q + args.max_q) // 2 q_list.append(middle_point) last_q = middle_point score = vmaf_probe(chunk, last_q, args) vmaf_cq.append((score, last_q)) # Branch if score < args.vmaf_target: next_q = args.min_q q_list.append(args.min_q) else: next_q = args.max_q q_list.append(args.max_q) # Edge case check score = vmaf_probe(chunk, next_q, args) vmaf_cq.append((score, next_q)) if next_q == args.min_q and score < args.vmaf_target: log(f"Chunk: {chunk.name}, Fr: {frames}\n" f"Q: {sorted([x[1] for x in vmaf_cq])}, Early Skip Low CQ\n" f"Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n" f"Target Q: {vmaf_cq[-1][1]} Vmaf: {vmaf_cq[-1][0]}\n\n") return next_q elif next_q == args.max_q and score > args.vmaf_target: log(f"Chunk: {chunk.name}, Fr: {frames}\n" f"Q: {sorted([x[1] for x in vmaf_cq])}, Early Skip High CQ\n" f"Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n" f"Target Q: {vmaf_cq[-1][1]} Vmaf: {vmaf_cq[-1][0]}\n\n") return next_q # VMAF search for _ in range(args.vmaf_steps - 2): new_point = weighted_search(vmaf_cq[-2][1], vmaf_cq[-2][0], vmaf_cq[-1][1], vmaf_cq[-1][0], args.vmaf_target) if new_point in [x[1] for x in vmaf_cq]: break last_q = new_point q_list.append(new_point) score = vmaf_probe(chunk, new_point, args) next_q = get_closest(q_list, last_q, positive=score >= args.vmaf_target) vmaf_cq.append((score, new_point)) q, q_vmaf = get_target_q(vmaf_cq, args.vmaf_target) log(f'Chunk: {chunk.name}, Fr: {frames}\n' f'Q: {sorted([x[1] for x in vmaf_cq])}\n' f'Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n' f'Target Q: {q} Vmaf: {q_vmaf}\n\n') # Plot Probes if args.vmaf_plots and len(vmaf_cq) > 3: plot_probes(args, vmaf_cq, chunk, frames) return q
def on_before_chunk(self, project: Project, chunk: Chunk) -> None: # vvc requires a yuv files as input, make it here log(f'Creating yuv for chunk {chunk.name}\n') Vvc.to_yuv(chunk) log(f'Created yuv for chunk {chunk.name}\n') super().on_before_chunk(project, chunk)
def pyscene(video, threshold, min_scene_len, is_vs, temp): """ Running PySceneDetect detection on source video for segmenting. Optimal threshold settings 15-50 """ if not min_scene_len: min_scene_len = 15 log(f'Starting PySceneDetect:\nThreshold: {threshold}, Min scene length: {min_scene_len}\n Is Vapoursynth input: {is_vs}\n' ) if is_vs: # Handling vapoursynth, so we need to create a named pipe to feed to VideoManager. # TODO: Do we clean this up after pyscenedetect has run, or leave it as part of the temp dir, where it will be cleaned up later? if sys.platform == "linux": vspipe_fifo = temp / 'vspipe.y4m' mkfifo(vspipe_fifo) else: vspipe_fifo = None vspipe_cmd = compose_vapoursynth_pipe(video, vspipe_fifo) vspipe_process = Popen(vspipe_cmd) # Get number of frames from Vapoursynth script to pass as duration to VideoManager. # We need to pass the number of frames to the manager, otherwise it won't close the # receiving end of the pipe, and will simply sit waiting after vspipe has finished sending # the last frame. frames = frame_probe(video) video_manager = VideoManager([str(vspipe_fifo if is_vs else video)]) scene_manager = SceneManager() scene_manager.add_detector( ContentDetector(threshold=threshold, min_scene_len=min_scene_len)) base_timecode = video_manager.get_base_timecode() video_manager.set_duration(duration=FrameTimecode( frames, video_manager.get_framerate()) if is_vs else None) # Set downscale factor to improve processing speed. video_manager.set_downscale_factor() # Start video_manager. video_manager.start() scene_manager.detect_scenes(frame_source=video_manager, show_progress=True) # If fed using a vspipe process, ensure that vspipe has finished. if is_vs: vspipe_process.wait() # Obtain list of detected scenes. scene_list = scene_manager.get_scene_list(base_timecode) scenes = [int(scene[0].get_frames()) for scene in scene_list] # Remove 0 from list if scenes[0] == 0: scenes.remove(0) log(f'Found scenes: {len(scenes)}\n') return scenes
def target_vmaf(chunk: Chunk, args: Args): vmaf_cq = [] frames = chunk.frames q_list = [] score = 0 # Make middle probe middle_point = (args.min_q + args.max_q) // 2 q_list.append(middle_point) last_q = middle_point score = vmaf_probe(chunk, last_q, args) vmaf_cq.append((score, last_q)) if args.vmaf_steps < 3: #Use Euler's method with known relation between cq and vmaf vmaf_cq_deriv = -0.18 ## Formula -ln(1-score/100) = vmaf_cq_deriv*last_q + constant #constant = -ln(1-score/100) - vmaf_cq_deriv*last_q ## Formula -ln(1-args.vmaf_target/100) = vmaf_cq_deriv*cq + constant #cq = (-ln(1-args.vmaf_target/100) - constant)/vmaf_cq_deriv next_q = int(round(last_q + (transform_vmaf(args.vmaf_target)-transform_vmaf(score))/vmaf_cq_deriv)) #Clamp if next_q < args.min_q: next_q = args.min_q if args.max_q < next_q: next_q = args.max_q #Single probe cq guess or exit to avoid divide by zero if args.vmaf_steps == 1 or next_q == last_q: return next_q #Second probe at guessed value score_2 = vmaf_probe(chunk, next_q, args) #Calculate slope vmaf_cq_deriv = (transform_vmaf(score_2)-transform_vmaf(score))/(next_q-last_q) #Same deal different slope next_q = int(round(next_q+(transform_vmaf(args.vmaf_target)-transform_vmaf(score_2))/vmaf_cq_deriv)) #Clamp if next_q < args.min_q: next_q = args.min_q if args.max_q < next_q: next_q = args.max_q return next_q # Initialize search boundary vmaf_lower = score vmaf_upper = score vmaf_cq_lower = last_q vmaf_cq_upper = last_q # Branch if score < args.vmaf_target: next_q = args.min_q q_list.append(args.min_q) else: next_q = args.max_q q_list.append(args.max_q) # Edge case check score = vmaf_probe(chunk, next_q, args) vmaf_cq.append((score, next_q)) if next_q == args.min_q and score < args.vmaf_target: log(f"Chunk: {chunk.name}, Fr: {frames}\n" f"Q: {sorted([x[1] for x in vmaf_cq])}, Early Skip Low CQ\n" f"Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n" f"Target Q: {vmaf_cq[-1][1]} Vmaf: {vmaf_cq[-1][0]}\n\n") return next_q elif next_q == args.max_q and score > args.vmaf_target: log(f"Chunk: {chunk.name}, Fr: {frames}\n" f"Q: {sorted([x[1] for x in vmaf_cq])}, Early Skip High CQ\n" f"Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n" f"Target Q: {vmaf_cq[-1][1]} Vmaf: {vmaf_cq[-1][0]}\n\n") return next_q # Set boundary if score < args.vmaf_target: vmaf_lower = score vmaf_cq_lower = next_q else: vmaf_upper = score vmaf_cq_upper = next_q # VMAF search for _ in range(args.vmaf_steps - 2): new_point = weighted_search(vmaf_cq_lower, vmaf_lower, vmaf_cq_upper, vmaf_upper, args.vmaf_target) if new_point in [x[1] for x in vmaf_cq]: break q_list.append(new_point) score = vmaf_probe(chunk, new_point, args) vmaf_cq.append((score, new_point)) # Update boundary if score < args.vmaf_target: vmaf_lower = score vmaf_cq_lower = new_point else: vmaf_upper = score vmaf_cq_upper = new_point q, q_vmaf = get_target_q(vmaf_cq, args.vmaf_target) log(f'Chunk: {chunk.name}, Fr: {frames}\n' f'Q: {sorted([x[1] for x in vmaf_cq])}\n' f'Vmaf: {sorted([x[0] for x in vmaf_cq], reverse=True)}\n' f'Target Q: {q} Vmaf: {q_vmaf}\n\n') # Plot Probes if args.vmaf_plots and len(vmaf_cq) > 3: plot_probes(args, vmaf_cq, chunk, frames) return q
def aom_keyframes(video_path: Path, stat_file, min_scene_len, ffmpeg_pipe, video_params): """[Get frame numbers for splits from aomenc 1 pass stat file] """ log(f'Started aom_keyframes scenedetection\nParams: {video_params}\n') # Get CV2 fast framecount video = cv2.VideoCapture(video_path.as_posix()) total = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) video.release() if total < 1: total = frame_probe(video_path) f, e = compose_aomsplit_first_pass_command(video_path, stat_file, ffmpeg_pipe, video_params) tqdm_bar = tqdm(total=total, initial=0, dynamic_ncols=True, unit="fr", leave=True, smoothing=0.2) ffmpeg_pipe = subprocess.Popen(f, stdout=PIPE, stderr=STDOUT) pipe = subprocess.Popen(e, stdin=ffmpeg_pipe.stdout, stdout=PIPE, stderr=STDOUT, universal_newlines=True) encoder_history = deque(maxlen=20) frame = 0 while True: line = pipe.stdout.readline() if len(line) == 0 and pipe.poll() is not None: break line = line.strip() if line: encoder_history.append(line) match = re.search(r"frame.*?/([^ ]+?) ", line) if match: new = int(match.group(1)) if new > frame: tqdm_bar.update(new - frame) frame = new if pipe.returncode != 0 and pipe.returncode != -2: # -2 is Ctrl+C for aom enc_hist = '\n'.join(encoder_history) er = f"\nAom first pass encountered an error: {pipe.returncode}\n{enc_hist}" log(er) print(er) if not stat_file.exists(): terminate() else: # aom crashed, but created keyframes.log, so we will try to continue print( "WARNING: Aom first pass crashed, but created a first pass file. Keyframe splitting may not be accurate." ) # aom kf-min-dist defaults to 0, but hardcoded to 3 in pass2_strategy.c test_candidate_kf. 0 matches default aom behavior # https://aomedia.googlesource.com/aom/+/8ac928be918de0d502b7b492708d57ad4d817676/av1/av1_cx_iface.c#2816 # https://aomedia.googlesource.com/aom/+/ce97de2724d7ffdfdbe986a14d49366936187298/av1/encoder/pass2_strategy.c#1907 min_scene_len = 0 if min_scene_len is None else min_scene_len keyframes = find_aom_keyframes(stat_file, min_scene_len) return keyframes