def convert_file(self, data, process_id): file_in = data.get("file_in") file_out = data.get("file_out") ffmpeg_args = data.get("ffmpeg_args") # Create output path if not exists common.ensure_dir(file_out) # Convert file success = False try: # Reset to defaults self.ffmpeg.set_info_defaults() # Fetch source file info self.ffmpeg.set_file_in(file_in) # Read video information for the input file file_probe = self.ffmpeg.file_in['file_probe'] if not file_probe: return False if ffmpeg_args: success = self.ffmpeg.convert_file_and_fetch_progress(file_in, ffmpeg_args) self.current_task.save_ffmpeg_log(self.ffmpeg.ffmpeg_cmd_stdout) except ffmpeg.FFMPEGHandleConversionError as e: # Fetch ffmpeg stdout and append it to the current task object (to be saved during post process) self.current_task.save_ffmpeg_log(self.ffmpeg.ffmpeg_cmd_stdout) self._log("Error while executing the FFMPEG command {}. " "Download FFMPEG command dump from history for more information.".format(file_in), message2=str(e), level="error") return success
def process_item(self): abspath = self.current_task.get_source_abspath() self._log("{} processing job - {}".format(self.name, abspath)) # Create output path if not exists common.ensure_dir(self.current_task.cache_path) # Convert file success = False try: ffmpeg_args = self.current_task.ffmpeg.generate_ffmpeg_args() if ffmpeg_args: success = self.current_task.ffmpeg.convert_file_and_fetch_progress( abspath, self.current_task.cache_path, ffmpeg_args) self.current_task.ffmpeg_log = self.current_task.ffmpeg.ffmpeg_cmd_stdout except ffmpeg.FFMPEGHandleConversionError as e: # Fetch ffmpeg stdout and append it to the current task object (to be saved during post process) self.current_task.ffmpeg_log = self.current_task.ffmpeg.ffmpeg_cmd_stdout self._log( "Error while executing the FFMPEG command {}. " "Download FFMPEG command dump from history for more information." .format(abspath), message2=str(e), level="error") if success: # If file conversion was successful, we will get here self._log("Successfully converted file '{}'".format(abspath)) return True self._log("Failed to convert file '{}'".format(abspath), level='warning') return False
def process_file_with_configured_settings(self, vid_file_path): # Parse input path src_file = os.path.basename(vid_file_path) src_path = os.path.abspath(vid_file_path) src_folder = os.path.dirname(src_path) # Get container extension container = unffmpeg.containers.grab_module(self.settings.OUT_CONTAINER) container_extension = container.container_extension() # Parse an output cache path out_folder = "unmanic_file_conversion-{}".format(time.time()) out_file = "{}-{}.{}".format(os.path.splitext(src_file)[0], time.time(), container_extension) out_path = os.path.join(self.settings.CACHE_PATH, out_folder, out_file) # Create output path if not exists common.ensure_dir(out_path) # Reset all info self.set_info_defaults() # Fetch file info self.set_file_in(vid_file_path) # Convert file success = False ffmpeg_args = self.generate_ffmpeg_args() if ffmpeg_args: success = self.convert_file_and_fetch_progress(src_path, out_path, ffmpeg_args) if success: # Move file back to original folder and remove source success = self.post_process_file(out_path) if success: destPath = os.path.join(src_folder, out_file) self._log("Moving file {} --> {}".format(out_path, destPath)) shutil.move(out_path, destPath) try: self.post_process_file(destPath) except FFMPEGHandlePostProcessError: success = False if success: # If successful move, remove source # TODO: Add env variable option to keep src if src_path != destPath: self._log("Removing source: {}".format(src_path)) os.remove(src_path) else: self._log("Copy / Replace failed during post processing '{}'".format(out_path), level='warning') return False else: self._log("Encoded file failed post processing test '{}'".format(out_path), level='warning') return False else: self._log("Failed processing file '{}'".format(src_path), level='warning') return False # If file conversion was successful, we will get here self._log("Successfully processed file '{}'".format(src_path)) return True
def test_convert_all_files_for_success(self): """ Ensure all small test files are able to be converted. :return: """ # Test all small files of various containers for video_file in os.listdir(os.path.join(self.tests_videos_dir, 'small')): filename, file_extension = os.path.splitext(os.path.basename(video_file)) infile = os.path.join(self.tests_videos_dir, 'small', video_file) outfile = os.path.join(self.tests_tmp_dir, filename + '.mkv') common.ensure_dir(outfile) self.convert_single_file(infile, outfile)
def test_read_file_info_for_failure(self): """ Ensure that and exception is thrown if ffprobe is unable to read a file. :return: """ # Set project root path tmp_dir = self.tests_tmp_dir fail_file = os.path.join(tmp_dir, 'test_failure.mkv') # Test common.ensure_dir(fail_file) common.touch(fail_file) with pytest.raises(unffmpeg.exceptions.ffprobe.FFProbeError): self.ffmpeg.file_probe(fail_file)
def test_convert_all_faulty_files_for_success(self): """ Ensure all faulty test files are able to be converted. These files have various issues with them that may cause ffmpeg to fail if not configured correctly. :return: """ # Test all faulty files can be successfully converted (these files have assorted issues) for video_file in os.listdir(os.path.join(self.tests_videos_dir, 'faulty')): filename, file_extension = os.path.splitext(os.path.basename(video_file)) infile = os.path.join(self.tests_videos_dir, 'faulty', video_file) outfile = os.path.join(self.tests_tmp_dir, filename + '.mkv') common.ensure_dir(outfile) self.convert_single_file(infile, outfile)
def test_process_file_for_success(self): """ This will test the FFMPEGHandle for processing a file automatically using configured settings. :return: """ # Set project root path tmp_dir = self.tests_tmp_dir # Test just the first file found in the med folder for video_file in os.listdir( os.path.join(self.tests_videos_dir, 'small')): filename, file_extension = os.path.splitext( os.path.basename(video_file)) infile = os.path.join(self.tests_videos_dir, 'small', video_file) # Copy the file to a tmp location (it will be replaced) testfile = os.path.join(tmp_dir, filename + file_extension) self._log(infile, testfile) common.ensure_dir(testfile) shutil.copy(infile, testfile) assert self.ffmpeg.process_file_with_configured_settings(testfile) break
def convert_single_file(self, infile, outfile, test_for_failure=False): if not os.path.exists(infile): self._log("No such file: {}".format(infile)) sys.exit(1) # Ensure the directory exists common.ensure_dir(outfile) # Remove the output file if it already exists if os.path.exists(outfile): os.remove(outfile) # Setup ffmpeg args built_args = self.build_ffmpeg_args(infile, outfile, test_for_failure) # Run conversion process self._log("Converting {} -> {}".format(infile, outfile)) # Fetch file info self.ffmpeg.set_file_in(infile) assert self.ffmpeg.convert_file_and_fetch_progress(infile, built_args) if not test_for_failure: assert self.ffmpeg.post_process_file(outfile) elif test_for_failure: with pytest.raises(ffmpeg.FFMPEGHandlePostProcessError): self.ffmpeg.post_process_file(outfile)
def __exec_command_subprocess(self, data): """ Executes a command as a shell subprocess. Uses the given parser to record progress data from the shell STDOUT. :param data: :return: """ # Fetch command to execute. exec_command = data.get("exec_command", []) # Fetch the command progress parser function command_progress_parser = data.get("command_progress_parser", default_progress_parser) # Log the command for debugging command_string = exec_command if isinstance(exec_command, list): command_string = ' '.join(exec_command) self._log("Executing: {}".format(command_string), level='debug') # Append start of command to worker subprocess stdout self.worker_log += [ '\n\n', 'COMMAND:\n', command_string, '\n\n', 'LOG:\n', ] # Create output path if not exists common.ensure_dir(data.get("file_out")) # Convert file try: proc_pause_time = 0 proc_start_time = time.time() # Execute command if isinstance(exec_command, list): sub_proc = subprocess.Popen(exec_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, errors='replace') elif isinstance(exec_command, str): sub_proc = subprocess.Popen(exec_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, errors='replace', shell=True) else: raise Exception( "Plugin's returned 'exec_command' object must be either a list or a string. Received type {}.".format( type(exec_command))) # Fetch process using psutil for control (sending SIGSTOP on windows will not work) proc = psutil.Process(pid=sub_proc.pid) # Set process priority on posix systems # TODO: Test how this will work on Windows if os.name == "posix": try: parent_proc = psutil.Process(os.getpid()) parent_proc_nice = parent_proc.nice() proc.nice(parent_proc_nice + 1) except Exception as e: self._log("Unable to lower priority of subprocess. Subprocess should continue to run at normal priority", str(e), level='warning') # Record PID and PROC self.worker_subprocess = sub_proc self.worker_subprocess_pid = sub_proc.pid # Poll process for new output until finished while not self.redundant_flag.is_set(): line_text = sub_proc.stdout.readline() # Fetch command stdout and append it to the current task object (to be saved during post process) self.worker_log.append(line_text) # Check if the command has completed. If it has, exit the loop if line_text == '' and sub_proc.poll() is not None: self._log("Subprocess task completed!", level='debug') break # Parse the progress try: progress_dict = command_progress_parser(line_text) self.worker_subprocess_percent = progress_dict.get('percent', '0') self.worker_subprocess_elapsed = str(time.time() - proc_start_time - proc_pause_time) except Exception as e: # Only need to show any sort of exception if we have debugging enabled. # So we should log it as a debug rather than an exception. self._log("Exception while parsing command progress", str(e), level='debug') # Stop the process if the worker is paused # Then resume it when the worker is resumed if self.paused_flag.is_set(): self._log("Pausing PID {}".format(sub_proc.pid), level='debug') proc.suspend() self.paused = True start_pause = time.time() while not self.redundant_flag.is_set(): time.sleep(1) if not self.paused_flag.is_set(): self._log("Resuming PID {}".format(sub_proc.pid), level='debug') proc.resume() self.paused = False # Elapsed time is used for calculating etc. # We account for this by counting the time we are paused also. # This is then subtracted from the elapsed time in the calculation above. proc_pause_time = int(proc_pause_time + time.time() - start_pause) break continue # Get the final output and the exit status if not self.redundant_flag.is_set(): communicate = sub_proc.communicate()[0] # If the process is still running, kill it if proc.is_running(): self._log("Found worker subprocess is still running. Killing it.", level='warning') self.__terminate_proc_tree(proc) if sub_proc.returncode == 0: return True else: self._log("Command run against '{}' exited with non-zero status. " "Download command dump from history for more information.".format(data.get("file_in")), message2=str(exec_command), level="error") return False except Exception as e: self._log("Error while executing the command against file{}.".format(data.get("file_in")), message2=str(e), level="error") return False