def convert_file(self, data, process_id):
file_in = data.get("file_in")
file_out = data.get("file_out")
ffmpeg_args = data.get("ffmpeg_args")
# Create output path if not exists
common.ensure_dir(file_out)
# Convert file
success = False
try:
# Reset to defaults
self.ffmpeg.set_info_defaults()
# Fetch source file info
self.ffmpeg.set_file_in(file_in)
# Read video information for the input file
file_probe = self.ffmpeg.file_in['file_probe']
if not file_probe:
return False
if ffmpeg_args:
success = self.ffmpeg.convert_file_and_fetch_progress(file_in, ffmpeg_args)
self.current_task.save_ffmpeg_log(self.ffmpeg.ffmpeg_cmd_stdout)
except ffmpeg.FFMPEGHandleConversionError as e:
# Fetch ffmpeg stdout and append it to the current task object (to be saved during post process)
self.current_task.save_ffmpeg_log(self.ffmpeg.ffmpeg_cmd_stdout)
self._log("Error while executing the FFMPEG command {}. "
"Download FFMPEG command dump from history for more information.".format(file_in),
message2=str(e), level="error")
return success
def process_item(self):
abspath = self.current_task.get_source_abspath()
self._log("{} processing job - {}".format(self.name, abspath))
# Create output path if not exists
common.ensure_dir(self.current_task.cache_path)
# Convert file
success = False
try:
ffmpeg_args = self.current_task.ffmpeg.generate_ffmpeg_args()
if ffmpeg_args:
success = self.current_task.ffmpeg.convert_file_and_fetch_progress(
abspath, self.current_task.cache_path, ffmpeg_args)
self.current_task.ffmpeg_log = self.current_task.ffmpeg.ffmpeg_cmd_stdout
except ffmpeg.FFMPEGHandleConversionError as e:
# Fetch ffmpeg stdout and append it to the current task object (to be saved during post process)
self.current_task.ffmpeg_log = self.current_task.ffmpeg.ffmpeg_cmd_stdout
self._log(
"Error while executing the FFMPEG command {}. "
"Download FFMPEG command dump from history for more information."
.format(abspath),
message2=str(e),
level="error")
if success:
# If file conversion was successful, we will get here
self._log("Successfully converted file '{}'".format(abspath))
return True
self._log("Failed to convert file '{}'".format(abspath),
level='warning')
return False
def process_file_with_configured_settings(self, vid_file_path):
# Parse input path
src_file = os.path.basename(vid_file_path)
src_path = os.path.abspath(vid_file_path)
src_folder = os.path.dirname(src_path)
# Get container extension
container = unffmpeg.containers.grab_module(self.settings.OUT_CONTAINER)
container_extension = container.container_extension()
# Parse an output cache path
out_folder = "unmanic_file_conversion-{}".format(time.time())
out_file = "{}-{}.{}".format(os.path.splitext(src_file)[0], time.time(), container_extension)
out_path = os.path.join(self.settings.CACHE_PATH, out_folder, out_file)
# Create output path if not exists
common.ensure_dir(out_path)
# Reset all info
self.set_info_defaults()
# Fetch file info
self.set_file_in(vid_file_path)
# Convert file
success = False
ffmpeg_args = self.generate_ffmpeg_args()
if ffmpeg_args:
success = self.convert_file_and_fetch_progress(src_path, out_path, ffmpeg_args)
if success:
# Move file back to original folder and remove source
success = self.post_process_file(out_path)
if success:
destPath = os.path.join(src_folder, out_file)
self._log("Moving file {} --> {}".format(out_path, destPath))
shutil.move(out_path, destPath)
try:
self.post_process_file(destPath)
except FFMPEGHandlePostProcessError:
success = False
if success:
# If successful move, remove source
# TODO: Add env variable option to keep src
if src_path != destPath:
self._log("Removing source: {}".format(src_path))
os.remove(src_path)
else:
self._log("Copy / Replace failed during post processing '{}'".format(out_path), level='warning')
return False
else:
self._log("Encoded file failed post processing test '{}'".format(out_path), level='warning')
return False
else:
self._log("Failed processing file '{}'".format(src_path), level='warning')
return False
# If file conversion was successful, we will get here
self._log("Successfully processed file '{}'".format(src_path))
return True
def test_convert_all_files_for_success(self):
"""
Ensure all small test files are able to be converted.
:return:
"""
# Test all small files of various containers
for video_file in os.listdir(os.path.join(self.tests_videos_dir, 'small')):
filename, file_extension = os.path.splitext(os.path.basename(video_file))
infile = os.path.join(self.tests_videos_dir, 'small', video_file)
outfile = os.path.join(self.tests_tmp_dir, filename + '.mkv')
common.ensure_dir(outfile)
self.convert_single_file(infile, outfile)
def test_read_file_info_for_failure(self):
"""
Ensure that and exception is thrown if ffprobe is unable to read a file.
:return:
"""
# Set project root path
tmp_dir = self.tests_tmp_dir
fail_file = os.path.join(tmp_dir, 'test_failure.mkv')
# Test
common.ensure_dir(fail_file)
common.touch(fail_file)
with pytest.raises(unffmpeg.exceptions.ffprobe.FFProbeError):
self.ffmpeg.file_probe(fail_file)
def test_convert_all_faulty_files_for_success(self):
"""
Ensure all faulty test files are able to be converted.
These files have various issues with them that may cause ffmpeg to fail if not configured correctly.
:return:
"""
# Test all faulty files can be successfully converted (these files have assorted issues)
for video_file in os.listdir(os.path.join(self.tests_videos_dir, 'faulty')):
filename, file_extension = os.path.splitext(os.path.basename(video_file))
infile = os.path.join(self.tests_videos_dir, 'faulty', video_file)
outfile = os.path.join(self.tests_tmp_dir, filename + '.mkv')
common.ensure_dir(outfile)
self.convert_single_file(infile, outfile)
def test_process_file_for_success(self):
"""
This will test the FFMPEGHandle for processing a file automatically using configured settings.
:return:
"""
# Set project root path
tmp_dir = self.tests_tmp_dir
# Test just the first file found in the med folder
for video_file in os.listdir(
os.path.join(self.tests_videos_dir, 'small')):
filename, file_extension = os.path.splitext(
os.path.basename(video_file))
infile = os.path.join(self.tests_videos_dir, 'small', video_file)
# Copy the file to a tmp location (it will be replaced)
testfile = os.path.join(tmp_dir, filename + file_extension)
self._log(infile, testfile)
common.ensure_dir(testfile)
shutil.copy(infile, testfile)
assert self.ffmpeg.process_file_with_configured_settings(testfile)
break
def convert_single_file(self, infile, outfile, test_for_failure=False):
if not os.path.exists(infile):
self._log("No such file: {}".format(infile))
sys.exit(1)
# Ensure the directory exists
common.ensure_dir(outfile)
# Remove the output file if it already exists
if os.path.exists(outfile):
os.remove(outfile)
# Setup ffmpeg args
built_args = self.build_ffmpeg_args(infile, outfile, test_for_failure)
# Run conversion process
self._log("Converting {} -> {}".format(infile, outfile))
# Fetch file info
self.ffmpeg.set_file_in(infile)
assert self.ffmpeg.convert_file_and_fetch_progress(infile, built_args)
if not test_for_failure:
assert self.ffmpeg.post_process_file(outfile)
elif test_for_failure:
with pytest.raises(ffmpeg.FFMPEGHandlePostProcessError):
self.ffmpeg.post_process_file(outfile)
def __exec_command_subprocess(self, data):
"""
Executes a command as a shell subprocess.
Uses the given parser to record progress data from the shell STDOUT.
:param data:
:return:
"""
# Fetch command to execute.
exec_command = data.get("exec_command", [])
# Fetch the command progress parser function
command_progress_parser = data.get("command_progress_parser", default_progress_parser)
# Log the command for debugging
command_string = exec_command
if isinstance(exec_command, list):
command_string = ' '.join(exec_command)
self._log("Executing: {}".format(command_string), level='debug')
# Append start of command to worker subprocess stdout
self.worker_log += [
'\n\n',
'COMMAND:\n',
command_string,
'\n\n',
'LOG:\n',
]
# Create output path if not exists
common.ensure_dir(data.get("file_out"))
# Convert file
try:
proc_pause_time = 0
proc_start_time = time.time()
# Execute command
if isinstance(exec_command, list):
sub_proc = subprocess.Popen(exec_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True, errors='replace')
elif isinstance(exec_command, str):
sub_proc = subprocess.Popen(exec_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True, errors='replace', shell=True)
else:
raise Exception(
"Plugin's returned 'exec_command' object must be either a list or a string. Received type {}.".format(
type(exec_command)))
# Fetch process using psutil for control (sending SIGSTOP on windows will not work)
proc = psutil.Process(pid=sub_proc.pid)
# Set process priority on posix systems
# TODO: Test how this will work on Windows
if os.name == "posix":
try:
parent_proc = psutil.Process(os.getpid())
parent_proc_nice = parent_proc.nice()
proc.nice(parent_proc_nice + 1)
except Exception as e:
self._log("Unable to lower priority of subprocess. Subprocess should continue to run at normal priority",
str(e), level='warning')
# Record PID and PROC
self.worker_subprocess = sub_proc
self.worker_subprocess_pid = sub_proc.pid
# Poll process for new output until finished
while not self.redundant_flag.is_set():
line_text = sub_proc.stdout.readline()
# Fetch command stdout and append it to the current task object (to be saved during post process)
self.worker_log.append(line_text)
# Check if the command has completed. If it has, exit the loop
if line_text == '' and sub_proc.poll() is not None:
self._log("Subprocess task completed!", level='debug')
break
# Parse the progress
try:
progress_dict = command_progress_parser(line_text)
self.worker_subprocess_percent = progress_dict.get('percent', '0')
self.worker_subprocess_elapsed = str(time.time() - proc_start_time - proc_pause_time)
except Exception as e:
# Only need to show any sort of exception if we have debugging enabled.
# So we should log it as a debug rather than an exception.
self._log("Exception while parsing command progress", str(e), level='debug')
# Stop the process if the worker is paused
# Then resume it when the worker is resumed
if self.paused_flag.is_set():
self._log("Pausing PID {}".format(sub_proc.pid), level='debug')
proc.suspend()
self.paused = True
start_pause = time.time()
while not self.redundant_flag.is_set():
time.sleep(1)
if not self.paused_flag.is_set():
self._log("Resuming PID {}".format(sub_proc.pid), level='debug')
proc.resume()
self.paused = False
# Elapsed time is used for calculating etc.
# We account for this by counting the time we are paused also.
# This is then subtracted from the elapsed time in the calculation above.
proc_pause_time = int(proc_pause_time + time.time() - start_pause)
break
continue
# Get the final output and the exit status
if not self.redundant_flag.is_set():
communicate = sub_proc.communicate()[0]
# If the process is still running, kill it
if proc.is_running():
self._log("Found worker subprocess is still running. Killing it.", level='warning')
self.__terminate_proc_tree(proc)
if sub_proc.returncode == 0:
return True
else:
self._log("Command run against '{}' exited with non-zero status. "
"Download command dump from history for more information.".format(data.get("file_in")),
message2=str(exec_command), level="error")
return False
except Exception as e:
self._log("Error while executing the command against file{}.".format(data.get("file_in")), message2=str(e),
level="error")
return False
