def upload(job: Job, destinations: List[str], upload_file: str,
rclone_config: str, flags: str) -> bool:
"""
Upload the completed new hardsub file into the rclone destinations
Returns a boolean based on success
job: Job to do! This is the job of the HARDSUB file
destinations: list of rlcone destinations (e.g., EncoderConf.uploading_destinations)
upload_file: Path to the file to be uploaded
rclone_config: Path to the rclone config file
flags: rclone flag
This method will upload the file and include its show name:
e.g., 'temp.mp4' --> destination/show/episode.mp4
"""
for dest in destinations:
LoggingUtils.debug("Uploading to {}".format(dest))
rclone_dest = PathUtils.clean_directory_path(
dest) + PathUtils.clean_directory_path(job.show) + job.episode
command = [BinUtils.rclone]
LoggingUtils.debug("Using temporary rclone file at " +
rclone_config,
color=LoggingUtils.YELLOW)
command.extend(["--config={}".format(rclone_config)])
command.extend(["copyto", upload_file, rclone_dest])
command.extend(flags.split())
Rclone._run(
command,
RcloneUploadError(
"An error occured when rclone was uploading a file", "",
job.episode, dest))
def add_font(job: Job, src_file: str, tempfolder: str) -> bool:
"""Adds the OpenSans-Semibold.ttf font file"""
info = Haikan.scan(src_file)
binary = FFmpeg._binary(info)
dest_temp_file = PathUtils.clean_directory_path(
tempfolder) + "temp2.mkv"
LoggingUtils.debug(
"Creating new fonted file at {}".format(dest_temp_file))
command = [binary, "-i", src_file
] # The base command, this is akin to "ffmpeg -i temp.mkv"
command.extend([
"-attach", AssetUtils.opensans_semibold
]) # The attach command, "-attach OpenSans-Semibold.ttf"
command.extend([
"-metadata:s:{}".format(info.streams),
"mimetype=application/x-truetype-font"
]) # Add metadata
command.extend(
["-c:a", "copy", "-c:v", "copy", "-c:s", "copy", dest_temp_file])
FFmpeg._run(
command,
"Error occured while ffmpeg was attaching font to the episode")
# Replace the original file with this
LoggingUtils.debug(
"Successfully added font to file, now replacing original...",
color=LoggingUtils.GREEN)
shutil.move(dest_temp_file, src_file)
LoggingUtils.debug(
"Successfully replaced original file with fonted episode",
color=LoggingUtils.GREEN)
def opensans_semibold(cls) -> str:
if DockerUtils.docker:
return PathUtils.clean_directory_path(
DockerUtils.path) + cls._OPENSANS_PATH
else:
return PathUtils.abspath(cls._OPENSANS_PATH)
def _download_episode(job: Job, source: str, tempfolder: str,
rclone_config: str, flags: str) -> str:
"""Helper to download file from remote to local temp"""
# Note 1: The file will always ben downloaded as "temp.mkv"
rclone_src_file = source + PathUtils.clean_directory_path(
job.show) + job.episode
rclone_dest_file = tempfolder + "temp"
LoggingUtils.debug("Sourcing file from \"{}\"".format(rclone_src_file))
LoggingUtils.debug(
"Downloading to temp file at \"{}\"".format(rclone_dest_file),
color=LoggingUtils.YELLOW)
LoggingUtils.debug("Beginning download...", color=LoggingUtils.YELLOW)
command = [BinUtils.rclone]
LoggingUtils.debug("Using temporary rclone file at " + rclone_config,
color=LoggingUtils.YELLOW)
command.extend(["--config={}".format(rclone_config)])
command.extend(["copyto", rclone_src_file, rclone_dest_file])
command.extend(flags.split())
Rclone._run(
command,
RcloneDownloadError(
"Unable to copy episode file to local folder using rclone", "",
job.show, job.episode))
LoggingUtils.debug("Download complete.", color=LoggingUtils.GREEN)
return rclone_dest_file
def download(job: Job, sources: List[str], tempfolder: str,
rclone_config: str, flags: str) -> str:
"""
Download the provided episode from sources
Returns the path of the downloaded file
job: Job to do!
sources: list of rclone sources (EncoderConf.downloading_sources)
tmppath: Path of the temporary folder
rclone_config: Path to the rclone config file
flags: rclone flags
"""
# Step 1: Download the episode if possible
dl_source = None
for source in sources:
if Rclone._check_episode_exists(job, rclone_config, source):
LoggingUtils.debug(
"Found episode in source {}, downloading...".format(
source),
color=LoggingUtils.GREEN)
dl_source = source
break
# If the file was not found in any sources, return False for failure
if not dl_source:
raise RcloneDownloadNotFoundError(
"No sources contained the episode, cancelling operation", "",
job.show, job.episode)
# Download the episode
tempfolder = PathUtils.clean_directory_path(tempfolder)
episode_src_file = Rclone._download_episode(job, dl_source, tempfolder,
rclone_config, flags)
return episode_src_file
def prepare(job: Job, src_file: str,
tempfolder: str) -> Tuple[bool, str, str]:
"""
Takes the temporary file downloaded and copies over streams that we need.
Does NOT do any encoding or format conversion, just copies to set metadata and remove attachments.
Important: This will replace the original temp file
job: The job to handle
src_file: The downloaded source file (temp.mkv in a tempfolder)
tempfolder: The tempfolder path itself
Returns the success of the operation
"""
info = Haikan.scan(src_file) # Get the information of the source file
binary = FFmpeg._binary(
info) # This is the ffmpeg/ffmpeg-10bit executable path
# Extract the main sub file if it exists - simplifies future encoding
sub1_file = PathUtils.clean_directory_path(tempfolder) + "sub1.ass"
LoggingUtils.debug(
"Extracting main subtitle file to {}".format(sub1_file))
command = [binary, "-i", src_file] # ffmpeg -i temp.mkv
command.extend(
["-map", "0:{}".format(info.subtitle_main_index),
sub1_file]) # -map 0:2 sub1.ass
FFmpeg._run(command,
"Error occured while extracting the main subtitle file")
LoggingUtils.debug("Successfully extracted main subtitle file",
color=LoggingUtils.GREEN)
# Extract the secondary sub file if it exists - simplifies future encoding
sub2_file = None
if info.subtitle_extra_index != -1:
sub2_file = PathUtils.clean_directory_path(tempfolder) + "sub2.ass"
LoggingUtils.debug(
"Extracting secondary subtitle file to {}".format(sub2_file))
command = [binary, "-i", src_file] # ffmpeg -i temp.mkv
command.extend(
["-map", "0:{}".format(info.subtitle_extra_index), sub2_file])
FFmpeg._run(
command,
"Error occured while extracting secondary subtitle file, ignoring"
)
LoggingUtils.debug("Sucessfully extracted secondary subtitle file",
color=LoggingUtils.GREEN)
dest_temp_file = PathUtils.clean_directory_path(
tempfolder
) + "temp2.mkv" # Where the new file will be temporarily stored
LoggingUtils.debug(
"Creating new prepared file at {}".format(dest_temp_file))
command = [binary, "-i", src_file
] # The base command, this is akin to "ffmpeg -i temp.mkv"
command.extend(["-map", "0:{}".format(info.video_stream_index)
]) # Add the video map stream
command.extend(["-map", "0:{}".format(info.audio_stream_index)
]) # Add the audio map stream
command.extend(["-c:a", "copy", "-c:v", "copy", dest_temp_file])
# Create the new file
FFmpeg._run(command,
"Error occured while ffmpeg was preparing the episode")
# Replace the original file with this
LoggingUtils.debug(
"Successfully prepared the episode, now replacing original...",
color=LoggingUtils.GREEN)
shutil.move(dest_temp_file, src_file)
LoggingUtils.debug(
"Successfully replaced original file with prepared episode",
color=LoggingUtils.GREEN)
# We have to run another clear copy - this removes any corrupted tracks, usually subtitles
LoggingUtils.debug("Running final clean pass on file...",
color=LoggingUtils.YELLOW)
command = [binary, "-i", src_file, "-c", "copy", dest_temp_file]
FFmpeg._run(
command,
"Error occured while ffmpeg was running a final clean pass on the episode"
)
shutil.move(dest_temp_file, src_file)
LoggingUtils.debug("Successfully ran clean pass on file...",
color=LoggingUtils.GREEN)
return sub1_file, sub2_file
def hardsub(job: Job, src_file: str, tempfolder: str, sub1_file: str,
sub2_file: str) -> Tuple[bool, str]:
"""Hardsubs the video"""
info = Haikan.scan(src_file)
binary = FFmpeg._binary(info)
# Encode the new hardsub with the main track into a temp.mp4 file
hardsub_file = PathUtils.clean_directory_path(tempfolder) + "temp.mp4"
LoggingUtils.debug(
"Creating new hardsub file at {}".format(hardsub_file))
command = [binary, "-i",
src_file] # The base command, akin fo "ffmpeg -i temp.mkv"
# If we're in Docker, the ONLY font that will be available should be the Encoder Assets folder
if DockerUtils.docker:
LoggingUtils.debug(
"Detected Docker mode, referencing assets folder for fonts",
color=LoggingUtils.YELLOW)
command.extend([
"-vf",
"subtitles={sub}:force_style='FontName=Open Sans Semibold:fontsdir={ftd}'"
.format(sub=sub1_file,
ftd=PathUtils.clean_directory_path(DockerUtils.path) +
"src/encoder/assets/")
])
# Otherwise, we have to reference it from PathUTils
else:
LoggingUtils.debug(
"Detected interactive mode, referencing assets folder for fonts",
color=LoggingUtils.YELLOW)
command.extend([
"-vf",
"subtitles={sub}:force_style='FontName=Open Sans Semibold:fontsdir={ftd}'"
.format(sub=sub1_file,
ftd=PathUtils.clean_directory_path(
PathUtils.abspath_root()) + "src/encoder/assets/")
])
#command.extend(["-ss", "00:03:00", "-t", "00:00:15"]) # Dev mode stuff
# Sometimes, the audio provided is not HTML5 friendly. If the audio track is not FLAC/MP3/AAC, re-encode it:
audio_codec = info.audio_codec_name.lower()
if not (audio_codec == "aac" or audio_codec == "mp3"
or audio_codec == "flac"):
LoggingUtils.debug(
"Detected audio codec is not HTML5 supported, changing to AAC."
)
command.extend(["-acodec", "aac", "-ab", "320k"])
else:
LoggingUtils.debug(
"Detected audio codec is HTML5-supported codec {}, not changing"
.format(audio_codec))
command.extend(["-c:a", "copy"])
# Some extra flags we use for encoding
command.extend(["-strict", "-2", "-y", hardsub_file])
LoggingUtils.debug(
"Starting hardsub encode of file with main subtitle track...",
color=LoggingUtils.YELLOW)
FFmpeg._run(
command,
"Error occured while encoding the file with the main subtitle track"
)
LoggingUtils.debug(
"Successfully encoded hardsub of file with main subtitle track",
color=LoggingUtils.GREEN)
# If there's a secondary track, we want to encode that too
if sub2_file:
LoggingUtils.debug(
"Detected extra subtitle track, now hardsubbing again with extra track"
)
extra_file = PathUtils.clean_directory_path(
tempfolder) + "temp2.mp4"
LoggingUtils.debug(
"Creating secondary hardsub file at {}".format(extra_file))
command = [binary, "-i", hardsub_file] # ffmpeg -i temp.mp4
command.extend([
"-vf",
"subtitles={}:force_style='FontName=Open Sans Semibold'".
format(sub2_file)
])
command.extend(["-c:a", "copy", "-strict", "-2", "-y", extra_file])
LoggingUtils.debug(
"Starting hardsub encode of file with extra subtitle track...",
color=LoggingUtils.YELLOW)
FFmpeg._run(
command,
"Error occured while encoding the file with extra subtitle track"
)
LoggingUtils.debug(
"Successfully encoded hardsub of file with extra subtitle track",
color=LoggingUtils.GREEN)
shutil.move(extra_file, hardsub_file)
else:
LoggingUtils.debug(
"Didn't detect an extra subtitle track, skipping secondary hardsub run"
)
return hardsub_file
def conf(cls) -> str:
"""Returns the conf path in Docker"""
return PathUtils.clean_directory_path(cls._CONF)
def bin(cls) -> str:
"""Returns the bin path in Docker"""
return PathUtils.clean_directory_path(cls._BIN)
def path(cls) -> str:
"""Returns root path of project when in Docker"""
return PathUtils.clean_directory_path(cls._PATH)
def clean_uploading_destinations(dests: List[str]) -> List[str]:
destinations_copy = list()
for i in range(len(dests)):
destinations_copy.append(PathUtils.clean_directory_path(dests[i]))
return dests
def clean_downloading_sources(sources: List[str]) -> List[str]:
sources_copy = list()
for i in range(len(sources)):
sources_copy.append(PathUtils.clean_directory_path(sources[i]))
return sources_copy