def test_converter_alpha3t(self):
self.assertEqual(Language('fra').alpha3t, 'fra')
self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
self.assertRaises(LanguageReverseError, lambda: Language.fromalpha3t('zzz'))
self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha3t)
self.assertEqual(len(language_converters['alpha3t'].codes), 418)
def test_converter_alpha3t(self):
self.assertEqual(Language('fra').alpha3t, 'fra')
self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
with self.assertRaises(LanguageReverseError):
Language.fromalpha3t('zzz')
with self.assertRaises(LanguageConvertError):
Language('aaa').alpha3t
self.assertEqual(len(get_language_converter('alpha3t').codes), 418)
def getAlpha3TCode(code): # We need to make sure that language codes are alpha3T
"""
:param code: Alpha2, Alpha3, or Alpha3b code.
:type code: C{str}
:return: Alpha3t language code (ISO 639-2/T) as C{str}
"""
lang = 'und'
code = code.strip().lower()
if len(code) == 3:
try:
lang = Language(code).alpha3t
except:
try:
lang = Language.fromalpha3b(code).alpha3t
except:
try:
lang = Language.fromalpha3t(code).alpha3t
except:
pass
elif len(code) == 2:
lang = Language.fromalpha2(code).alpha3t
return lang
def checkLanguage(self, language):
if not language:
return None
if len(language) < 2:
self.log.error("Unable to set tag language, not enough characters [tag-language].")
return None
try:
from babelfish import Language
except:
self.log.exception("Unable to important Language from babelfish [tag-language].")
return None
if len(language) == 2:
try:
return Language.fromalpha2(language).alpha3
self.log.exception("Unable to set tag language [tag-language].")
except:
return None
try:
return Language(language).alpha3
except:
try:
return Language.fromalpha3b(language).alpha3
except:
try:
return Language.fromalpha3t(language).alpha3
except:
self.log.exception("Unable to set tag language [tag-language].")
return None
def getAlpha3TCode(code): # We need to make sure that language codes are alpha3T
"""
:param code: Alpha2, Alpha3, or Alpha3b code.
:type code: C{str}
:return: Alpha3t language code (ISO 639-2/T) as C{str}
"""
lang = 'und'
code = code.strip().lower()
if len(code) == 3:
try:
lang = Language(code).alpha3t
except:
try:
lang = Language.fromalpha3b(code).alpha3t
except:
try:
lang = Language.fromalpha3t(code).alpha3t
except:
pass
elif len(code) == 2:
lang = Language.fromalpha2(code).alpha3t
return lang
def getAlpha3TCode(code, default=None):
lang = default or UNDEFINED
if not code or code == UNDEFINED:
return lang
code = code.strip().lower().replace('.', '')
if len(code) == 3:
try:
lang = Language(code).alpha3t
except:
try:
lang = Language.fromalpha3b(code).alpha3t
except:
try:
lang = Language.fromalpha3t(code).alpha3t
except:
pass
elif len(code) == 2:
try:
lang = Language.fromalpha2(code).alpha3t
except:
pass
return lang
def generateOptions(self, inputfile, original=None):
# Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
# Video stream
self.log.info("Reading video stream.")
self.log.info("Video codec detected: %s." % info.video.codec)
try:
vbr = self.estimateVideoBitrate(info)
except:
vbr = info.format.bitrate / 1000
if info.video.codec.lower() in self.video_codec:
vcodec = 'copy'
else:
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate if self.video_bitrate else vbr
self.log.info("Pix Fmt: %s." % info.video.pix_fmt)
if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt:
self.log.debug("Overriding video pix_fmt. Codec cannot be copied because pix_fmt is not approved.")
vcodec = self.video_codec[0]
pix_fmt = self.pix_fmt[0]
if self.video_profile:
vprofile = self.video_profile[0]
else:
pix_fmt = None
if self.video_bitrate is not None and vbr > self.video_bitrate:
self.log.debug("Overriding video bitrate. Codec cannot be copied because video bitrate is too high.")
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate
if self.video_width is not None and self.video_width < info.video.video_width:
self.log.debug("Video width is over the max width, it will be downsampled. Video stream can no longer be copied.")
vcodec = self.video_codec[0]
vwidth = self.video_width
else:
vwidth = None
if '264' in info.video.codec.lower() and self.h264_level and info.video.video_level and (info.video.video_level / 10 > self.h264_level):
self.log.info("Video level %0.1f." % (info.video.video_level / 10))
vcodec = self.video_codec[0]
self.log.debug("Video codec: %s." % vcodec)
self.log.debug("Video bitrate: %s." % vbitrate)
self.log.info("Profile: %s." % info.video.profile)
if self.video_profile and info.video.profile.lower().replace(" ", "") not in self.video_profile:
self.log.debug("Video profile is not supported. Video stream can no longer be copied.")
vcodec = self.video_codec[0]
vprofile = self.video_profile[0]
if self.pix_fmt:
pix_fmt = self.pix_fmt[0]
else:
vprofile = None
# Audio streams
self.log.info("Reading audio streams.")
overrideLang = True
for a in info.audio:
try:
a.metadata['language'] = languagecode.validateLangCode(a.metadata['language'])
except KeyError:
a.metadata['language'] = 'und'
if (a.metadata['language'] == 'und' and self.adl != 'und') or (
self.awl != 'und' and a.metadata['language'] in self.awl):
overrideLang = False
break
if overrideLang:
self.awl = 'und'
self.log.info("No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present.")
audio_settings = {}
l = 0
for a in info.audio:
try:
a.metadata['language'] = languagecode.validateLangCode(a.metadata['language'])
except KeyError:
a.metadata['language'] = 'und'
self.log.info("Audio detected for stream #%s: %s [%s]." % (a.index, a.codec, a.metadata['language']))
if a.codec.lower() == 'truehd': # Need to skip it early so that it flags the next track as default.
self.log.info( "MP4 containers do not support truehd audio, and converting it is inconsistent due to video/audio sync issues. Skipping stream %s as typically the 2nd audio track is the AC3 core of the truehd stream." % a.index )
continue
# Set undefined language to default language if specified
if self.adl != 'und' and a.metadata['language'] == 'und':
self.log.debug("Undefined language detected, defaulting to [%s]." % self.adl)
a.metadata['language'] = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
iosdata = None
if self.awl == 'und' or a.metadata['language'] in self.awl:
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS and a.audio_channels > 2:
iOSbitrate = 256 if (self.audio_bitrate * 2) > 256 else (self.audio_bitrate * 2)
self.log.info("Creating audio stream %s from source audio stream %s [iOS-audio]." % (str(l), a.index))
self.log.debug("Audio codec: %s." % self.iOS[0])
self.log.debug("Channels: 2.")
self.log.debug("Filter: %s." % self.iOS_filter)
self.log.debug("Bitrate: %s." % iOSbitrate)
self.log.debug("Language: %s." % a.metadata['language'])
if l == 0:
disposition = 'default'
self.log.info("Audio track is number %s setting disposition to %s" % (str(l), disposition))
else:
disposition = 'none'
self.log.info("Audio track is number %s setting disposition to %s" % (str(l), disposition))
iosdata = {
'map': a.index,
'codec': self.iOS[0],
'channels': 2,
'bitrate': iOSbitrate,
'filter': self.iOS_filter,
'language': a.metadata['language'],
'disposition': disposition,
}
if not self.iOSLast:
audio_settings.update({l: iosdata})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
self.log.info("Creating audio stream %s from source stream %s." % (str(l), a.index))
if self.iOS and a.audio_channels <= 2:
self.log.debug("Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio].")
acodec = 'copy' if a.codec in self.iOS else self.iOS[0]
audio_channels = a.audio_channels
afilter = self.iOS_filter
abitrate = a.audio_channels * 128 if (a.audio_channels * self.audio_bitrate) > (a.audio_channels * 128) else (a.audio_channels * self.audio_bitrate)
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = 'copy' if a.codec.lower() in self.audio_codec else self.audio_codec[0]
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == 'copy':
acodec = self.audio_codec[0]
abitrate = self.maxchannels * self.audio_bitrate
else:
audio_channels = a.audio_channels
abitrate = a.audio_channels * self.audio_bitrate
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug("Attempting to set bitrate based on source stream bitrate.")
try:
abitrate = a.bitrate / 1000
except:
self.log.warning("Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel." % a.index)
abitrate = a.audio_channels * 256
afilter = self.audio_filter
self.log.debug("Audio codec: %s." % acodec)
self.log.debug("Channels: %s." % audio_channels)
self.log.debug("Bitrate: %s." % abitrate)
self.log.debug("Language: %s" % a.metadata['language'])
self.log.debug("Filter: %s" % afilter)
# If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed
if self.iOS and self.iOSFirst:
self.log.debug("Not creating any additional iOS audio streams.")
self.iOS = False
# Set first track as default disposition
if l == 0:
disposition = 'default'
self.log.info("Audio Track is number %s setting disposition to %s" % (a.index, disposition))
else:
disposition = 'none'
self.log.info("Audio Track is number %s setting disposition to %s" % (a.index, disposition))
audio_settings.update({l: {
'map': a.index,
'codec': acodec,
'channels': audio_channels,
'bitrate': abitrate,
'filter': afilter,
'language': a.metadata['language'],
'disposition': disposition,
}})
if acodec == 'copy' and a.codec == 'aac' and self.aac_adtstoasc:
audio_settings[l]['bsf'] = 'aac_adtstoasc'
l += 1
# Add the iOS track last instead
if self.iOSLast and iosdata:
iosdata['disposition'] = 'none'
audio_settings.update({l: iosdata})
l += 1
if self.audio_copyoriginal and acodec != 'copy':
self.log.info("Adding copy of original audio track in format %s" % a.codec)
audio_settings.update({l: {
'map': a.index,
'codec': 'copy',
'language': a.metadata['language'],
'disposition': 'none',
}})
# Subtitle streams
subtitle_settings = {}
l = 0
self.log.info("Reading subtitle streams.")
for s in info.subtitle:
try:
s.metadata['language'] = languagecode.validateLangCode(s.metadata['language'])
except KeyError:
s.metadata['language'] = 'und'
self.log.info("Subtitle detected for stream #%s: %s [%s]." % (s.index, s.codec, s.metadata['language']))
# Set undefined language to default language if specified
if self.sdl != 'und' and s.metadata['language'] == 'und':
self.log.debug("Undefined language detected, defaulting to [%s]." % self.sdl)
s.metadata['language'] = self.sdl
# Make sure its not an image based codec
if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl == 'und' or s.metadata['language'] in self.swl:
subtitle_settings.update({l: {
'map': s.index,
'codec': self.scodec[0],
'language': s.metadata['language'],
'encoding': self.subencoding,
# 'forced': s.sub_forced,
# 'default': s.sub_default
}})
self.log.info("Creating subtitle stream %s from source stream %s." % (l, s.index))
l = l + 1
elif s.codec.lower() not in bad_subtitle_codecs and not self.embedsubs:
if self.swl == 'und' or s.metadata['language'] in self.swl:
for codec in self.scodec:
ripsub = {0: {
'map': s.index,
'codec': codec,
'language': s.metadata['language']
}}
options = {
'format': codec,
'subtitle': ripsub,
}
try:
extension = subtitle_codec_extensions[codec]
except:
self.log.info("Wasn't able to determine subtitle file extension, defaulting to '.srt'.")
extension = 'srt'
forced = ".forced" if s.sub_forced else ""
input_dir, filename, input_extension = self.parseFile(inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + forced + "." + extension)
i = 2
while os.path.isfile(outputfile):
self.log.debug("%s exists, appending %s to filename." % (outputfile, i))
outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + forced + "." + str(i) + "." + extension)
i += 1
try:
self.log.info("Ripping %s subtitle from source stream %s into external file." % (s.metadata['language'], s.index))
conv = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).convert(inputfile, outputfile, options, timeout=None)
for timecode in conv:
pass
self.log.info("%s created." % outputfile)
except:
self.log.exception("Unabled to create external subtitle file for stream %s." % (s.index))
# Attempt to download subtitles if they are missing using subliminal
languages = set()
try:
if self.swl != 'und':
for alpha3t in self.swl:
languages.add(Language.fromalpha3t(alpha3t))
elif self.sdl != 'und':
languages.add(Language.fromalpha3t(self.sdl))
else:
self.downloadsubs = False
self.log.error("No valid subtitle language specified, cannot download subtitles.")
except:
self.log.exception("Unable to verify subtitle languages for download.")
self.downloadsubs = False
if self.downloadsubs:
import subliminal
self.log.info("Attempting to download subtitles.")
# Attempt to set the dogpile cache
try:
subliminal.region.configure('dogpile.cache.memory')
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile), subtitles=True, embedded_subtitles=True)
subtitles = subliminal.download_best_subtitles([video], languages, hearing_impaired=False, providers=self.subproviders)
try:
subliminal.save_subtitles(video, subtitles[video])
except:
# Support for older versions of subliminal
subliminal.save_subtitles(subtitles)
self.log.info("Please update to the latest version of subliminal.")
except Exception as e:
self.log.info("Unable to download subtitles.", exc_info=True)
self.log.debug("Unable to download subtitles.", exc_info=True)
# External subtitle import
if self.embedsubs and not self.embedonlyinternalsubs: # Don't bother if we're not embeddeding subtitles and external subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
# Using babelfish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3t
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
self.log.info("External %s subtitle file detected." % lang)
if self.swl == 'und' or lang in self.swl:
self.log.info("Creating subtitle stream %s by importing %s." % (l, fname))
subtitle_settings.update({l: {
'path': os.path.join(dirName, fname),
'source': src,
'map': 0,
'codec': 'mov_text',
'language': lang}})
self.log.debug("Path: %s." % os.path.join(dirName, fname))
self.log.debug("Source: %s." % src)
self.log.debug("Codec: mov_text.")
self.log.debug("Language: %s." % lang)
l = l + 1
src = src + 1
self.deletesubs.add(os.path.join(dirName, fname))
else:
self.log.info("Ignoring %s external subtitle stream due to language %s." % (fname, lang))
# Collect all options
options = {
'format': self.output_format,
'video': {
'codec': vcodec,
'map': info.video.index,
'bitrate': vbitrate,
'level': self.h264_level,
'profile': vprofile,
'pix_fmt': pix_fmt
},
'audio': audio_settings,
'subtitle': subtitle_settings,
'preopts': [],
'postopts': ['-threads', self.threads]
}
# If a CRF option is set, override the determine bitrate
if self.vcrf:
del options['video']['bitrate']
options['video']['crf'] = self.vcrf
if len(options['subtitle']) > 0:
options['preopts'].append('-fix_sub_duration')
if self.preopts:
options['preopts'].extend(self.preopts)
if self.postopts:
options['postopts'].extend(self.postopts)
if self.dxva2_decoder: # DXVA2 will fallback to CPU decoding when it hits a file that it cannot handle, so we don't need to check if the file is supported.
options['preopts'].extend(['-hwaccel', 'dxva2'])
elif info.video.codec.lower() == "hevc" and self.hevc_qsv_decoder:
options['preopts'].extend(['-vcodec', 'hevc_qsv'])
elif vcodec == "h264qsv" and info.video.codec.lower() == "h264" and self.qsv_decoder and (info.video.video_level / 10) < 5:
options['preopts'].extend(['-vcodec', 'h264_qsv'])
# Add width option
if vwidth:
options['video']['width'] = vwidth
# HEVC Tagging for copied streams
if info.video.codec.lower() in ['x265', 'h265', 'hevc'] and vcodec == 'copy':
options['postopts'].extend(['-tag:v', 'hvc1'])
self.log.info("Tagging copied video stream as hvc1")
self.options = options
return options
