def test_converter_alpha3t(self): self.assertEqual(Language('fra').alpha3t, 'fra') self.assertEqual(Language.fromalpha3t('fra'), Language('fra')) self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra')) self.assertRaises(LanguageReverseError, lambda: Language.fromalpha3t('zzz')) self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha3t) self.assertEqual(len(language_converters['alpha3t'].codes), 418)
def test_converter_alpha3t(self): self.assertEqual(Language('fra').alpha3t, 'fra') self.assertEqual(Language.fromalpha3t('fra'), Language('fra')) self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra')) with self.assertRaises(LanguageReverseError): Language.fromalpha3t('zzz') with self.assertRaises(LanguageConvertError): Language('aaa').alpha3t self.assertEqual(len(get_language_converter('alpha3t').codes), 418)
def getAlpha3TCode(code): # We need to make sure that language codes are alpha3T """ :param code: Alpha2, Alpha3, or Alpha3b code. :type code: C{str} :return: Alpha3t language code (ISO 639-2/T) as C{str} """ lang = 'und' code = code.strip().lower() if len(code) == 3: try: lang = Language(code).alpha3t except: try: lang = Language.fromalpha3b(code).alpha3t except: try: lang = Language.fromalpha3t(code).alpha3t except: pass elif len(code) == 2: lang = Language.fromalpha2(code).alpha3t return lang
def checkLanguage(self, language): if not language: return None if len(language) < 2: self.log.error("Unable to set tag language, not enough characters [tag-language].") return None try: from babelfish import Language except: self.log.exception("Unable to important Language from babelfish [tag-language].") return None if len(language) == 2: try: return Language.fromalpha2(language).alpha3 self.log.exception("Unable to set tag language [tag-language].") except: return None try: return Language(language).alpha3 except: try: return Language.fromalpha3b(language).alpha3 except: try: return Language.fromalpha3t(language).alpha3 except: self.log.exception("Unable to set tag language [tag-language].") return None
def getAlpha3TCode(code, default=None): lang = default or UNDEFINED if not code or code == UNDEFINED: return lang code = code.strip().lower().replace('.', '') if len(code) == 3: try: lang = Language(code).alpha3t except: try: lang = Language.fromalpha3b(code).alpha3t except: try: lang = Language.fromalpha3t(code).alpha3t except: pass elif len(code) == 2: try: lang = Language.fromalpha2(code).alpha3t except: pass return lang
def generateOptions(self, inputfile, original=None): # Get path information from the input file input_dir, filename, input_extension = self.parseFile(inputfile) info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile) # Video stream self.log.info("Reading video stream.") self.log.info("Video codec detected: %s." % info.video.codec) try: vbr = self.estimateVideoBitrate(info) except: vbr = info.format.bitrate / 1000 if info.video.codec.lower() in self.video_codec: vcodec = 'copy' else: vcodec = self.video_codec[0] vbitrate = self.video_bitrate if self.video_bitrate else vbr self.log.info("Pix Fmt: %s." % info.video.pix_fmt) if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt: self.log.debug("Overriding video pix_fmt. Codec cannot be copied because pix_fmt is not approved.") vcodec = self.video_codec[0] pix_fmt = self.pix_fmt[0] if self.video_profile: vprofile = self.video_profile[0] else: pix_fmt = None if self.video_bitrate is not None and vbr > self.video_bitrate: self.log.debug("Overriding video bitrate. Codec cannot be copied because video bitrate is too high.") vcodec = self.video_codec[0] vbitrate = self.video_bitrate if self.video_width is not None and self.video_width < info.video.video_width: self.log.debug("Video width is over the max width, it will be downsampled. Video stream can no longer be copied.") vcodec = self.video_codec[0] vwidth = self.video_width else: vwidth = None if '264' in info.video.codec.lower() and self.h264_level and info.video.video_level and (info.video.video_level / 10 > self.h264_level): self.log.info("Video level %0.1f." % (info.video.video_level / 10)) vcodec = self.video_codec[0] self.log.debug("Video codec: %s." % vcodec) self.log.debug("Video bitrate: %s." % vbitrate) self.log.info("Profile: %s." % info.video.profile) if self.video_profile and info.video.profile.lower().replace(" ", "") not in self.video_profile: self.log.debug("Video profile is not supported. Video stream can no longer be copied.") vcodec = self.video_codec[0] vprofile = self.video_profile[0] if self.pix_fmt: pix_fmt = self.pix_fmt[0] else: vprofile = None # Audio streams self.log.info("Reading audio streams.") overrideLang = True for a in info.audio: try: a.metadata['language'] = languagecode.validateLangCode(a.metadata['language']) except KeyError: a.metadata['language'] = 'und' if (a.metadata['language'] == 'und' and self.adl != 'und') or ( self.awl != 'und' and a.metadata['language'] in self.awl): overrideLang = False break if overrideLang: self.awl = 'und' self.log.info("No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present.") audio_settings = {} l = 0 for a in info.audio: try: a.metadata['language'] = languagecode.validateLangCode(a.metadata['language']) except KeyError: a.metadata['language'] = 'und' self.log.info("Audio detected for stream #%s: %s [%s]." % (a.index, a.codec, a.metadata['language'])) if a.codec.lower() == 'truehd': # Need to skip it early so that it flags the next track as default. self.log.info( "MP4 containers do not support truehd audio, and converting it is inconsistent due to video/audio sync issues. Skipping stream %s as typically the 2nd audio track is the AC3 core of the truehd stream." % a.index ) continue # Set undefined language to default language if specified if self.adl != 'und' and a.metadata['language'] == 'und': self.log.debug("Undefined language detected, defaulting to [%s]." % self.adl) a.metadata['language'] = self.adl # Proceed if no whitelist is set, or if the language is in the whitelist iosdata = None if self.awl == 'und' or a.metadata['language'] in self.awl: # Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo) if self.iOS and a.audio_channels > 2: iOSbitrate = 256 if (self.audio_bitrate * 2) > 256 else (self.audio_bitrate * 2) self.log.info("Creating audio stream %s from source audio stream %s [iOS-audio]." % (str(l), a.index)) self.log.debug("Audio codec: %s." % self.iOS[0]) self.log.debug("Channels: 2.") self.log.debug("Filter: %s." % self.iOS_filter) self.log.debug("Bitrate: %s." % iOSbitrate) self.log.debug("Language: %s." % a.metadata['language']) if l == 0: disposition = 'default' self.log.info("Audio track is number %s setting disposition to %s" % (str(l), disposition)) else: disposition = 'none' self.log.info("Audio track is number %s setting disposition to %s" % (str(l), disposition)) iosdata = { 'map': a.index, 'codec': self.iOS[0], 'channels': 2, 'bitrate': iOSbitrate, 'filter': self.iOS_filter, 'language': a.metadata['language'], 'disposition': disposition, } if not self.iOSLast: audio_settings.update({l: iosdata}) l += 1 # If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels self.log.info("Creating audio stream %s from source stream %s." % (str(l), a.index)) if self.iOS and a.audio_channels <= 2: self.log.debug("Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio].") acodec = 'copy' if a.codec in self.iOS else self.iOS[0] audio_channels = a.audio_channels afilter = self.iOS_filter abitrate = a.audio_channels * 128 if (a.audio_channels * self.audio_bitrate) > (a.audio_channels * 128) else (a.audio_channels * self.audio_bitrate) else: # If desired codec is the same as the source codec, copy to avoid quality loss acodec = 'copy' if a.codec.lower() in self.audio_codec else self.audio_codec[0] # Audio channel adjustments if self.maxchannels and a.audio_channels > self.maxchannels: audio_channels = self.maxchannels if acodec == 'copy': acodec = self.audio_codec[0] abitrate = self.maxchannels * self.audio_bitrate else: audio_channels = a.audio_channels abitrate = a.audio_channels * self.audio_bitrate # Bitrate calculations/overrides if self.audio_bitrate is 0: self.log.debug("Attempting to set bitrate based on source stream bitrate.") try: abitrate = a.bitrate / 1000 except: self.log.warning("Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel." % a.index) abitrate = a.audio_channels * 256 afilter = self.audio_filter self.log.debug("Audio codec: %s." % acodec) self.log.debug("Channels: %s." % audio_channels) self.log.debug("Bitrate: %s." % abitrate) self.log.debug("Language: %s" % a.metadata['language']) self.log.debug("Filter: %s" % afilter) # If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed if self.iOS and self.iOSFirst: self.log.debug("Not creating any additional iOS audio streams.") self.iOS = False # Set first track as default disposition if l == 0: disposition = 'default' self.log.info("Audio Track is number %s setting disposition to %s" % (a.index, disposition)) else: disposition = 'none' self.log.info("Audio Track is number %s setting disposition to %s" % (a.index, disposition)) audio_settings.update({l: { 'map': a.index, 'codec': acodec, 'channels': audio_channels, 'bitrate': abitrate, 'filter': afilter, 'language': a.metadata['language'], 'disposition': disposition, }}) if acodec == 'copy' and a.codec == 'aac' and self.aac_adtstoasc: audio_settings[l]['bsf'] = 'aac_adtstoasc' l += 1 # Add the iOS track last instead if self.iOSLast and iosdata: iosdata['disposition'] = 'none' audio_settings.update({l: iosdata}) l += 1 if self.audio_copyoriginal and acodec != 'copy': self.log.info("Adding copy of original audio track in format %s" % a.codec) audio_settings.update({l: { 'map': a.index, 'codec': 'copy', 'language': a.metadata['language'], 'disposition': 'none', }}) # Subtitle streams subtitle_settings = {} l = 0 self.log.info("Reading subtitle streams.") for s in info.subtitle: try: s.metadata['language'] = languagecode.validateLangCode(s.metadata['language']) except KeyError: s.metadata['language'] = 'und' self.log.info("Subtitle detected for stream #%s: %s [%s]." % (s.index, s.codec, s.metadata['language'])) # Set undefined language to default language if specified if self.sdl != 'und' and s.metadata['language'] == 'und': self.log.debug("Undefined language detected, defaulting to [%s]." % self.sdl) s.metadata['language'] = self.sdl # Make sure its not an image based codec if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs: # Proceed if no whitelist is set, or if the language is in the whitelist if self.swl == 'und' or s.metadata['language'] in self.swl: subtitle_settings.update({l: { 'map': s.index, 'codec': self.scodec[0], 'language': s.metadata['language'], 'encoding': self.subencoding, # 'forced': s.sub_forced, # 'default': s.sub_default }}) self.log.info("Creating subtitle stream %s from source stream %s." % (l, s.index)) l = l + 1 elif s.codec.lower() not in bad_subtitle_codecs and not self.embedsubs: if self.swl == 'und' or s.metadata['language'] in self.swl: for codec in self.scodec: ripsub = {0: { 'map': s.index, 'codec': codec, 'language': s.metadata['language'] }} options = { 'format': codec, 'subtitle': ripsub, } try: extension = subtitle_codec_extensions[codec] except: self.log.info("Wasn't able to determine subtitle file extension, defaulting to '.srt'.") extension = 'srt' forced = ".forced" if s.sub_forced else "" input_dir, filename, input_extension = self.parseFile(inputfile) output_dir = input_dir if self.output_dir is None else self.output_dir outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + forced + "." + extension) i = 2 while os.path.isfile(outputfile): self.log.debug("%s exists, appending %s to filename." % (outputfile, i)) outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + forced + "." + str(i) + "." + extension) i += 1 try: self.log.info("Ripping %s subtitle from source stream %s into external file." % (s.metadata['language'], s.index)) conv = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).convert(inputfile, outputfile, options, timeout=None) for timecode in conv: pass self.log.info("%s created." % outputfile) except: self.log.exception("Unabled to create external subtitle file for stream %s." % (s.index)) # Attempt to download subtitles if they are missing using subliminal languages = set() try: if self.swl != 'und': for alpha3t in self.swl: languages.add(Language.fromalpha3t(alpha3t)) elif self.sdl != 'und': languages.add(Language.fromalpha3t(self.sdl)) else: self.downloadsubs = False self.log.error("No valid subtitle language specified, cannot download subtitles.") except: self.log.exception("Unable to verify subtitle languages for download.") self.downloadsubs = False if self.downloadsubs: import subliminal self.log.info("Attempting to download subtitles.") # Attempt to set the dogpile cache try: subliminal.region.configure('dogpile.cache.memory') except: pass try: video = subliminal.scan_video(os.path.abspath(inputfile), subtitles=True, embedded_subtitles=True) subtitles = subliminal.download_best_subtitles([video], languages, hearing_impaired=False, providers=self.subproviders) try: subliminal.save_subtitles(video, subtitles[video]) except: # Support for older versions of subliminal subliminal.save_subtitles(subtitles) self.log.info("Please update to the latest version of subliminal.") except Exception as e: self.log.info("Unable to download subtitles.", exc_info=True) self.log.debug("Unable to download subtitles.", exc_info=True) # External subtitle import if self.embedsubs and not self.embedonlyinternalsubs: # Don't bother if we're not embeddeding subtitles and external subtitles src = 1 # FFMPEG input source number for dirName, subdirList, fileList in os.walk(input_dir): for fname in fileList: subname, subextension = os.path.splitext(fname) # Watch for appropriate file extension if subextension[1:] in valid_subtitle_extensions: x, lang = os.path.splitext(subname) # Using babelfish to convert a 2 language code to a 3 language code if len(lang) is 2: try: babel = Language.fromalpha2(lang) lang = babel.alpha3t except: pass # If subtitle file name and input video name are the same, proceed if x == filename: self.log.info("External %s subtitle file detected." % lang) if self.swl == 'und' or lang in self.swl: self.log.info("Creating subtitle stream %s by importing %s." % (l, fname)) subtitle_settings.update({l: { 'path': os.path.join(dirName, fname), 'source': src, 'map': 0, 'codec': 'mov_text', 'language': lang}}) self.log.debug("Path: %s." % os.path.join(dirName, fname)) self.log.debug("Source: %s." % src) self.log.debug("Codec: mov_text.") self.log.debug("Language: %s." % lang) l = l + 1 src = src + 1 self.deletesubs.add(os.path.join(dirName, fname)) else: self.log.info("Ignoring %s external subtitle stream due to language %s." % (fname, lang)) # Collect all options options = { 'format': self.output_format, 'video': { 'codec': vcodec, 'map': info.video.index, 'bitrate': vbitrate, 'level': self.h264_level, 'profile': vprofile, 'pix_fmt': pix_fmt }, 'audio': audio_settings, 'subtitle': subtitle_settings, 'preopts': [], 'postopts': ['-threads', self.threads] } # If a CRF option is set, override the determine bitrate if self.vcrf: del options['video']['bitrate'] options['video']['crf'] = self.vcrf if len(options['subtitle']) > 0: options['preopts'].append('-fix_sub_duration') if self.preopts: options['preopts'].extend(self.preopts) if self.postopts: options['postopts'].extend(self.postopts) if self.dxva2_decoder: # DXVA2 will fallback to CPU decoding when it hits a file that it cannot handle, so we don't need to check if the file is supported. options['preopts'].extend(['-hwaccel', 'dxva2']) elif info.video.codec.lower() == "hevc" and self.hevc_qsv_decoder: options['preopts'].extend(['-vcodec', 'hevc_qsv']) elif vcodec == "h264qsv" and info.video.codec.lower() == "h264" and self.qsv_decoder and (info.video.video_level / 10) < 5: options['preopts'].extend(['-vcodec', 'h264_qsv']) # Add width option if vwidth: options['video']['width'] = vwidth # HEVC Tagging for copied streams if info.video.codec.lower() in ['x265', 'h265', 'hevc'] and vcodec == 'copy': options['postopts'].extend(['-tag:v', 'hvc1']) self.log.info("Tagging copied video stream as hvc1") self.options = options return options