def test_ne_with_country(self): self.assertTrue(Language('Portuguese') != Language('Portuguese (BR)')) self.assertTrue(Language('English (US)') != Language('English (GB)'))
def run(self, scan_path, scan_age, languages, encoding, min_score, providers, provider_configs, max_workers, plex_url=None, plex_token=None, *args, **kwargs): if not os.path.isdir(scan_path): raise IOError('Path \'%s\' doesn\'t exist!' % scan_path) if not scan_age >= 1: raise ValueError('\'scan_age\' must by at least 1!') if not len(languages) >= 1: raise ValueError('\'languages\' list can\'t be empty!') if not providers: raise ValueError('\'providers\' argument can\'t be empty!') if not max_workers >= 1: raise ValueError('\'max_workers\' must be at least 1!') if not provider_configs: provider_configs = {} __tree_dict = lambda: defaultdict(__tree_dict) result = __tree_dict() encoding = codecs.lookup(encoding).name age = timedelta(weeks=scan_age) languages = set([Language(l) for l in languages]) plex = None if plex_url and plex_token: plex = PlexServer(plex_url, plex_token) scan_start = datetime.now() videos = [] ignored_videos = [] if not region.is_configured: region.configure('dogpile.cache.dbm', expiration_time=timedelta(days=30), arguments={ 'filename': 'subliminal.dbm', 'lock_factory': MutexLock }) # scan videos scanned_videos = scan_videos(scan_path, age=age) for video in scanned_videos: video.subtitle_languages |= set( search_external_subtitles(video.name).values()) if check_video(video, languages=languages, age=age, undefined=False): refine(video) if languages - video.subtitle_languages: videos.append(video) else: ignored_videos.append(video) else: ignored_videos.append(video) if videos: result['videos']['collected'] = [ os.path.split(v.name)[1] for v in videos ] if ignored_videos: result['videos']['ignored'] = len(ignored_videos) if videos: # download best subtitles downloaded_subtitles = defaultdict(list) with AsyncProviderPool(max_workers=max_workers, providers=providers, provider_configs=provider_configs) as p: for video in videos: scores = get_scores(video) subtitles_to_download = p.list_subtitles( video, languages - video.subtitle_languages) downloaded_subtitles[video] = p.download_best_subtitles( subtitles_to_download, video, languages, min_score=scores['hash'] * min_score / 100) if p.discarded_providers: result['providers']['discarded'] = list( p.discarded_providers) # filter subtitles with TinyDB('subtitle_db.json') as db: table = db.table('downloaded') query = Query() for video, subtitles in downloaded_subtitles.items(): discarded_subtitles = list() discarded_subtitles_info = list() for s in subtitles: subtitle_hash = hashlib.sha256(s.content).hexdigest() subtitle_file = get_subtitle_path( os.path.split(video.name)[1], s.language) dbo = {'hash': subtitle_hash, 'file': subtitle_file} if table.search((query.hash == subtitle_hash) & (query.file == subtitle_file)): discarded_subtitles.append(s) discarded_subtitles_info.append(dbo) else: table.insert(dbo) downloaded_subtitles[video] = [ x for x in subtitles if x not in discarded_subtitles ] if discarded_subtitles_info: result['subtitles'][ 'discarded'] = result['subtitles'].get( 'discarded', []) + discarded_subtitles_info downloaded_subtitles = { k: v for k, v in downloaded_subtitles.items() if v } # save subtitles saved_subtitles = {} for video, subtitles in downloaded_subtitles.items(): saved_subtitles[video] = save_subtitles(video, subtitles, directory=None, encoding=encoding) for key, group in groupby(saved_subtitles[video], lambda x: x.provider_name): subtitle_filenames = [ get_subtitle_path( os.path.split(video.name)[1], s.language) for s in list(group) ] result['subtitles'][key] = result['subtitles'].get( key, []) + subtitle_filenames result['subtitles']['total'] = sum( len(v) for v in saved_subtitles.values()) # refresh plex for video, subtitles in saved_subtitles.items(): if plex and subtitles: item_found = False for section in plex.library.sections(): try: if isinstance(section, MovieSection) and isinstance( video, Movie): results = section.search(title=video.title, year=video.year, libtype='movie', sort='addedAt:desc', maxresults=1) if not results: raise NotFound plex_item = results[0] elif isinstance(section, ShowSection) and isinstance( video, Episode): results = section.search(title=video.series, year=video.year, libtype='show', sort='addedAt:desc', maxresults=1) if not results: raise NotFound plex_item = results[0].episode( season=video.season, episode=video.episode) else: continue except NotFound: continue except BadRequest: continue if plex_item: plex_item.refresh() result['plex']['refreshed'] = result['plex'].get( 'refreshed', []) + [ '%s%s' % (repr(plex_item.section()), repr(video)) ] item_found = True if not item_found: result['plex']['failed'] = result['plex'].get( 'failed', []) + [repr(video)] # convert subtitles for video, subtitles in saved_subtitles.items(): target_format = aeidon.formats.SUBRIP for s in subtitles: subtitle_path = get_subtitle_path(video.name, s.language) source_format = aeidon.util.detect_format( subtitle_path, encoding) source_file = aeidon.files.new( source_format, subtitle_path, aeidon.encodings.detect_bom(subtitle_path) or encoding) if source_format != target_format: format_info = { 'file': get_subtitle_path( os.path.split(video.name)[1], s.language), 'from': source_format.label, 'to': target_format.label } result['subtitles'][ 'converted'] = result['subtitles'].get( 'converted', []) + [format_info] aeidon_subtitles = source_file.read() for f in [ aeidon.formats.SUBRIP, aeidon.formats.MICRODVD, aeidon.formats.MPL2 ]: markup = aeidon.markups.new(f) for s in aeidon_subtitles: s.main_text = markup.decode(s.main_text) markup = aeidon.markups.new(target_format) for s in aeidon_subtitles: s.main_text = markup.encode(s.main_text) target_file = aeidon.files.new(target_format, subtitle_path, encoding) target_file.write(aeidon_subtitles, aeidon.documents.MAIN) scan_end = datetime.now() result['meta']['start'] = scan_start.isoformat() result['meta']['end'] = scan_end.isoformat() result['meta']['duration'] = str(scan_end - scan_start) return result
x = 'n' while x != 'y': index = randint(0, countGS - 1) print("Launching " + str(stashnames[index])) findGenre(stashnames[index]) print("\n\n......Good enough? Y/N\n") x = str(getch(), 'utf-8') print( "\n=================================================================================\n" ) if x.lower() == 'y': if dlSub == True: print("\n\nGetting subtitles..Please Wait..\n\n") try: region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) videos = Video.fromname(stash[index]) subtitles = download_best_subtitles([videos], {Language('eng')}) best_subtitle = subtitles[videos][0] save_subtitles(videos, [best_subtitle]) print("\nSubtitle downloaded.\n") except: print("\nCould not get subtitle :\\\n") os.startfile(stash[index]) elif x.lower() == 'n': print("Moving on...\n") else: print("\nWut? wut? i'll just assume it wasn't of your taste sire.\n")
def generateOptions(self, inputfile, original=None, force_transcode=False): # Get path information from the input file input_dir, filename, input_extension = self.parseFile(inputfile) info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile) # Video stream self.log.info("Reading video stream.") self.log.info("Video codec detected: %s." % info.video.codec) try: vbr = self.estimateVideoBitrate(info) except: vbr = info.format.bitrate / 1000 if info.video.codec.lower( ) in self.video_codec and not force_transcode: vcodec = 'copy' else: vcodec = self.video_codec[0] vbitrate = self.video_bitrate if self.video_bitrate else vbr self.log.info("Pix Fmt: %s." % info.video.pix_fmt) if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt: self.log.debug( "Overriding video pix_fmt. Codec cannot be copied because pix_fmt is not approved." ) vcodec = self.video_codec[0] pix_fmt = self.pix_fmt[0] if self.video_profile: vprofile = self.video_profile[0] else: pix_fmt = None if self.video_bitrate is not None and vbr > self.video_bitrate: self.log.debug( "Overriding video bitrate. Codec cannot be copied because video bitrate is too high." ) vcodec = self.video_codec[0] vbitrate = self.video_bitrate if self.video_width is not None and self.video_width < info.video.video_width: self.log.debug( "Video width is over the max width, it will be downsampled. Video stream can no longer be copied." ) vcodec = self.video_codec[0] vwidth = self.video_width else: vwidth = None if '264' in info.video.codec.lower( ) and self.h264_level and info.video.video_level and ( info.video.video_level / 10 > self.h264_level): self.log.info("Video level %0.1f." % (info.video.video_level / 10)) vcodec = self.video_codec[0] self.log.debug("Video codec: %s." % vcodec) self.log.debug("Video bitrate: %s." % vbitrate) self.log.info("Profile: %s." % info.video.profile) if self.video_profile and info.video.profile.lower().replace( " ", "") not in self.video_profile: self.log.debug( "Video profile is not supported. Video stream can no longer be copied." ) vcodec = self.video_codec[0] vprofile = self.video_profile[0] if self.pix_fmt: pix_fmt = self.pix_fmt[0] else: vprofile = None # Audio streams self.log.info("Reading audio streams.") overrideLang = True for a in info.audio: try: if a.metadata['language'].strip( ) == "" or a.metadata['language'] is None: a.metadata['language'] = 'und' except KeyError: a.metadata['language'] = 'und' if (a.metadata['language'] == 'und' and self.adl) or ( self.awl and a.metadata['language'].lower() in self.awl): overrideLang = False break if overrideLang: self.awl = None self.log.info( "No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present." ) audio_settings = {} blocked_audio_languages = [] l = 0 for a in info.audio: try: if a.metadata['language'].strip( ) == "" or a.metadata['language'] is None: a.metadata['language'] = 'und' except KeyError: a.metadata['language'] = 'und' self.log.info("Audio detected for stream #%s: %s [%s]." % (a.index, a.codec, a.metadata['language'])) if self.output_extension in valid_tagging_extensions and a.codec.lower( ) == 'truehd' and self.ignore_truehd: # Need to skip it early so that it flags the next track as default. self.log.info( "MP4 containers do not support truehd audio, and converting it is inconsistent due to video/audio sync issues. Skipping stream %s as typically the 2nd audio track is the AC3 core of the truehd stream." % a.index) continue # Set undefined language to default language if specified if self.adl is not None and a.metadata['language'] == 'und': self.log.debug( "Undefined language detected, defaulting to [%s]." % self.adl) a.metadata['language'] = self.adl # Proceed if no whitelist is set, or if the language is in the whitelist iosdata = None if self.awl is None or (a.metadata['language'].lower() in self.awl and a.metadata['language'].lower() not in blocked_audio_languages): # Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo) if self.iOS and a.audio_channels > 2: iOSbitrate = 256 if (self.audio_bitrate * 2) > 256 else ( self.audio_bitrate * 2) self.log.info( "Creating audio stream %s from source audio stream %s [iOS-audio]." % (str(l), a.index)) self.log.debug("Audio codec: %s." % self.iOS[0]) self.log.debug("Channels: 2.") self.log.debug("Filter: %s." % self.iOS_filter) self.log.debug("Bitrate: %s." % iOSbitrate) self.log.debug("Language: %s." % a.metadata['language']) iosdata = { 'map': a.index, 'codec': self.iOS[0], 'channels': 2, 'bitrate': iOSbitrate, 'filter': self.iOS_filter, 'language': a.metadata['language'], 'disposition': 'none', } if not self.iOSLast: audio_settings.update({l: iosdata}) l += 1 # If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels self.log.info( "Creating audio stream %s from source stream %s." % (str(l), a.index)) if self.iOS and a.audio_channels <= 2: self.log.debug( "Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio]." ) acodec = 'copy' if a.codec in self.iOS else self.iOS[0] audio_channels = a.audio_channels afilter = self.iOS_filter abitrate = a.audio_channels * 128 if ( a.audio_channels * self.audio_bitrate) > ( a.audio_channels * 128) else (a.audio_channels * self.audio_bitrate) else: # If desired codec is the same as the source codec, copy to avoid quality loss acodec = 'copy' if a.codec.lower( ) in self.audio_codec else self.audio_codec[0] # Audio channel adjustments if self.maxchannels and a.audio_channels > self.maxchannels: audio_channels = self.maxchannels if acodec == 'copy': acodec = self.audio_codec[0] abitrate = self.maxchannels * self.audio_bitrate else: audio_channels = a.audio_channels abitrate = a.audio_channels * self.audio_bitrate # Bitrate calculations/overrides if self.audio_bitrate is 0: self.log.debug( "Attempting to set bitrate based on source stream bitrate." ) try: abitrate = a.bitrate / 1000 except: self.log.warning( "Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel." % a.index) abitrate = a.audio_channels * 256 afilter = self.audio_filter self.log.debug("Audio codec: %s." % acodec) self.log.debug("Channels: %s." % audio_channels) self.log.debug("Bitrate: %s." % abitrate) self.log.debug("Language: %s" % a.metadata['language']) self.log.debug("Filter: %s" % afilter) # If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed if self.iOS and self.iOSFirst: self.log.debug( "Not creating any additional iOS audio streams.") self.iOS = False audio_settings.update({ l: { 'map': a.index, 'codec': acodec, 'channels': audio_channels, 'bitrate': abitrate, 'filter': afilter, 'language': a.metadata['language'], 'disposition': 'none', } }) if acodec == 'copy' and a.codec == 'aac' and self.aac_adtstoasc: audio_settings[l]['bsf'] = 'aac_adtstoasc' l += 1 # Add the iOS track last instead if self.iOSLast and iosdata: audio_settings.update({l: iosdata}) l += 1 if self.audio_copyoriginal and acodec != 'copy': self.log.info( "Adding copy of original audio track in format %s" % a.codec) audio_settings.update({ l: { 'map': a.index, 'codec': 'copy', 'language': a.metadata['language'], 'disposition': 'none', } }) # Remove the language if we only want the first track from a given language if self.audio_first_language_track and self.awl: try: blocked_audio_languages.append( a.metadata['language'].lower()) self.log.debug( "Removing language from whitelist to prevent multiple tracks of the same: %s." % a.metadata['language']) except: self.log.error( "Unable to remove language %s from whitelist." % a.metadata['language']) # Audio Default if len(audio_settings) > 0 and self.adl: try: default_track = [ x for x in audio_settings.values() if x['language'] == self.adl ][0] default_track['disposition'] = 'default' except: audio_settings[0]['disposition'] = 'default' else: self.log.error("Audio language array is empty.") # Subtitle streams subtitle_settings = {} l = 0 self.log.info("Reading subtitle streams.") for s in info.subtitle: try: if s.metadata['language'].strip( ) == "" or s.metadata['language'] is None: s.metadata['language'] = 'und' except KeyError: s.metadata['language'] = 'und' self.log.info("Subtitle detected for stream #%s: %s [%s]." % (s.index, s.codec, s.metadata['language'])) # Set undefined language to default language if specified if self.sdl is not None and s.metadata['language'] == 'und': self.log.debug( "Undefined language detected, defaulting to [%s]." % self.sdl) s.metadata['language'] = self.sdl # Make sure its not an image based codec if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs: # Proceed if no whitelist is set, or if the language is in the whitelist if self.swl is None or s.metadata['language'].lower( ) in self.swl: subtitle_settings.update({ l: { 'map': s.index, 'codec': self.scodec[0], 'language': s.metadata['language'], 'encoding': self.subencoding, 'disposition': 'none', # 'forced': s.sub_forced, # 'default': s.sub_default } }) self.log.info( "Creating subtitle stream %s from source stream %s." % (l, s.index)) l = l + 1 elif s.codec.lower( ) not in bad_subtitle_codecs and not self.embedsubs: if self.swl is None or s.metadata['language'].lower( ) in self.swl: for codec in self.scodec: ripsub = { 0: { 'map': s.index, 'codec': codec, 'language': s.metadata['language'] } } options = { 'format': codec, 'subtitle': ripsub, } try: extension = subtitle_codec_extensions[codec] except: self.log.info( "Wasn't able to determine subtitle file extension, defaulting to '.srt'." ) extension = 'srt' forced = ".forced" if s.sub_forced else "" input_dir, filename, input_extension = self.parseFile( inputfile) output_dir = input_dir if self.output_dir is None else self.output_dir outputfile = os.path.join( output_dir, filename + "." + s.metadata['language'] + forced + "." + extension) i = 2 while os.path.isfile(outputfile): self.log.debug( "%s exists, appending %s to filename." % (outputfile, i)) outputfile = os.path.join( output_dir, filename + "." + s.metadata['language'] + forced + "." + str(i) + "." + extension) i += 1 try: self.log.info( "Ripping %s subtitle from source stream %s into external file." % (s.metadata['language'], s.index)) conv = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).convert( inputfile, outputfile, options, timeout=None) for timecode in conv: pass self.log.info("%s created." % outputfile) except: self.log.exception( "Unabled to create external subtitle file for stream %s." % (s.index)) try: os.chmod(outputfile, self.permissions ) # Set permissions of newly created file except: self.log.exception( "Unable to set new file permissions.") # Attempt to download subtitles if they are missing using subliminal languages = set() try: if self.swl: for alpha3 in self.swl: languages.add(Language(alpha3)) elif self.sdl: languages.add(Language(self.sdl)) else: self.downloadsubs = False self.log.error( "No valid subtitle language specified, cannot download subtitles." ) except: self.log.exception( "Unable to verify subtitle languages for download.") self.downloadsubs = False if self.downloadsubs: import subliminal self.log.info("Attempting to download subtitles.") # Attempt to set the dogpile cache try: subliminal.region.configure('dogpile.cache.memory') except: pass try: video = subliminal.scan_video(os.path.abspath(inputfile), subtitles=True, embedded_subtitles=True) subtitles = subliminal.download_best_subtitles( [video], languages, hearing_impaired=False, providers=self.subproviders) try: subliminal.save_subtitles(video, subtitles[video]) except: # Support for older versions of subliminal subliminal.save_subtitles(subtitles) self.log.info( "Please update to the latest version of subliminal.") except Exception as e: self.log.info("Unable to download subtitles.", exc_info=True) self.log.debug("Unable to download subtitles.", exc_info=True) # External subtitle import if self.embedsubs and not self.embedonlyinternalsubs: # Don't bother if we're not embeddeding subtitles and external subtitles src = 1 # FFMPEG input source number for dirName, subdirList, fileList in os.walk(input_dir): for fname in fileList: subname, subextension = os.path.splitext(fname) # Watch for appropriate file extension if subextension[1:] in valid_subtitle_extensions: x, lang = os.path.splitext(subname) lang = lang[1:] # Using bablefish to convert a 2 language code to a 3 language code if len(lang) is 2: try: babel = Language.fromalpha2(lang) lang = babel.alpha3 except: pass # If subtitle file name and input video name are the same, proceed if x == filename: self.log.info( "External %s subtitle file detected." % lang) if self.swl is None or lang in self.swl: self.log.info( "Creating subtitle stream %s by importing %s." % (l, fname)) subtitle_settings.update({ l: { 'path': os.path.join(dirName, fname), 'source': src, 'map': 0, 'codec': 'mov_text', 'disposition': 'none', 'language': lang } }) self.log.debug("Path: %s." % os.path.join(dirName, fname)) self.log.debug("Source: %s." % src) self.log.debug("Codec: mov_text.") self.log.debug("Langauge: %s." % lang) l = l + 1 src = src + 1 self.deletesubs.add( os.path.join(dirName, fname)) else: self.log.info( "Ignoring %s external subtitle stream due to language %s." % (fname, lang)) # Subtitle Default if len(subtitle_settings) > 0 and self.sdl: try: default_track = [ x for x in subtitle_settings.values() if x['language'] == self.sdl ][0] default_track['disposition'] = 'default' except: subtitle_settings[0]['disposition'] = 'default' else: self.log.warning("Subtitle language array is empty.") # Collect all options options = { 'format': self.output_format, 'video': { 'codec': vcodec, 'map': info.video.index, 'bitrate': vbitrate, 'level': self.h264_level, 'profile': vprofile, 'pix_fmt': pix_fmt }, 'audio': audio_settings, 'subtitle': subtitle_settings, 'preopts': [], 'postopts': ['-threads', self.threads] } # If a CRF option is set, override the determine bitrate if self.vcrf: del options['video']['bitrate'] options['video']['crf'] = self.vcrf if len(options['subtitle']) > 0: options['preopts'].append('-fix_sub_duration') if self.preopts: options['preopts'].extend(self.preopts) if self.postopts: options['postopts'].extend(self.postopts) if self.dxva2_decoder: # DXVA2 will fallback to CPU decoding when it hits a file that it cannot handle, so we don't need to check if the file is supported. options['preopts'].extend(['-hwaccel', 'dxva2']) elif info.video.codec.lower() == "hevc" and self.hevc_qsv_decoder: options['preopts'].extend(['-vcodec', 'hevc_qsv']) elif vcodec == "h264qsv" and info.video.codec.lower( ) == "h264" and self.qsv_decoder and (info.video.video_level / 10) < 5: options['preopts'].extend(['-vcodec', 'h264_qsv']) if self.auto_crop: options['video']['mode'] = 'auto_crop' # Add width option if vwidth: options['video']['width'] = vwidth # HEVC Tagging for copied streams if info.video.codec.lower() in ['x265', 'h265', 'hevc' ] and vcodec == 'copy': options['postopts'].extend(['-tag:v', 'hvc1']) self.log.info("Tagging copied video stream as hvc1") self.options = options return options
def from_code(language): language = language.strip() if language and language in language_converters['opensubtitles'].codes: return Language.fromopensubtitles(language) # pylint: disable=no-member return Language('und')
class ItaSAProvider(Provider): languages = {Language('ita')} video_types = (Episode,) server_url = 'https://api.italiansubs.net/api/rest/' apikey = 'd86ad6ec041b334fac1e512174ee04d5' def __init__(self, username=None, password=None): if username is not None and password is None or username is None and password is not None: raise ConfigurationError('Username and password must be specified') self.username = username self.password = password self.logged_in = False self.login_itasa = False self.session = None self.auth_code = None def initialize(self): self.session = Session() self.session.headers['User-Agent'] = 'Subliminal/{}'.format(__version__) # login if self.username is not None and self.password is not None: logger.info('Logging in') params = { 'username': self.username, 'password': self.password, 'apikey': self.apikey } r = self.session.get(self.server_url + 'users/login', params=params, timeout=10) root = etree.fromstring(r.content) if root.find('status').text == 'fail': raise AuthenticationError(root.find('error/message').text) self.auth_code = root.find('data/user/authcode').text data = { 'username': self.username, 'passwd': self.password, 'remember': 'yes', 'option': 'com_user', 'task': 'login', 'silent': 'true' } r = self.session.post('http://www.italiansubs.net/index.php', data=data, timeout=30) r.raise_for_status() self.logged_in = True def terminate(self): self.session.close() self.logged_in = False @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME) def _get_show_ids(self): """Get the ``dict`` of show ids per series by querying the `shows` page. :return: show id per series, lower case and without quotes. :rtype: dict """ # get the show page logger.info('Getting show ids') params = {'apikey': self.apikey} r = self.session.get(self.server_url + 'shows', timeout=10, params=params) r.raise_for_status() root = etree.fromstring(r.content) # populate the show ids show_ids = {} for show in root.findall('data/shows/show'): if show.find('name').text is None: # pragma: no cover continue show_ids[sanitize(show.find('name').text).lower()] = int(show.find('id').text) logger.debug('Found %d show ids', len(show_ids)) return show_ids @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME) def _search_show_id(self, series): """Search the show id from the `series` :param str series: series of the episode. :return: the show id, if found. :rtype: int or None """ # build the param params = {'apikey': self.apikey, 'q': series} # make the search logger.info('Searching show ids with %r', params) r = self.session.get(self.server_url + 'shows/search', params=params, timeout=10) r.raise_for_status() root = etree.fromstring(r.content) if int(root.find('data/count').text) == 0: logger.warning('Show id not found: no suggestion') return None # Looking for show in first page for show in root.findall('data/shows/show'): if sanitize(show.find('name').text).lower() == sanitize(series.lower()): show_id = int(show.find('id').text) logger.debug('Found show id %d', show_id) return show_id # Not in the first page of result try next (if any) next_page = root.find('data/next') while next_page.text is not None: # pragma: no cover r = self.session.get(next_page.text, timeout=10) r.raise_for_status() root = etree.fromstring(r.content) logger.info('Loading suggestion page %r', root.find('data/page').text) # Looking for show in following pages for show in root.findall('data/shows/show'): if sanitize(show.find('name').text).lower() == sanitize(series.lower()): show_id = int(show.find('id').text) logger.debug('Found show id %d', show_id) return show_id next_page = root.find('data/next') # No matches found logger.warning('Show id not found: suggestions does not match') return None def get_show_id(self, series, country_code=None): """Get the best matching show id for `series`. First search in the result of :meth:`_get_show_ids` and fallback on a search with :meth:`_search_show_id` :param str series: series of the episode. :param str country_code: the country in which teh show is aired. :return: the show id, if found. :rtype: int or None """ series_sanitized = sanitize(series).lower() show_ids = self._get_show_ids() show_id = None # attempt with country if not show_id and country_code: logger.debug('Getting show id with country') show_id = show_ids.get('{0} {1}'.format(series_sanitized, country_code.lower())) # attempt clean if not show_id: logger.debug('Getting show id') show_id = show_ids.get(series_sanitized) # search as last resort if not show_id: logger.warning('Series not found in show ids') show_id = self._search_show_id(series) return show_id @region.cache_on_arguments(expiration_time=EPISODE_EXPIRATION_TIME) def _download_zip(self, sub_id): # download the subtitle logger.info('Downloading subtitle %r', sub_id) params = { 'authcode': self.auth_code, 'apikey': self.apikey, 'subtitle_id': sub_id } r = self.session.get(self.server_url + 'subtitles/download', params=params, timeout=30) r.raise_for_status() return r.content def _get_season_subtitles(self, show_id, season, sub_format): params = { 'apikey': self.apikey, 'show_id': show_id, 'q': 'Stagione %{}'.format(season), 'version': sub_format } r = self.session.get(self.server_url + 'subtitles/search', params=params, timeout=30) r.raise_for_status() root = etree.fromstring(r.content) if int(root.find('data/count').text) == 0: logger.warning('Subtitles for season not found, try with rip suffix') params['version'] = sub_format + 'rip' r = self.session.get(self.server_url + 'subtitles/search', params=params, timeout=30) r.raise_for_status() root = etree.fromstring(r.content) if int(root.find('data/count').text) == 0: logger.warning('Subtitles for season not found') return [] subs = [] # Looking for subtitles in first page season_re = re.compile('.*?stagione 0*?{}.*'.format(season)) for subtitle in root.findall('data/subtitles/subtitle'): if season_re.match(subtitle.find('name').text.lower()): logger.debug('Found season zip id %d - %r - %r', int(subtitle.find('id').text), subtitle.find('name').text, subtitle.find('version').text) content = self._download_zip(int(subtitle.find('id').text)) if not is_zipfile(io.BytesIO(content)): # pragma: no cover if 'limite di download' in content: raise TooManyRequests() else: raise ConfigurationError('Not a zip file: {!r}'.format(content)) with ZipFile(io.BytesIO(content)) as zf: episode_re = re.compile('s(\d{1,2})e(\d{1,2})') for index, name in enumerate(zf.namelist()): match = episode_re.search(name) if not match: # pragma: no cover logger.debug('Cannot decode subtitle %r', name) else: sub = ItaSASubtitle( int(subtitle.find('id').text), subtitle.find('show_name').text, int(match.group(1)), int(match.group(2)), None, None, None, name) sub.content = fix_line_ending(zf.read(name)) subs.append(sub) return subs def query(self, series, season, episode, video_format, resolution, country=None): # To make queries you need to be logged in if not self.logged_in: # pragma: no cover raise ConfigurationError('Cannot query if not logged in') # get the show id show_id = self.get_show_id(series, country) if show_id is None: logger.error('No show id found for %r ', series) return [] # get the page of the season of the show logger.info('Getting the subtitle of show id %d, season %d episode %d, format %r', show_id, season, episode, video_format) subtitles = [] # Default format is SDTV if not video_format or video_format.lower() == 'hdtv': if resolution in ('1080i', '1080p', '720p'): sub_format = resolution else: sub_format = 'normale' else: sub_format = video_format.lower() # Look for year params = { 'apikey': self.apikey } r = self.session.get(self.server_url + 'shows/' + str(show_id), params=params, timeout=30) r.raise_for_status() root = etree.fromstring(r.content) year = root.find('data/show/started').text if year: year = int(year.split('-', 1)[0]) tvdb_id = root.find('data/show/id_tvdb').text if tvdb_id: tvdb_id = int(tvdb_id) params = { 'apikey': self.apikey, 'show_id': show_id, 'q': '{0}x{1:02}'.format(season, episode), 'version': sub_format } r = self.session.get(self.server_url + 'subtitles/search', params=params, timeout=30) r.raise_for_status() root = etree.fromstring(r.content) if int(root.find('data/count').text) == 0: logger.warning('Subtitles not found, try with rip suffix') params['version'] = sub_format + 'rip' r = self.session.get(self.server_url + 'subtitles/search', params=params, timeout=30) r.raise_for_status() root = etree.fromstring(r.content) if int(root.find('data/count').text) == 0: logger.warning('Subtitles not found, go season mode') # If no subtitle are found for single episode try to download all season zip subs = self._get_season_subtitles(show_id, season, sub_format) if subs: for subtitle in subs: subtitle.format = video_format subtitle.year = year subtitle.tvdb_id = tvdb_id return subs else: return [] # Looking for subtitles in first page for subtitle in root.findall('data/subtitles/subtitle'): if '{0}x{1:02}'.format(season, episode) in subtitle.find('name').text.lower(): logger.debug('Found subtitle id %d - %r - %r', int(subtitle.find('id').text), subtitle.find('name').text, subtitle.find('version').text) sub = ItaSASubtitle( int(subtitle.find('id').text), subtitle.find('show_name').text, season, episode, video_format, year, tvdb_id, subtitle.find('name').text) subtitles.append(sub) # Not in the first page of result try next (if any) next_page = root.find('data/next') while next_page.text is not None: # pragma: no cover r = self.session.get(next_page.text, timeout=30) r.raise_for_status() root = etree.fromstring(r.content) logger.info('Loading subtitles page %r', root.data.page.text) # Looking for show in following pages for subtitle in root.findall('data/subtitles/subtitle'): if '{0}x{1:02}'.format(season, episode) in subtitle.find('name').text.lower(): logger.debug('Found subtitle id %d - %r - %r', int(subtitle.find('id').text), subtitle.find('name').text, subtitle.find('version').text) sub = ItaSASubtitle( int(subtitle.find('id').text), subtitle.find('show_name').text, season, episode, video_format, year, tvdb_id, subtitle.find('name').text) subtitles.append(sub) next_page = root.find('data/next') # Download the subs found, can be more than one in zip additional_subs = [] for sub in subtitles: # open the zip content = self._download_zip(sub.sub_id) if not is_zipfile(io.BytesIO(content)): # pragma: no cover if 'limite di download' in content: raise TooManyRequests() else: raise ConfigurationError('Not a zip file: {!r}'.format(content)) with ZipFile(io.BytesIO(content)) as zf: if len(zf.namelist()) > 1: # pragma: no cover for index, name in enumerate(zf.namelist()): if index == 0: # First element sub.content = fix_line_ending(zf.read(name)) sub.full_data = name else: add_sub = copy.deepcopy(sub) add_sub.content = fix_line_ending(zf.read(name)) add_sub.full_data = name additional_subs.append(add_sub) else: sub.content = fix_line_ending(zf.read(zf.namelist()[0])) sub.full_data = zf.namelist()[0] return subtitles + additional_subs def list_subtitles(self, video, languages): return self.query(video.series, video.season, video.episode, video.format, video.resolution) def download_subtitle(self, subtitle): # pragma: no cover pass
def test_query_not_enough_information(): languages = {Language('eng')} with OpenSubtitlesProvider(USERNAME, PASSWORD) as provider: with pytest.raises(ValueError) as excinfo: provider.query(languages) assert str(excinfo.value) == 'Not enough information'
def test_get_matches_no_match(episodes): subtitle = SubdivxSubtitle( Language('es'), None, None, 'The.Big.Bang.Theory.S07E05.720p.x264-dimension.mkv', None, None, None) matches = subtitle.get_matches(episodes['house_of_cards_us_s06e01']) assert matches == set()
def test_subtitle_text(): subtitle = Subtitle(Language('eng')) subtitle.content = b'Some ascii text' assert subtitle.text == 'Some ascii text'
def scan_video(path, subtitles=True, embedded_subtitles=True): """Scan a video and its subtitle languages from a video `path`. :param str path: existing path to the video. :param bool subtitles: scan for subtitles with the same name. :param bool embedded_subtitles: scan for embedded subtitles. :return: the scanned video. :rtype: :class:`Video` """ # check for non-existing path if not os.path.exists(path): raise ValueError('Path does not exist') # check video extension if not path.endswith(VIDEO_EXTENSIONS): raise ValueError('%s is not a valid video extension' % os.path.splitext(path)[1]) dirpath, filename = os.path.split(path) logger.info('Scanning video %r in %r', filename, dirpath) # guess video = Video.fromguess(path, guess_file_info(path)) # size and hashes video.size = os.path.getsize(path) if video.size > 10485760: logger.debug('Size is %d', video.size) video.hashes['opensubtitles'] = hash_opensubtitles(path) video.hashes['thesubdb'] = hash_thesubdb(path) logger.debug('Computed hashes %r', video.hashes) else: logger.warning('Size is lower than 10MB: hashes not computed') # external subtitles if subtitles: video.subtitle_languages |= set(search_external_subtitles(path).values()) # video metadata with enzyme try: if filename.endswith('.mkv'): with open(path, 'rb') as f: mkv = MKV(f) # main video track if mkv.video_tracks: video_track = mkv.video_tracks[0] # resolution if video_track.height in (480, 720, 1080): if video_track.interlaced: video.resolution = '%di' % video_track.height else: video.resolution = '%dp' % video_track.height logger.debug('Found resolution %s with enzyme', video.resolution) # video codec if video_track.codec_id == 'V_MPEG4/ISO/AVC': video.video_codec = 'h264' logger.debug('Found video_codec %s with enzyme', video.video_codec) elif video_track.codec_id == 'V_MPEG4/ISO/SP': video.video_codec = 'DivX' logger.debug('Found video_codec %s with enzyme', video.video_codec) elif video_track.codec_id == 'V_MPEG4/ISO/ASP': video.video_codec = 'XviD' logger.debug('Found video_codec %s with enzyme', video.video_codec) else: logger.warning('MKV has no video track') # main audio track if mkv.audio_tracks: audio_track = mkv.audio_tracks[0] # audio codec if audio_track.codec_id == 'A_AC3': video.audio_codec = 'AC3' logger.debug('Found audio_codec %s with enzyme', video.audio_codec) elif audio_track.codec_id == 'A_DTS': video.audio_codec = 'DTS' logger.debug('Found audio_codec %s with enzyme', video.audio_codec) elif audio_track.codec_id == 'A_AAC': video.audio_codec = 'AAC' logger.debug('Found audio_codec %s with enzyme', video.audio_codec) else: logger.warning('MKV has no audio track') # subtitle tracks if mkv.subtitle_tracks: if embedded_subtitles: embedded_subtitle_languages = set() for st in mkv.subtitle_tracks: if st.language: try: embedded_subtitle_languages.add(Language.fromalpha3b(st.language)) except BabelfishError: logger.error('Embedded subtitle track language %r is not a valid language', st.language) embedded_subtitle_languages.add(Language('und')) elif st.name: try: embedded_subtitle_languages.add(Language.fromname(st.name)) except BabelfishError: logger.debug('Embedded subtitle track name %r is not a valid language', st.name) embedded_subtitle_languages.add(Language('und')) else: embedded_subtitle_languages.add(Language('und')) logger.debug('Found embedded subtitle %r with enzyme', embedded_subtitle_languages) video.subtitle_languages |= embedded_subtitle_languages else: logger.debug('MKV has no subtitle track') except EnzymeError: logger.exception('Parsing video metadata with enzyme failed') return video
def test_get_matches_release_group(episodes): subtitle = SubdivxSubtitle( Language('es'), None, None, 'The.Big.Bang.Theory.S07E05.720p.x264-dimension.mkv', None, None, None) matches = subtitle.get_matches(episodes['bbt_s07e05']) assert matches == {'series', 'season', 'episode', 'release_group'}
class ItaSAProvider(Provider): languages = {Language('ita')} video_types = (Episode, ) server_url = 'https://api.italiansubs.net/api/rest/' apikey = 'd86ad6ec041b334fac1e512174ee04d5' def __init__(self, username=None, password=None): if username is not None and password is None or username is None and password is not None: raise ConfigurationError('Username and password must be specified') self.username = username self.password = password self.logged_in = False self.login_itasa = False def initialize(self): self.session = Session() self.session.headers = {'User-Agent': 'Subliminal/%s' % __version__} # login if self.username is not None and self.password is not None: logger.info('Logging in') params = { 'username': self.username, 'password': self.password, 'apikey': self.apikey } r = self.session.get(self.server_url + 'users/login', params=params, allow_redirects=False, timeout=10) root = etree.fromstring(r.content) if root.find('status').text == 'fail': raise AuthenticationError(root.find('error/message').text) # logger.debug('Logged in: \n' + etree.tostring(root)) self.auth_code = root.find('data/user/authcode').text data = { 'username': self.username, 'passwd': self.password, 'remember': 'yes', 'option': 'com_user', 'task': 'login', 'silent': 'true' } r = self.session.post('http://www.italiansubs.net/index.php', data=data, allow_redirects=False, timeout=30) r.raise_for_status() self.logged_in = True def terminate(self): self.session.close() self.logged_in = False @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME) def _get_show_ids(self): """Get the ``dict`` of show ids per series by querying the `shows` page. :return: show id per series, lower case and without quotes. :rtype: dict """ # get the show page logger.info('Getting show ids') params = {'apikey': self.apikey} r = self.session.get(self.server_url + 'shows', timeout=10, params=params) r.raise_for_status() root = etree.fromstring(r.content) # populate the show ids show_ids = {} for show in root.findall('data/shows/show'): if show.find('name').text is None: continue show_ids[sanitize(show.find('name').text).lower()] = int( show.find('id').text) logger.debug('Found %d show ids', len(show_ids)) return show_ids @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME) def _search_show_id(self, series): """Search the show id from the `series` :param str series: series of the episode. :return: the show id, if found. :rtype: int or None """ # build the param params = {'apikey': self.apikey, 'q': series} # make the search logger.info('Searching show ids with %r', params) r = self.session.get(self.server_url + 'shows/search', params=params, timeout=10) r.raise_for_status() root = etree.fromstring(r.content) if int(root.find('data/count').text) == 0: logger.warning('Show id not found: no suggestion') return None # Looking for show in first page for show in root.findall('data/shows/show'): if sanitize(show.find('name').text).lower() == sanitize( series.lower()): show_id = int(show.find('id').text) logger.debug('Found show id %d', show_id) return show_id # Not in the first page of result try next (if any) next = root.find('data/next') while next.text is not None: r = self.session.get(next.text, timeout=10) r.raise_for_status() root = etree.fromstring(r.content) logger.info('Loading suggestion page %s', root.find('data/page').text) # Looking for show in following pages for show in root.findall('data/shows/show'): if sanitize(show.find('name').text).lower() == sanitize( series.lower()): show_id = int(show.find('id').text) logger.debug('Found show id %d', show_id) return show_id next = root.find('data/next') # No matches found logger.warning('Show id not found: suggestions does not match') return None def get_show_id(self, series, country_code=None): """Get the best matching show id for `series`. First search in the result of :meth:`_get_show_ids` and fallback on a search with :meth:`_search_show_id` :param str series: series of the episode. :return: the show id, if found. :rtype: int or None """ series_sanitized = sanitize(series).lower() show_ids = self._get_show_ids() show_id = None # attempt with country if not show_id and country_code: logger.debug('Getting show id with country') show_id = show_ids.get('%s %s' % (series_sanitized, country_code.lower())) # attempt clean if not show_id: logger.debug('Getting show id') show_id = show_ids.get(series_sanitized) # search as last resort if not show_id: logger.warning('Series not found in show ids') show_id = self._search_show_id(series) return show_id def _download_zip(self, sub_id): # download the subtitle logger.info('Downloading subtitle %r', sub_id) params = { 'authcode': self.auth_code, 'apikey': self.apikey, 'subtitle_id': sub_id } r = self.session.get(self.server_url + 'subtitles/download', params=params, timeout=30) r.raise_for_status() return r.content def query(self, series, season, episode, format, country=None): # To make queries you need to be logged in if not self.logged_in: raise ConfigurationError('Cannot query if not logged in') # get the show id show_id = self.get_show_id(series, country) if show_id is None: logger.error('No show id found for %r ', series) return [] # get the page of the season of the show logger.info( 'Getting the subtitle of show id %d, season %d episode %d, format %s', show_id, season, episode, format) subtitles = [] # Default format is HDTV sub_format = '' if format is None or format.lower() == 'hdtv': sub_format = 'normale' else: sub_format = format.lower() params = { 'apikey': self.apikey, 'show_id': show_id, 'q': '%dx%02d' % (season, episode), 'version': sub_format } logger.debug(params) r = self.session.get(self.server_url + 'subtitles/search', params=params, timeout=30) r.raise_for_status() root = etree.fromstring(r.content) if int(root.find('data/count').text) == 0: logger.warning('Subtitles not found') return [] # Looking for subtitlles in first page for subtitle in root.findall('data/subtitles/subtitle'): if '%dx%02d' % (season, episode) in subtitle.find('name').text.lower(): logger.debug('Found subtitle id %d - %s - %s', int(subtitle.find('id').text), subtitle.find('name').text, subtitle.find('version').text) sub = ItaSASubtitle(int(subtitle.find('id').text), subtitle.find('show_name').text, season, episode, format, subtitle.find('name').text) subtitles.append(sub) # Not in the first page of result try next (if any) next = root.find('data/next') while next.text is not None: r = self.session.get(next.text, timeout=30) r.raise_for_status() root = etree.fromstring(r.content) logger.info('Loading subtitles page %s', root.data.page.text) # Looking for show in following pages for subtitle in root.findall('data/subtitles/subtitle'): if '%dx%02d' % (season, episode) in subtitle.find('name').text.lower(): logger.debug('Found subtitle id %d - %s - %s', int(subtitle.find('id').text), subtitle.find('name').text, subtitle.find('version').text) sub = ItaSASubtitle(int(subtitle.find('id').text), subtitle.find('show_name').text, season, episode, format, subtitle.find('name').text) subtitles.append(sub) next = root.find('data/next') # Dowload the subs found, can be more than one in zip additional_subs = [] for sub in subtitles: # open the zip content = self._download_zip(sub.sub_id) if not is_zipfile(io.BytesIO(content)): if 'limite di download' in content: raise TooManyRequests() else: raise ConfigurationError('Not a zip file: %r' % content) with ZipFile(io.BytesIO(content)) as zf: if len(zf.namelist()) > 1: for name in enumerate(zf.namelist()): if name[0] == 0: # First elemnent sub.content = fix_line_ending(zf.read(name[1])) sub.full_data = name[1] else: add_sub = copy.deepcopy(sub) add_sub.content = fix_line_ending(zf.read(name[1])) add_sub.full_data = name[1] additional_subs.append(add_sub) else: sub.content = fix_line_ending(zf.read(zf.namelist()[0])) sub.full_data = zf.namelist()[0] return subtitles + additional_subs def list_subtitles(self, video, languages): return self.query(video.series, video.season, video.episode, video.format) def download_subtitle(self, subtitle): pass
def test_missing(self): with self.assertRaises(ValueError): Language('zzz')
def test_hash(self): self.assertTrue(hash(Language('French')) == hash('fre'))
def __init__(self, directory, filename, logger=None): # Setup logging if logger: log = logger else: log = logging.getLogger(__name__) # Setup encoding to avoid UTF-8 errors if sys.version[0] == '2': SYS_ENCODING = None try: locale.setlocale(locale.LC_ALL, "") SYS_ENCODING = locale.getpreferredencoding() except (locale.Error, IOError): pass # For OSes that are poorly configured just force UTF-8 if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): SYS_ENCODING = 'UTF-8' if not hasattr(sys, "setdefaultencoding"): reload(sys) try: # pylint: disable=E1101 # On non-unicode builds this will raise an AttributeError, if encoding type is not valid it throws a LookupError sys.setdefaultencoding(SYS_ENCODING) except: log.exception("Sorry, your environment is not setup correctly for utf-8 support. Please fix your setup and try again") sys.exit("Sorry, your environment is not setup correctly for utf-8 support. Please fix your setup and try again") log.info(sys.executable) # Default settings for SickBeard sb_defaults = {'host': 'localhost', 'port': '8081', 'ssl': "False", 'api_key': '', 'web_root': '', 'username': '', 'password': ''} ffmpeg = 'ffmpeg' ffprobe = 'ffprobe' if os.name == 'nt': ffmpeg = 'ffmpeg.exe' ffprobe = 'ffprobe.exe' # Default MP4 conversion settings mp4_defaults = {'ffmpeg': ffmpeg, 'ffprobe': ffprobe, 'threads': '0', 'output_directory': '', 'copy_to': '', 'move_to': '', 'output_extension': 'mp4', 'temp_extension': '', 'output_format': 'mp4', 'delete_original': 'True', 'relocate_moov': 'True', 'ios-audio': 'True', 'ios-first-track-only': 'False', 'ios-move-last': 'False', 'ios-audio-filter': '', 'max-audio-channels': '', 'audio-language': '', 'audio-default-language': '', 'audio-codec': 'ac3', 'ignore-trudhd': 'True', 'audio-filter': '', 'audio-channel-bitrate': '256', 'audio-copy-original': 'False', 'audio-first-track-of-language': 'False', 'video-codec': 'h264, x264', 'video-bitrate': '', 'video-crf': '', 'video-max-width': '', 'video-profile': '', 'h264-max-level': '', 'aac_adtstoasc': 'False', 'use-qsv-decoder-with-encoder': 'True', 'use-hevc-qsv-decoder': 'False', 'enable_dxva2_gpu_decode': 'False', 'subtitle-codec': 'mov_text', 'subtitle-language': '', 'subtitle-default-language': '', 'subtitle-encoding': '', 'bad-internal-subtitle-sources': 'pgssub, hdmv_pgs_subtitle, s_hdmv/pgs, dvdsub, dvd_subtitle, dvb_teletext, dvb_subtitle', 'bad-external-subtitle-sources': 'dvdsub, dvd_subtitle, dvb_teletext, dvb_subtitle', 'convert-mp4': 'False', 'force-convert': 'False', 'fullpathguess': 'True', 'tagfile': 'True', 'tag-language': 'en', 'download-artwork': 'poster', 'download-subs': 'False', 'embed-subs': 'True', 'embed-only-internal-subs': 'False', 'sub-providers': 'addic7ed, podnapisi, thesubdb, opensubtitles', 'permissions': '777', 'post-process': 'False', 'pix-fmt': '', 'preopts': '', 'postopts': ''} # Default settings for CouchPotato cp_defaults = {'host': 'localhost', 'port': '5050', 'username': '', 'password': '', 'apikey': '', 'delay': '65', 'method': 'renamer', 'delete_failed': 'False', 'ssl': 'False', 'web_root': ''} # Default settings for Sonarr sonarr_defaults = {'host': 'localhost', 'port': '8989', 'apikey': '', 'ssl': 'False', 'web_root': ''} # Default settings for Radarr radarr_defaults = {'host': 'localhost', 'port': '7878', 'apikey': '', 'ssl': 'False', 'web_root': ''} # Default uTorrent settings utorrent_defaults = {'couchpotato-label': 'couchpotato', 'sickbeard-label': 'sickbeard', 'sickrage-label': 'sickrage', 'sonarr-label': 'sonarr', 'radarr-label': 'radarr', 'bypass-label': 'bypass', 'convert': 'True', 'webui': 'False', 'action_before': 'stop', 'action_after': 'removedata', 'host': 'http://localhost:8080/', 'username': '', 'password': '', 'output_directory': ''} # Default SAB settings sab_defaults = {'convert': 'True', 'Sickbeard-category': 'sickbeard', 'Sickrage-category': 'sickrage', 'Couchpotato-category': 'couchpotato', 'Sonarr-category': 'sonarr', 'Radarr-category': 'radarr', 'Bypass-category': 'bypass', 'output_directory': ''} # Default Sickrage Settings sr_defaults = {'host': 'localhost', 'port': '8081', 'ssl': "False", 'api_key': '', 'web_root': '', 'username': '', 'password': ''} # Default deluge settings deluge_defaults = {'couchpotato-label': 'couchpotato', 'sickbeard-label': 'sickbeard', 'sickrage-label': 'sickrage', 'sonarr-label': 'sonarr', 'radarr-label': 'radarr', 'bypass-label': 'bypass', 'convert': 'True', 'host': 'localhost', 'port': '58846', 'username': '', 'password': '', 'output_directory': '', 'remove': 'false'} # Default QBT settings qbt_defaults = {'couchpotato-label': 'couchpotato', 'sickbeard-label': 'sickbeard', 'sickrage-label': 'sickrage', 'sonarr-label': 'sonarr', 'radarr-label': 'radarr', 'bypass-label': 'bypass', 'convert': 'True', 'action_before': '', 'action_after': '', 'host': 'http://localhost:8080/', 'username': '', 'password': '', 'output_directory': ''} # Default Plex Settings plex_defaults = {'host': 'localhost', 'port': '32400', 'refresh': 'true', 'token': ''} defaults = {'SickBeard': sb_defaults, 'CouchPotato': cp_defaults, 'Sonarr': sonarr_defaults, 'Radarr': radarr_defaults, 'MP4': mp4_defaults, 'uTorrent': utorrent_defaults, 'qBittorrent': qbt_defaults, 'SABNZBD': sab_defaults, 'Sickrage': sr_defaults, 'Deluge': deluge_defaults, 'Plex': plex_defaults} write = False # Will be changed to true if a value is missing from the config file and needs to be written config = configparser.SafeConfigParser() configFile = os.path.join(directory, filename) if os.path.isfile(configFile): config.read(configFile) else: log.error("Config file not found, creating %s." % configFile) # config.filename = filename write = True # Make sure all sections and all keys for each section are present for s in defaults: if not config.has_section(s): config.add_section(s) write = True for k in defaults[s]: if not config.has_option(s, k): config.set(s, k, defaults[s][k]) write = True # If any keys are missing from the config file, write them if write: self.writeConfig(config, configFile) # Read relevant MP4 section information section = "MP4" self.ffmpeg = os.path.normpath(self.raw(config.get(section, "ffmpeg"))) # Location of FFMPEG.exe self.ffprobe = os.path.normpath(self.raw(config.get(section, "ffprobe"))) # Location of FFPROBE.exe self.threads = config.get(section, "threads") # Number of FFMPEG threads try: int(self.threads) except: self.threads = "0" self.output_dir = config.get(section, "output_directory") if self.output_dir == '': self.output_dir = None else: self.output_dir = os.path.normpath(self.raw(self.output_dir)) # Output directory self.copyto = config.get(section, "copy_to") # Directories to make copies of the final product if self.copyto == '': self.copyto = None else: self.copyto = self.copyto.split('|') for i in range(len(self.copyto)): self.copyto[i] = os.path.normpath(self.copyto[i]) if not os.path.isdir(self.copyto[i]): try: os.makedirs(self.copyto[i]) except: log.exception("Error making directory %s." % (self.copyto[i])) self.moveto = config.get(section, "move_to") # Directory to move final product to if self.moveto == '': self.moveto = None else: self.moveto = os.path.normpath(self.moveto) if not os.path.isdir(self.moveto): try: os.makedirs(self.moveto) except: log.exception("Error making directory %s." % (self.moveto)) self.moveto = None self.output_extension = config.get(section, "output_extension") # Output extension self.temp_extension = config.get(section, "temp_extension") # Temporary extension used during processing if self.temp_extension == '': self.temp_extension = None elif self.temp_extension.startswith('.'): self.temp_extension = self.temp_extension[1:] self.output_format = config.get(section, "output_format") # Output format if self.output_format not in valid_formats: self.output_format = 'mov' self.delete = config.getboolean(section, "delete_original") # Delete original file self.relocate_moov = config.getboolean(section, "relocate_moov") # Relocate MOOV atom to start of file self.ignore_truehd = config.getboolean(section, "ignore-trudhd") # Ignore truehd if self.relocate_moov: try: import qtfaststart except: log.error("Please install QTFastStart via PIP, relocate_moov will be disabled without this module.") self.relocate_moov = False self.acodec = config.get(section, "audio-codec").lower() # Gets the desired audio codec, if no valid codec selected, default to AC3 if self.acodec == '': self.acodec == ['ac3'] else: self.acodec = self.acodec.lower().replace(' ', '').split(',') self.abitrate = config.get(section, "audio-channel-bitrate") try: self.abitrate = int(self.abitrate) except: self.abitrate = 256 log.warning("Audio bitrate was invalid, defaulting to 256 per channel.") if self.abitrate > 256: log.warning("Audio bitrate >256 may create errors with common codecs.") self.audio_copyoriginal = config.getboolean(section, "audio-copy-original") # Copies the original audio track regardless of format if a converted track is being generated self.afilter = config.get(section, "audio-filter").lower().strip() # Audio filter if self.afilter == '': self.afilter = None self.audio_first_language_track = config.getboolean(section, "audio-first-track-of-language") # Only take the first audio track in a whitelisted language, then no more self.iOS = config.get(section, "ios-audio") # Creates a second audio channel if the standard output methods are different from this for iOS compatability if self.iOS == "" or self.iOS.lower() in ['false', 'no', 'f', '0']: self.iOS = False else: if self.iOS.lower() in ['true', 'yes', 't', '1']: self.iOS = ['aac'] else: self.iOS = self.iOS.lower().replace(' ', '').split(',') self.iOSFirst = config.getboolean(section, "ios-first-track-only") # Enables the iOS audio option only for the first track self.iOSLast = config.getboolean(section, "ios-move-last") # Moves the iOS audio track to the last in the series of tracks self.iOSfilter = config.get(section, "ios-audio-filter").lower().strip() # iOS audio filter if self.iOSfilter == '': self.iOSfilter = None self.downloadsubs = config.getboolean(section, "download-subs") # Enables downloading of subtitles from the internet sources using subliminal if self.downloadsubs: try: import subliminal except Exception as e: self.downloadsubs = False log.exception("Subliminal is not installed, automatically downloading of subs has been disabled.") self.subproviders = config.get(section, 'sub-providers').lower() if self.subproviders == '': self.downloadsubs = False log.warning("You must specifiy at least one subtitle provider to downlaod subs automatically, subtitle downloading disabled.") else: self.subproviders = self.subproviders.lower().replace(' ', '').split(',') self.embedsubs = config.getboolean(section, 'embed-subs') self.embedonlyinternalsubs = config.getboolean(section, 'embed-only-internal-subs') self.permissions = config.get(section, 'permissions') try: self.permissions = int(self.permissions, 8) except: log.exception("Invalid permissions, defaulting to 777.") self.permissions = int("0777", 8) try: self.postprocess = config.getboolean(section, 'post-process') except: self.postprocess = False self.aac_adtstoasc = config.getboolean(section, 'aac_adtstoasc') # Setup variable for maximum audio channels self.maxchannels = config.get(section, 'max-audio-channels') if self.maxchannels == "": self.maxchannels = None else: try: self.maxchannels = int(self.maxchannels) except: log.exception("Invalid number of audio channels specified.") self.maxchannels = None if self.maxchannels is not None and self.maxchannels < 1: log.warning("Must have at least 1 audio channel.") self.maxchannels = None self.vcodec = config.get(section, "video-codec") if self.vcodec == '': self.vcodec == ['h264', 'x264'] else: self.vcodec = self.vcodec.lower().replace(' ', '').split(',') self.vbitrate = config.get(section, "video-bitrate") if self.vbitrate == '': self.vbitrate = None else: try: self.vbitrate = int(self.vbitrate) if not (self.vbitrate > 0): self.vbitrate = None log.warning("Video bitrate must be greater than 0, defaulting to no video bitrate cap.") except: log.exception("Invalid video bitrate, defaulting to no video bitrate cap.") self.vbitrate = None self.vcrf = config.get(section, "video-crf") if self.vcrf == '': self.vcrf = None else: try: self.vcrf = int(self.vcrf) except: log.exception("Invalid CRF setting, defaulting to none.") self.vcrf = None self.vwidth = config.get(section, "video-max-width") if self.vwidth == '': self.vwidth = None else: try: self.vwidth = int(self.vwidth) except: log.exception("Invalid video width, defaulting to none.") self.vwidth = None self.h264_level = config.get(section, "h264-max-level") if self.h264_level == '': self.h264_level = None else: try: self.h264_level = float(self.h264_level) except: log.exception("Invalid h264 level, defaulting to none.") self.h264_level = None self.vprofile = config.get(section, "video-profile") if self.vprofile == '': self.vprofile = None else: self.vprofile = self.vprofile.lower().strip().replace(' ', '').split(',') self.qsv_decoder = config.getboolean(section, "use-qsv-decoder-with-encoder") # Use Intel QuickSync Decoder when using QuickSync Encoder self.hevc_qsv_decoder = config.getboolean( section, "use-hevc-qsv-decoder") #only supported on 6th gen intel and up. self.dxva2_decoder = config.getboolean( section, "enable_dxva2_gpu_decode" ) self.pix_fmt = config.get(section, "pix-fmt").strip().lower() if self.pix_fmt == '': self.pix_fmt = None else: self.pix_fmt = self.pix_fmt.lower().replace(' ', '').split(',') self.awl = config.get(section, 'audio-language').strip().lower() # List of acceptable languages for audio streams to be carried over from the original file, separated by a comma. Blank for all if self.awl == '': self.awl = None else: self.awl = self.awl.replace(' ', '').split(',') self.scodec = config.get(section, 'subtitle-codec').strip().lower() if not self.scodec or self.scodec == "": if self.embedsubs: self.scodec = ['mov_text'] else: self.scodec = ['srt'] log.warning("Invalid subtitle codec, defaulting to '%s'." % self.scodec) else: self.scodec = self.scodec.replace(' ', '').split(',') if self.embedsubs: if len(self.scodec) > 1: log.warning("Can only embed one subtitle type, defaulting to 'mov_text'.") self.scodec = ['mov_text'] if self.scodec[0] not in valid_internal_subcodecs: log.warning("Invalid interal subtitle codec %s, defaulting to 'mov_text'." % self.scodec[0]) self.scodec = ['mov_text'] else: for codec in self.scodec: if codec not in valid_external_subcodecs: log.warning("Invalid external subtitle codec %s, ignoring." % codec) self.scodec.remove(codec) if len(self.scodec) == 0: log.warning("No valid subtitle formats found, defaulting to 'srt'.") self.scodec = ['srt'] self.swl = config.get(section, 'subtitle-language').strip().lower() # List of acceptable languages for subtitle streams to be carried over from the original file, separated by a comma. Blank for all if self.swl == '': self.swl = None else: self.swl = self.swl.replace(' ', '').split(',') self.subencoding = config.get(section, 'subtitle-encoding').strip().lower() if self.subencoding == '': self.subencoding = None # Bad subtitle codec formats for both internal and external destinations self.bad_internal_subtitle_codecs = config.get(section, 'bad-internal-subtitle-sources').strip().lower().replace(' ', '') if self.bad_internal_subtitle_codecs == '': self.bad_internal_subtitle_codecs = [] else: self.bad_internal_subtitle_codecs = self.bad_internal_subtitle_codecs.split(",") self.bad_external_subtitle_codecs = config.get(section, 'bad-external-subtitle-sources').strip().lower().replace(' ', '') if self.bad_external_subtitle_codecs == '': self.bad_external_subtitle_codecs = [] else: self.bad_external_subtitle_codecs = self.bad_external_subtitle_codecs.split(",") self.adl = config.get(section, 'audio-default-language').strip().lower() # What language to default an undefinied audio language tag to. If blank, it will remain undefined. This is useful for single language releases which tend to leave things tagged as und if self.adl == "" or len(self.adl) > 3: self.adl = None self.sdl = config.get(section, 'subtitle-default-language').strip().lower() # What language to default an undefinied subtitle language tag to. If blank, it will remain undefined. This is useful for single language releases which tend to leave things tagged as und if self.sdl == ""or len(self.sdl) > 3: self.sdl = None # Prevent incompatible combination of settings if self.output_dir == "" and self.delete is False: log.error("You must specify an alternate output directory if you aren't going to delete the original file.") sys.exit() # Create output directory if it does not exist if self.output_dir is not None: if not os.path.isdir(self.output_dir): os.makedirs(self.output_dir) self.processMP4 = config.getboolean(section, "convert-mp4") # Determine whether or not to reprocess mp4 files or just tag them self.forceConvert = config.getboolean(section, "force-convert") # Force conversion even if everything is the same if self.forceConvert: self.processMP4 = True log.warning("Force-convert is true, so convert-mp4 is being overridden to true as well") self.fullpathguess = config.getboolean(section, "fullpathguess") # Guess using the full path or not self.tagfile = config.getboolean(section, "tagfile") # Tag files with metadata self.taglanguage = config.get(section, "tag-language").strip().lower() # Language to tag files if len(self.taglanguage) > 2: try: babel = Language(self.taglanguage) self.taglanguage = babel.alpha2 except: log.exception("Unable to set tag language, defaulting to English.") self.taglanguage = 'en' elif len(self.taglanguage) < 2: log.exception("Unable to set tag language, defaulting to English.") self.taglanguage = 'en' self.artwork = config.get(section, "download-artwork").lower() # Download and embed artwork if self.artwork == "poster": self.artwork = True self.thumbnail = False elif self.artwork == "thumb" or self.artwork == "thumbnail": self.artwork = True self.thumbnail = True else: self.thumbnail = False try: self.artwork = config.getboolean(section, "download-artwork") except: self.artwork = True log.error("Invalid download-artwork value, defaulting to 'poster'.") self.preopts = config.get(section, "preopts") if self.preopts == '': self.preopts = None else: self.preopts = self.preopts.split(',') [o.strip() for o in self.preopts] self.postopts = config.get(section, "postopts") if self.postopts == '': self.postopts = None else: self.postopts = self.postopts.split(',') [o.strip() for o in self.postopts] # Read relevant CouchPotato section information section = "CouchPotato" self.CP = {} self.CP['host'] = config.get(section, "host") self.CP['port'] = config.get(section, "port") self.CP['username'] = config.get(section, "username") self.CP['password'] = config.get(section, "password") self.CP['apikey'] = config.get(section, "apikey") self.CP['delay'] = config.get(section, "delay") self.CP['method'] = config.get(section, "method") self.CP['web_root'] = config.get(section, "web_root") try: self.CP['delay'] = float(self.CP['delay']) except ValueError: self.CP['delay'] = 60 try: self.CP['delete_failed'] = config.getboolean(section, "delete_failed") except (configparser.NoOptionError, ValueError): self.CP['delete_failed'] = False try: if config.getboolean(section, 'ssl'): self.CP['protocol'] = "https://" else: self.CP['protocol'] = "http://" except (configparser.NoOptionError, ValueError): self.CP['protocol'] = "http://" # Read relevant uTorrent section information section = "uTorrent" self.uTorrent = {} self.uTorrent['cp'] = config.get(section, "couchpotato-label").lower() self.uTorrent['sb'] = config.get(section, "sickbeard-label").lower() self.uTorrent['sr'] = config.get(section, "sickrage-label").lower() self.uTorrent['sonarr'] = config.get(section, "sonarr-label").lower() self.uTorrent['radarr'] = config.get(section, "radarr-label").lower() self.uTorrent['bypass'] = config.get(section, "bypass-label").lower() try: self.uTorrent['convert'] = config.getboolean(section, "convert") except: self.uTorrent['convert'] = False self.uTorrent['output_dir'] = config.get(section, "output_directory") if self.uTorrent['output_dir'] == '': self.uTorrent['output_dir'] = None else: self.uTorrent['output_dir'] = os.path.normpath(self.raw(self.uTorrent['output_dir'])) # Output directory self.uTorrentWebUI = config.getboolean(section, "webui") self.uTorrentActionBefore = config.get(section, "action_before").lower() self.uTorrentActionAfter = config.get(section, "action_after").lower() self.uTorrentHost = config.get(section, "host").lower() self.uTorrentUsername = config.get(section, "username") self.uTorrentPassword = config.get(section, "password") # Read relevant qBittorrent section information section = "qBittorrent" self.qBittorrent = {} self.qBittorrent['cp'] = config.get(section, "couchpotato-label").lower() self.qBittorrent['sb'] = config.get(section, "sickbeard-label").lower() self.qBittorrent['sr'] = config.get(section, "sickrage-label").lower() self.qBittorrent['sonarr'] = config.get(section, "sonarr-label").lower() self.qBittorrent['radarr'] = config.get(section, "radarr-label").lower() self.qBittorrent['bypass'] = config.get(section, "bypass-label").lower() try: self.qBittorrent['convert'] = config.getboolean(section, "convert") except: self.qBittorrent['convert'] = False self.qBittorrent['output_dir'] = config.get(section, "output_directory") if self.qBittorrent['output_dir'] == '': self.qBittorrent['output_dir'] = None else: self.qBittorrent['output_dir'] = os.path.normpath(self.raw(self.qBittorrent['output_dir'])) # Output directory self.qBittorrent['actionBefore'] = config.get(section, "action_before").lower() self.qBittorrent['actionAfter'] = config.get(section, "action_after").lower() self.qBittorrent['host'] = config.get(section, "host").lower() self.qBittorrent['username'] = config.get(section, "username") self.qBittorrent['password'] = config.get(section, "password") # Read relevant Deluge section information section = "Deluge" self.deluge = {} self.deluge['cp'] = config.get(section, "couchpotato-label").lower() self.deluge['sb'] = config.get(section, "sickbeard-label").lower() self.deluge['sr'] = config.get(section, "sickrage-label").lower() self.deluge['sonarr'] = config.get(section, "sonarr-label").lower() self.deluge['radarr'] = config.get(section, "radarr-label").lower() self.deluge['bypass'] = config.get(section, "bypass-label").lower() try: self.deluge['convert'] = config.getboolean(section, "convert") except: self.deluge['convert'] = False self.deluge['host'] = config.get(section, "host").lower() self.deluge['port'] = config.get(section, "port") self.deluge['user'] = config.get(section, "username") self.deluge['pass'] = config.get(section, "password") self.deluge['output_dir'] = config.get(section, "output_directory") self.deluge['remove'] = config.getboolean(section, "remove") if self.deluge['output_dir'] == '': self.deluge['output_dir'] = None else: self.deluge['output_dir'] = os.path.normpath(self.raw(self.deluge['output_dir'])) # Output directory # Read relevant Sonarr section information section = "Sonarr" self.Sonarr = {} self.Sonarr['host'] = config.get(section, "host") self.Sonarr['port'] = config.get(section, "port") self.Sonarr['apikey'] = config.get(section, "apikey") self.Sonarr['ssl'] = config.get(section, "ssl") self.Sonarr['web_root'] = config.get(section, "web_root") if not self.Sonarr['web_root'].startswith("/"): self.Sonarr['web_root'] = "/" + self.Sonarr['web_root'] if self.Sonarr['web_root'].endswith("/"): self.Sonarr['web_root'] = self.Sonarr['web_root'][:-1] # Read relevant Radarr section information section = "Radarr" self.Radarr = {} self.Radarr['host'] = config.get(section, "host") self.Radarr['port'] = config.get(section, "port") self.Radarr['apikey'] = config.get(section, "apikey") self.Radarr['ssl'] = config.get(section, "ssl") self.Radarr['web_root'] = config.get(section, "web_root") if not self.Radarr['web_root'].startswith("/"): self.Radarr['web_root'] = "/" + self.Radarr['web_root'] if self.Radarr['web_root'].endswith("/"): self.Radarr['web_root'] = self.Radarr['web_root'][:-1] # Read Sickbeard section information section = "SickBeard" self.Sickbeard = {} self.Sickbeard['host'] = config.get(section, "host") # Server Address self.Sickbeard['port'] = config.get(section, "port") # Server Port self.Sickbeard['api_key'] = config.get(section, "api_key") # Sickbeard API key self.Sickbeard['web_root'] = config.get(section, "web_root") # Sickbeard webroot self.Sickbeard['ssl'] = config.getboolean(section, "ssl") # SSL self.Sickbeard['user'] = config.get(section, "username") self.Sickbeard['pass'] = config.get(section, "password") # Read Sickrage section information section = "Sickrage" self.Sickrage = {} self.Sickrage['host'] = config.get(section, "host") # Server Address self.Sickrage['port'] = config.get(section, "port") # Server Port self.Sickrage['api_key'] = config.get(section, "api_key") # Sickbeard API key self.Sickrage['web_root'] = config.get(section, "web_root") # Sickbeard webroot self.Sickrage['ssl'] = config.getboolean(section, "ssl") # SSL self.Sickrage['user'] = config.get(section, "username") self.Sickrage['pass'] = config.get(section, "password") # Read SAB section information section = "SABNZBD" self.SAB = {} try: self.SAB['convert'] = config.getboolean(section, "convert") # Convert except: self.SAB['convert'] = False self.SAB['cp'] = config.get(section, "Couchpotato-category").lower() self.SAB['sb'] = config.get(section, "Sickbeard-category").lower() self.SAB['sr'] = config.get(section, "Sickrage-category").lower() self.SAB['sonarr'] = config.get(section, "Sonarr-category").lower() self.SAB['radarr'] = config.get(section, "Radarr-category").lower() self.SAB['bypass'] = config.get(section, "Bypass-category").lower() self.SAB['output_dir'] = config.get(section, "output_directory") if self.SAB['output_dir'] == '': self.SAB['output_dir'] = None else: self.SAB['output_dir'] = os.path.normpath(self.raw(self.SAB['output_dir'])) # Output directory # Read Plex section information section = "Plex" self.Plex = {} self.Plex['host'] = config.get(section, "host") self.Plex['port'] = config.get(section, "port") try: self.Plex['refresh'] = config.getboolean(section, "refresh") except: self.Plex['refresh'] = False self.Plex['token'] = config.get(section, "token") if self.Plex['token'] == '': self.Plex['token'] = None # Pass the values on self.config = config self.configFile = configFile
def test_get_subtitle_path_with_language(movies): video = movies['man_of_steel'] assert get_subtitle_path(video.name, Language( 'por', 'BR')) == os.path.splitext(video.name)[0] + '.pt-BR.srt'
def test_get_matches_no_match(episodes): subtitle = TVsubtitlesSubtitle(Language('por'), None, 261077, 'Game of Thrones', 3, 10, 2011, '1080p.BluRay', 'DEMAND') matches = subtitle.get_matches(episodes['bbt_s07e05']) assert matches == set()
def test_get_subtitle_path_with_language_undefined(movies): video = movies['man_of_steel'] assert get_subtitle_path( video.name, Language('und')) == os.path.splitext(video.name)[0] + '.srt'
def generateOptions(self, inputfile, original=None): #Get path information from the input file input_dir, filename, input_extension = self.parseFile(inputfile) info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile) #Video stream print "Video codec detected: " + info.video.codec vcodec = 'copy' if info.video.codec in self.video_codec else self.video_codec[ 0] #Audio streams audio_settings = {} l = 0 for a in info.audio: print "Audio stream detected: " + a.codec + " " + a.language + " [Stream " + str( a.index) + "]" # Set undefined language to default language if specified if self.adl is not None and a.language == 'und': print "Undefined language detected, defaulting to " + self.adl a.language = self.adl # Proceed if no whitelist is set, or if the language is in the whitelist if self.awl is None or a.language in self.awl: # Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo) if self.iOS: if a.audio_channels > 2: print "Creating dual audio channels for iOS compatability for this stream" audio_settings.update({ l: { 'map': a.index, 'codec': self.iOS, 'channels': 2, 'bitrate': 256, 'language': a.language, } }) l += 1 # If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels if self.iOS and a.audio_channels == 2: acodec = 'copy' if a.codec == self.iOS else self.iOS else: # If desired codec is the same as the source codec, copy to avoid quality loss acodec = 'copy' if a.codec in self.audio_codec else self.audio_codec[ 0] # Bitrate calculations/overrides if self.audio_bitrate is None or self.audio_bitrate > ( a.audio_channels * 256): abitrate = 256 * a.audio_channels else: abitrate = self.audio_bitrate audio_settings.update({ l: { 'map': a.index, 'codec': acodec, 'channels': a.audio_channels, 'bitrate': abitrate, 'language': a.language, } }) l = l + 1 # Subtitle streams subtitle_settings = {} l = 0 for s in info.subtitle: print "Subtitle stream detected: " + s.codec + " " + s.language + " [Stream " + str( s.index) + "]" # Make sure its not an image based codec if s.codec not in bad_subtitle_codecs: # Set undefined language to default language if specified if self.sdl is not None and s.language == 'und': s.language = self.sdl # Proceed if no whitelist is set, or if the language is in the whitelist if self.swl is None or s.language in self.swl: subtitle_settings.update({ l: { 'map': s.index, 'codec': 'mov_text', 'language': s.language #'forced': s.sub_forced, #'default': s.sub_default } }) l = l + 1 # External subtitle import # Attempt to download subtitles if they are missing using subliminal languages = set() if self.swl: for alpha3 in self.swl: languages.add(Language(alpha3)) elif self.sdl: languages.add(Language(self.sdl)) else: self.downloadsubs = False if self.downloadsubs: import subliminal try: subliminal.cache_region.configure('dogpile.cache.memory') except: pass try: video = subliminal.scan_video(inputfile, subtitles=True, embedded_subtitles=True, original=original) subtitles = subliminal.download_best_subtitles( [video], languages, hearing_impaired=False, providers=self.subproviders) subliminal.save_subtitles(subtitles) except Exception as e: print e print "Unable to download subtitle" src = 1 # FFMPEG input source number for dirName, subdirList, fileList in os.walk(input_dir): for fname in fileList: subname, subextension = os.path.splitext(fname) # Watch for appropriate file extension if subextension[1:] in valid_subtitle_extensions: x, lang = os.path.splitext(subname) lang = lang[1:] # Using bablefish to convert a 2 language code to a 3 language code if len(lang) is 2: try: babel = Language.fromalpha2(lang) lang = babel.alpha3 except: pass # If subtitle file name and input video name are the same, proceed if x == filename: print "External subtitle file detected, language " + lang if self.swl is None or lang in self.swl: print "Importing %s subtitle stream" % (fname) subtitle_settings.update({ l: { 'path': os.path.join(dirName, fname), 'source': src, 'map': 0, 'codec': 'mov_text', 'language': lang, } }) l = l + 1 src = src + 1 self.deletesubs.add(os.path.join(dirName, fname)) else: print "Ignoring %s external subtitle stream due to language: %s" % ( fname, lang) # Collect all options options = { 'format': self.output_format, 'video': { 'codec': vcodec, 'map': info.video.index, 'bitrate': info.format.bitrate }, 'audio': audio_settings, 'subtitle': subtitle_settings, } self.options = options return options
def test_subtitle_text_no_content(): subtitle = Subtitle(Language('eng')) assert subtitle.text is None
username=config['OPENSUBTITLE']['username'], password=config['OPENSUBTITLE']['password']) # configure the cache region.configure('dogpile.cache.dbm', arguments={'filename': 'cachefile.dbm'}) # configure the path to scan pathToScan = config['DEFAULT']['pathToScan'] # scan for videos newer than 2 weeks and their existing subtitles in a folder videos = scan_videos(pathToScan, age=timedelta(days=30)) logger.info('Analyse video % s ' % (videos)) # Download all shooters shooter_providers = ['shooter'] shooter_subtitles = list_subtitles(videos, {Language('zho')}, providers=shooter_providers) for movie, subtitles in shooter_subtitles.items(): try: download_subtitles(subtitles) for subtitle in subtitles: if subtitle.content is None: logger.error('Skipping subtitle %r: no content' % subtitle) continue # create subtitle path subtitle_path = get_subtitle_path(movie.name, subtitle.language) filename_language, file_extension = os.path.splitext(subtitle_path) filename, language = os.path.splitext(filename_language) subtitle_path = "%s.shooter-%s%s%s" % (
def test_subtitle_is_valid_no_content(): subtitle = Subtitle(Language('fra')) assert subtitle.is_valid() is False
def build_commands(file, new_dir, movie_name, bitbucket): if isinstance(file, string_types): input_file = file if 'concat:' in file: file = file.split('|')[0].replace('concat:', '') video_details, result = get_video_details(file) directory, name = os.path.split(file) name, ext = os.path.splitext(name) check = re.match('VTS_([0-9][0-9])_[0-9]+', name) if check and core.CONCAT: name = movie_name elif check: name = ('{0}.cd{1}'.format(movie_name, check.groups()[0])) elif core.CONCAT and re.match('(.+)[cC][dD][0-9]', name): name = re.sub('([ ._=:-]+[cC][dD][0-9])', '', name) if ext == core.VEXTENSION and new_dir == directory: # we need to change the name to prevent overwriting itself. core.VEXTENSION = '-transcoded{ext}'.format( ext=core.VEXTENSION) # adds '-transcoded.ext' new_file = file else: img, data = next(iteritems(file)) name = data['name'] new_file = [] rem_vid = [] for vid in data['files']: video_details, result = get_video_details(vid, img, bitbucket) if not check_vid_file( video_details, result ): #lets not transcode menu or other clips that don't have audio and video. rem_vid.append(vid) data['files'] = [f for f in data['files'] if f not in rem_vid] new_file = {img: {'name': data['name'], 'files': data['files']}} video_details, result = get_video_details(data['files'][0], img, bitbucket) input_file = '-' file = '-' newfile_path = os.path.normpath( os.path.join(new_dir, name) + core.VEXTENSION) map_cmd = [] video_cmd = [] audio_cmd = [] audio_cmd2 = [] sub_cmd = [] meta_cmd = [] other_cmd = [] if not video_details or not video_details.get( 'streams' ): # we couldn't read streams with ffprobe. Set defaults to try transcoding. video_streams = [] audio_streams = [] sub_streams = [] map_cmd.extend(['-map', '0']) if core.VCODEC: video_cmd.extend(['-c:v', core.VCODEC]) if core.VCODEC == 'libx264' and core.VPRESET: video_cmd.extend(['-pre', core.VPRESET]) else: video_cmd.extend(['-c:v', 'copy']) if core.VFRAMERATE: video_cmd.extend(['-r', str(core.VFRAMERATE)]) if core.VBITRATE: video_cmd.extend(['-b:v', str(core.VBITRATE)]) if core.VRESOLUTION: video_cmd.extend( ['-vf', 'scale={vres}'.format(vres=core.VRESOLUTION)]) if core.VPRESET: video_cmd.extend(['-preset', core.VPRESET]) if core.VCRF: video_cmd.extend(['-crf', str(core.VCRF)]) if core.VLEVEL: video_cmd.extend(['-level', str(core.VLEVEL)]) if core.ACODEC: audio_cmd.extend(['-c:a', core.ACODEC]) if core.ACODEC in [ 'aac', 'dts' ]: # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg audio_cmd.extend(['-strict', '-2']) else: audio_cmd.extend(['-c:a', 'copy']) if core.ACHANNELS: audio_cmd.extend(['-ac', str(core.ACHANNELS)]) if core.ABITRATE: audio_cmd.extend(['-b:a', str(core.ABITRATE)]) if core.OUTPUTQUALITYPERCENT: audio_cmd.extend(['-q:a', str(core.OUTPUTQUALITYPERCENT)]) if core.SCODEC and core.ALLOWSUBS: sub_cmd.extend(['-c:s', core.SCODEC]) elif core.ALLOWSUBS: # Not every subtitle codec can be used for every video container format! sub_cmd.extend(['-c:s', 'copy']) else: # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options sub_cmd.extend(['-sn']) # Don't copy the subtitles over if core.OUTPUTFASTSTART: other_cmd.extend(['-movflags', '+faststart']) else: video_streams = [ item for item in video_details['streams'] if item['codec_type'] == 'video' ] audio_streams = [ item for item in video_details['streams'] if item['codec_type'] == 'audio' ] sub_streams = [ item for item in video_details['streams'] if item['codec_type'] == 'subtitle' ] if core.VEXTENSION not in ['.mkv', '.mpegts']: sub_streams = [ item for item in video_details['streams'] if item['codec_type'] == 'subtitle' and item['codec_name'] != 'hdmv_pgs_subtitle' and item['codec_name'] != 'pgssub' ] for video in video_streams: codec = video['codec_name'] fr = video.get('avg_frame_rate', 0) width = video.get('width', 0) height = video.get('height', 0) scale = core.VRESOLUTION if codec in core.VCODEC_ALLOW or not core.VCODEC: video_cmd.extend(['-c:v', 'copy']) else: video_cmd.extend(['-c:v', core.VCODEC]) if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001): video_cmd.extend(['-r', str(core.VFRAMERATE)]) if scale: w_scale = width / float(scale.split(':')[0]) h_scale = height / float(scale.split(':')[1]) if w_scale > h_scale: # widescreen, Scale by width only. scale = '{width}:{height}'.format( width=scale.split(':')[0], height=int((height / w_scale) / 2) * 2, ) if w_scale > 1: video_cmd.extend( ['-vf', 'scale={width}'.format(width=scale)]) else: # lower or matching ratio, scale by height only. scale = '{width}:{height}'.format( width=int((width / h_scale) / 2) * 2, height=scale.split(':')[1], ) if h_scale > 1: video_cmd.extend( ['-vf', 'scale={height}'.format(height=scale)]) if core.VBITRATE: video_cmd.extend(['-b:v', str(core.VBITRATE)]) if core.VPRESET: video_cmd.extend(['-preset', core.VPRESET]) if core.VCRF: video_cmd.extend(['-crf', str(core.VCRF)]) if core.VLEVEL: video_cmd.extend(['-level', str(core.VLEVEL)]) no_copy = ['-vf', '-r', '-crf', '-level', '-preset', '-b:v'] if video_cmd[1] == 'copy' and any(i in video_cmd for i in no_copy): video_cmd[1] = core.VCODEC if core.VCODEC == 'copy': # force copy. therefore ignore all other video transcoding. video_cmd = ['-c:v', 'copy'] map_cmd.extend(['-map', '0:{index}'.format(index=video['index'])]) break # Only one video needed used_audio = 0 a_mapped = [] commentary = [] if audio_streams: for i, val in reversed(list(enumerate(audio_streams))): try: if 'Commentary' in val.get('tags').get( 'title'): # Split out commentry tracks. commentary.append(val) del audio_streams[i] except Exception: continue try: audio1 = [ item for item in audio_streams if item['tags']['language'] == core.ALANGUAGE ] except Exception: # no language tags. Assume only 1 language. audio1 = audio_streams try: audio2 = [ item for item in audio1 if item['codec_name'] in core.ACODEC_ALLOW ] except Exception: audio2 = [] try: audio3 = [ item for item in audio_streams if item['tags']['language'] != core.ALANGUAGE ] except Exception: audio3 = [] try: audio4 = [ item for item in audio3 if item['codec_name'] in core.ACODEC_ALLOW ] except Exception: audio4 = [] if audio2: # right (or only) language and codec... map_cmd.extend( ['-map', '0:{index}'.format(index=audio2[0]['index'])]) a_mapped.extend([audio2[0]['index']]) bitrate = int(float(audio2[0].get('bit_rate', 0))) / 1000 channels = int(float(audio2[0].get('channels', 0))) audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy']) elif audio1: # right (or only) language, wrong codec. map_cmd.extend( ['-map', '0:{index}'.format(index=audio1[0]['index'])]) a_mapped.extend([audio1[0]['index']]) bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000 channels = int(float(audio1[0].get('channels', 0))) audio_cmd.extend([ '-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy' ]) elif audio4: # wrong language, right codec. map_cmd.extend( ['-map', '0:{index}'.format(index=audio4[0]['index'])]) a_mapped.extend([audio4[0]['index']]) bitrate = int(float(audio4[0].get('bit_rate', 0))) / 1000 channels = int(float(audio4[0].get('channels', 0))) audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy']) elif audio3: # wrong language, wrong codec. just pick the default audio track map_cmd.extend( ['-map', '0:{index}'.format(index=audio3[0]['index'])]) a_mapped.extend([audio3[0]['index']]) bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000 channels = int(float(audio3[0].get('channels', 0))) audio_cmd.extend([ '-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy' ]) if core.ACHANNELS and channels and channels > core.ACHANNELS: audio_cmd.extend( ['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS)]) if audio_cmd[1] == 'copy': audio_cmd[1] = core.ACODEC if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1): audio_cmd.extend( ['-b:a:{0}'.format(used_audio), str(core.ABITRATE)]) if audio_cmd[1] == 'copy': audio_cmd[1] = core.ACODEC if core.OUTPUTQUALITYPERCENT: audio_cmd.extend([ '-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT) ]) if audio_cmd[1] == 'copy': audio_cmd[1] = core.ACODEC if audio_cmd[1] in ['aac', 'dts']: audio_cmd[2:2] = ['-strict', '-2'] if core.ACODEC2_ALLOW: used_audio += 1 try: audio5 = [ item for item in audio1 if item['codec_name'] in core.ACODEC2_ALLOW ] except Exception: audio5 = [] try: audio6 = [ item for item in audio3 if item['codec_name'] in core.ACODEC2_ALLOW ] except Exception: audio6 = [] if audio5: # right language and codec. map_cmd.extend( ['-map', '0:{index}'.format(index=audio5[0]['index'])]) a_mapped.extend([audio5[0]['index']]) bitrate = int(float(audio5[0].get('bit_rate', 0))) / 1000 channels = int(float(audio5[0].get('channels', 0))) audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) elif audio1: # right language wrong codec. map_cmd.extend( ['-map', '0:{index}'.format(index=audio1[0]['index'])]) a_mapped.extend([audio1[0]['index']]) bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000 channels = int(float(audio1[0].get('channels', 0))) if core.ACODEC2: audio_cmd2.extend( ['-c:a:{0}'.format(used_audio), core.ACODEC2]) else: audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) elif audio6: # wrong language, right codec map_cmd.extend( ['-map', '0:{index}'.format(index=audio6[0]['index'])]) a_mapped.extend([audio6[0]['index']]) bitrate = int(float(audio6[0].get('bit_rate', 0))) / 1000 channels = int(float(audio6[0].get('channels', 0))) audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) elif audio3: # wrong language, wrong codec just pick the default audio track map_cmd.extend( ['-map', '0:{index}'.format(index=audio3[0]['index'])]) a_mapped.extend([audio3[0]['index']]) bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000 channels = int(float(audio3[0].get('channels', 0))) if core.ACODEC2: audio_cmd2.extend( ['-c:a:{0}'.format(used_audio), core.ACODEC2]) else: audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy']) if core.ACHANNELS2 and channels and channels > core.ACHANNELS2: audio_cmd2.extend( ['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS2)]) if audio_cmd2[1] == 'copy': audio_cmd2[1] = core.ACODEC2 if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1): audio_cmd2.extend( ['-b:a:{0}'.format(used_audio), str(core.ABITRATE2)]) if audio_cmd2[1] == 'copy': audio_cmd2[1] = core.ACODEC2 if core.OUTPUTQUALITYPERCENT: audio_cmd2.extend([ '-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT) ]) if audio_cmd2[1] == 'copy': audio_cmd2[1] = core.ACODEC2 if audio_cmd2[1] in ['aac', 'dts']: audio_cmd2[2:2] = ['-strict', '-2'] if a_mapped[1] == a_mapped[0] and audio_cmd2[1:] == audio_cmd[ 1:]: # check for duplicate output track. del map_cmd[-2:] else: audio_cmd.extend(audio_cmd2) if core.AINCLUDE and core.ACODEC3: audio_streams.extend(commentary) # add commentry tracks back here. for audio in audio_streams: if audio['index'] in a_mapped: continue used_audio += 1 map_cmd.extend( ['-map', '0:{index}'.format(index=audio['index'])]) audio_cmd3 = [] bitrate = int(float(audio.get('bit_rate', 0))) / 1000 channels = int(float(audio.get('channels', 0))) if audio['codec_name'] in core.ACODEC3_ALLOW: audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy']) else: if core.ACODEC3: audio_cmd3.extend( ['-c:a:{0}'.format(used_audio), core.ACODEC3]) else: audio_cmd3.extend( ['-c:a:{0}'.format(used_audio), 'copy']) if core.ACHANNELS3 and channels and channels > core.ACHANNELS3: audio_cmd3.extend( ['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS3)]) if audio_cmd3[1] == 'copy': audio_cmd3[1] = core.ACODEC3 if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1): audio_cmd3.extend( ['-b:a:{0}'.format(used_audio), str(core.ABITRATE3)]) if audio_cmd3[1] == 'copy': audio_cmd3[1] = core.ACODEC3 if core.OUTPUTQUALITYPERCENT > 0: audio_cmd3.extend([ '-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT) ]) if audio_cmd3[1] == 'copy': audio_cmd3[1] = core.ACODEC3 if audio_cmd3[1] in ['aac', 'dts']: audio_cmd3[2:2] = ['-strict', '-2'] audio_cmd.extend(audio_cmd3) s_mapped = [] burnt = 0 n = 0 for lan in core.SLANGUAGES: try: subs1 = [ item for item in sub_streams if item['tags']['language'] == lan ] except Exception: subs1 = [] if core.BURN and not subs1 and not burnt and os.path.isfile(file): for subfile in get_subs(file): if lan in os.path.split(subfile)[1]: video_cmd.extend( ['-vf', 'subtitles={subs}'.format(subs=subfile)]) burnt = 1 for sub in subs1: if core.BURN and not burnt and os.path.isfile(input_file): subloc = 0 for index in range(len(sub_streams)): if sub_streams[index]['index'] == sub['index']: subloc = index break video_cmd.extend([ '-vf', 'subtitles={sub}:si={loc}'.format(sub=input_file, loc=subloc) ]) burnt = 1 if not core.ALLOWSUBS: break if sub['codec_name'] in [ 'dvd_subtitle', 'VobSub' ] and core.SCODEC == 'mov_text': # We can't convert these. continue map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])]) s_mapped.extend([sub['index']]) if core.SINCLUDE: for sub in sub_streams: if not core.ALLOWSUBS: break if sub['index'] in s_mapped: continue if sub['codec_name'] in [ 'dvd_subtitle', 'VobSub' ] and core.SCODEC == 'mov_text': # We can't convert these. continue map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])]) s_mapped.extend([sub['index']]) if core.OUTPUTFASTSTART: other_cmd.extend(['-movflags', '+faststart']) if core.OTHEROPTS: other_cmd.extend(core.OTHEROPTS) command = [core.FFMPEG, '-loglevel', 'warning'] if core.HWACCEL: command.extend(['-hwaccel', 'auto']) if core.GENERALOPTS: command.extend(core.GENERALOPTS) command.extend(['-i', input_file]) if core.SEMBED and os.path.isfile(file): for subfile in get_subs(file): sub_details, result = get_video_details(subfile) if not sub_details or not sub_details.get('streams'): continue if core.SCODEC == 'mov_text': subcode = [ stream['codec_name'] for stream in sub_details['streams'] ] if set(subcode).intersection(['dvd_subtitle', 'VobSub' ]): # We can't convert these. continue command.extend(['-i', subfile]) lan = os.path.splitext( os.path.splitext(subfile)[0])[1][1:].split('-')[0] lan = text_type(lan) metlan = None try: if len(lan) == 3: metlan = Language(lan) if len(lan) == 2: metlan = Language.fromalpha2(lan) except Exception: pass if metlan: meta_cmd.extend([ '-metadata:s:s:{x}'.format(x=len(s_mapped) + n), 'language={lang}'.format(lang=metlan.alpha3) ]) n += 1 map_cmd.extend(['-map', '{x}:0'.format(x=n)]) if not core.ALLOWSUBS or (not s_mapped and not n): sub_cmd.extend(['-sn']) else: if core.SCODEC: sub_cmd.extend(['-c:s', core.SCODEC]) else: sub_cmd.extend(['-c:s', 'copy']) command.extend(map_cmd) command.extend(video_cmd) command.extend(audio_cmd) command.extend(sub_cmd) command.extend(meta_cmd) command.extend(other_cmd) command.append(newfile_path) if platform.system() != 'Windows': command = core.NICENESS + command return command, new_file
class SubtitulamosProvider(Provider): """Subtitulamos Provider.""" languages = {Language('por', 'BR')} | {Language(l) for l in [ 'cat', 'eng', 'glg', 'por', 'spa' ]} video_types = (Episode,) server_url = 'https://www.subtitulamos.tv/' search_url = server_url + 'search/query' def __init__(self): self.session = None def initialize(self): self.session = Session() self.session.headers['User-Agent'] = 'Subliminal/%s' % __short_version__ # self.session.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:56.0) Gecko/20100101 ' \ # 'Firefox/56.0 ' def terminate(self): self.session.close() @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME) def _search_url_titles(self, series, season, episode, year=None): """Search the URL titles by kind for the given `title`, `season` and `episode`. :param str series: series to search for. :param int season: season to search for. :param int episode: episode to search for. :param int year: year to search for. :return: the episode URL. :rtype: str """ # make the search logger.info('Searching episode url for %s, season %d, episode %d', series, season, episode) episode_url = None search = '{} {}x{}'.format(series, season, episode) r = self.session.get(self.search_url, headers={'Referer': self.server_url}, params={'q': search}, timeout=10) r.raise_for_status() if r.status_code != 200: logger.error('Error getting episode url') raise ProviderError('Error getting episode url') results = json.loads(r.text) for result in results: title = sanitize(result['name']) # attempt series with year if sanitize('{} ({})'.format(series, year)) in title: for episode_data in result['episodes']: if season == episode_data['season'] and episode == episode_data['number']: episode_url = self.server_url + 'episodes/{}'.format(episode_data['id']) return episode_url # attempt series without year elif sanitize(series) in title: for episode_data in result['episodes']: if season == episode_data['season'] and episode == episode_data['number']: episode_url = self.server_url + 'episodes/{}'.format(episode_data['id']) return episode_url return episode_url def query(self, series, season, episode, year=None): # get the episode url episode_url = self._search_url_titles(series, season, episode, year) if episode_url is None: logger.info(f'[{self.provider_name}]: No episode url found for {series}, season {season}, episode {episode}') return [] r = self.session.get(episode_url, headers={'Referer': self.server_url}, timeout=10) r.raise_for_status() soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) # get episode title title_pattern = re.compile('{}(.+){}x{:02d}- (.+)'.format(series, season, episode).lower()) title = title_pattern.search(soup.select('#episode_title')[0].get_text().strip().lower()).group(2) subtitles = [] for sub in soup.find_all('div', attrs={'id': 'progress_buttons_row'}): # read the language language = Language.fromsubtitulamos(sub.find_previous('div', class_='subtitle_language') .get_text().strip()) hearing_impaired = False # modify spanish latino subtitle language to only spanish and set hearing_impaired = True # because if exists spanish and spanish latino subtitle for the same episode, the score will be # higher with spanish subtitle. Spanish subtitle takes priority. if language == Language('spa', 'MX'): language = Language('spa') hearing_impaired = True # read the release subtitle release = sub.find_next('div', class_='version_name').get_text().strip() # ignore incomplete subtitles status = sub.find_next('div', class_='subtitle_buttons').contents[1] if status.name != 'a': logger.debug('Ignoring subtitle in [%s] not finished', language) continue # read the subtitle url subtitle_url = self.server_url + status['href'][1:] subtitle = SubtitulamosSubtitle(language, hearing_impaired, episode_url, series, season, episode, title, year, release, subtitle_url) logger.debug('Found subtitle %r', subtitle) subtitles.append(subtitle) return subtitles def list_subtitles(self, video: Episode, languages): return [s for s in self.query(video.series, video.season, video.episode, video.year) if s.language in languages] def download_subtitle(self, subtitle: SubtitulamosSubtitle): # download the subtitle logger.info('Downloading subtitle %s', subtitle.download_link) r = self.session.get(subtitle.download_link, headers={'Referer': subtitle.page_link}, timeout=10) r.raise_for_status() subtitle.content = fix_line_ending(r.content)
class PodnapisiProvider(Provider): languages = ({Language('por', 'BR'), Language('srp', script='Latn')} | {Language.fromalpha2(l) for l in language_converters['alpha2'].codes}) server_url = 'http://podnapisi.eu/subtitles/' def initialize(self): self.session = Session() self.session.headers = {'User-Agent': 'Subliminal/%s' % get_version(__version__)} def terminate(self): self.session.close() def query(self, language, keyword, season=None, episode=None, year=None): # set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652 params = {'sXML': 1, 'sL': str(language), 'sK': keyword} is_episode = False if season and episode: is_episode = True params['sTS'] = season params['sTE'] = episode if year: params['sY'] = year # loop over paginated results logger.info('Searching subtitles %r', params) subtitles = [] pids = set() while True: # query the server xml = etree.fromstring(self.session.get(self.server_url + 'search/old', params=params, timeout=10).content) # exit if no results if not int(xml.find('pagination/results').text): logger.debug('No subtitles found') break # loop over subtitles for subtitle_xml in xml.findall('subtitle'): # read xml elements language = Language.fromietf(subtitle_xml.find('language').text) hearing_impaired = 'n' in (subtitle_xml.find('flags').text or '') page_link = subtitle_xml.find('url').text pid = subtitle_xml.find('pid').text releases = [] if subtitle_xml.find('release').text: for release in subtitle_xml.find('release').text.split(): release = re.sub(r'\.+$', '', release) # remove trailing dots release = ''.join(filter(lambda x: ord(x) < 128, release)) # remove non-ascii characters releases.append(release) title = subtitle_xml.find('title').text season = int(subtitle_xml.find('tvSeason').text) episode = int(subtitle_xml.find('tvEpisode').text) year = int(subtitle_xml.find('year').text) if is_episode: subtitle = PodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title, season=season, episode=episode, year=year) else: subtitle = PodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title, year=year) # ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321 if pid in pids: continue logger.debug('Found subtitle %r', subtitle) subtitles.append(subtitle) pids.add(pid) # stop on last page if int(xml.find('pagination/current').text) >= int(xml.find('pagination/count').text): break # increment current page params['page'] = int(xml.find('pagination/current').text) + 1 logger.debug('Getting page %d', params['page']) return subtitles def list_subtitles(self, video, languages): if isinstance(video, Episode): return [s for l in languages for s in self.query(l, video.series, season=video.season, episode=video.episode, year=video.year)] elif isinstance(video, Movie): return [s for l in languages for s in self.query(l, video.title, year=video.year)] def download_subtitle(self, subtitle): # download as a zip logger.info('Downloading subtitle %r', subtitle) r = self.session.get(self.server_url + subtitle.pid + '/download', params={'container': 'zip'}, timeout=10) r.raise_for_status() # open the zip with ZipFile(io.BytesIO(r.content)) as zf: if len(zf.namelist()) > 1: raise ProviderError('More than one file to unzip') subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))
def test_get_matches_only_year_country(episodes): subtitle = TVsubtitlesSubtitle(Language('por'), None, 261077, 'Game of Thrones', 3, 10, None, '1080p.BluRay', 'DEMAND') matches = subtitle.get_matches(episodes['bbt_s07e05']) assert matches == {'year', 'country'}
def from_code(language): language = language.strip() if language and language in language_converters['opensubtitles'].codes: return Language.fromopensubtitles(language) return Language('und')
class TVsubtitlesProvider(Provider): languages = {Language('por', 'BR')} | {Language(l) for l in [ 'ara', 'bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'fin', 'fra', 'hun', 'ita', 'jpn', 'kor', 'nld', 'pol', 'por', 'ron', 'rus', 'spa', 'swe', 'tur', 'ukr', 'zho' ]} video_types = (Episode,) server_url = 'http://www.tvsubtitles.net/' def initialize(self): self.session = Session() self.session.headers = {'User-Agent': 'Subliminal/%s' % get_version(__version__)} def terminate(self): self.session.close() @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME) def search_show_id(self, series, year=None): """Search the show id from the `series` and `year`. :param string series: series of the episode. :param year: year of the series, if any. :type year: int or None :return: the show id, if any. :rtype: int or None """ # make the search logger.info('Searching show id for %r', series) r = self.session.post(self.server_url + 'search.php', data={'q': series}, timeout=10) r.raise_for_status() # get the series out of the suggestions soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) show_id = None for suggestion in soup.select('div.left li div a[href^="/tvshow-"]'): match = link_re.match(suggestion.text) if not match: logger.error('Failed to match %s', suggestion.text) continue if match.group('series').lower() == series.lower(): if year is not None and int(match.group('first_year')) != year: logger.debug('Year does not match') continue show_id = int(suggestion['href'][8:-5]) logger.debug('Found show id %d', show_id) break return show_id @region.cache_on_arguments(expiration_time=EPISODE_EXPIRATION_TIME) def get_episode_ids(self, show_id, season): """Get episode ids from the show id and the season. :param int show_id: show id. :param int season: season of the episode. :return: episode ids per episode number. :rtype: dict """ # get the page of the season of the show logger.info('Getting the page of show id %d, season %d', show_id, season) r = self.session.get(self.server_url + 'tvshow-%d-%d.html' % (show_id, season), timeout=10) soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) # loop over episode rows episode_ids = {} for row in soup.select('table#table5 tr'): # skip rows that do not have a link to the episode page if not row('a', href=episode_id_re): continue # extract data from the cells cells = row('td') episode = int(cells[0].text.split('x')[1]) episode_id = int(cells[1].a['href'][8:-5]) episode_ids[episode] = episode_id if episode_ids: logger.debug('Found episode ids %r', episode_ids) else: logger.warning('No episode ids found') return episode_ids def query(self, series, season, episode, year=None): # search the show id show_id = self.search_show_id(series, year) if show_id is None: logger.error('No show id found for %r (%r)', series, {'year': year}) return [] # get the episode ids episode_ids = self.get_episode_ids(show_id, season) if episode not in episode_ids: logger.error('Episode %d not found', episode) return [] # get the episode page logger.info('Getting the page for episode %d', episode_ids[episode]) r = self.session.get(self.server_url + 'episode-%d.html' % episode_ids[episode], timeout=10) soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) # loop over subtitles rows subtitles = [] for row in soup.select('.subtitlen'): # read the item language = Language.fromtvsubtitles(row.h5.img['src'][13:-4]) subtitle_id = int(row.parent['href'][10:-5]) page_link = self.server_url + 'subtitle-%d.html' % subtitle_id rip = row.find('p', title='rip').text.strip() or None release = row.find('p', title='release').text.strip() or None subtitle = TVsubtitlesSubtitle(language, page_link, subtitle_id, series, season, episode, year, rip, release) logger.debug('Found subtitle %s', subtitle) subtitles.append(subtitle) return subtitles def list_subtitles(self, video, languages): return [s for s in self.query(video.series, video.season, video.episode, video.year) if s.language in languages] def download_subtitle(self, subtitle): # download as a zip logger.info('Downloading subtitle %r', subtitle) r = self.session.get(self.server_url + 'download-%d.html' % subtitle.subtitle_id, timeout=10) r.raise_for_status() # open the zip with ZipFile(io.BytesIO(r.content)) as zf: if len(zf.namelist()) > 1: raise ProviderError('More than one file to unzip') subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))
class Addic7edProvider(Provider): """Addic7ed Provider.""" languages = {Language('por', 'BR')} | { Language(l) for l in [ 'ara', 'aze', 'ben', 'bos', 'bul', 'cat', 'ces', 'dan', 'deu', 'ell', 'eng', 'eus', 'fas', 'fin', 'fra', 'glg', 'heb', 'hrv', 'hun', 'hye', 'ind', 'ita', 'jpn', 'kor', 'mkd', 'msa', 'nld', 'nor', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi', 'srp', 'swe', 'tha', 'tur', 'ukr', 'vie', 'zho' ] } video_types = (Episode, ) server_url = 'http://www.addic7ed.com/' subtitle_class = Addic7edSubtitle def __init__(self, username=None, password=None): if any((username, password)) and not all((username, password)): raise ConfigurationError('Username and password must be specified') self.username = username self.password = password self.logged_in = False self.session = None def initialize(self): self.session = Session() self.session.headers[ 'User-Agent'] = 'Subliminal/%s' % __short_version__ # login if self.username and self.password: logger.info('Logging in') data = { 'username': self.username, 'password': self.password, 'Submit': 'Log in' } r = self.session.post(self.server_url + 'dologin.php', data, allow_redirects=False, timeout=10) if r.status_code != 302: raise AuthenticationError(self.username) logger.debug('Logged in') self.logged_in = True def terminate(self): # logout if self.logged_in: logger.info('Logging out') r = self.session.get(self.server_url + 'logout.php', timeout=10) r.raise_for_status() logger.debug('Logged out') self.logged_in = False self.session.close() @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME) def _get_show_ids(self): """Get the ``dict`` of show ids per series by querying the `shows.php` page. :return: show id per series, lower case and without quotes. :rtype: dict """ # get the show page logger.info('Getting show ids') r = self.session.get(self.server_url + 'shows.php', timeout=10) r.raise_for_status() # LXML parser seems to fail when parsing Addic7ed.com HTML markup. # Last known version to work properly is 3.6.4 (next version, 3.7.0, fails) # Assuming the site's markup is bad, and stripping it down to only contain what's needed. show_cells = re.findall(show_cells_re, r.content) if show_cells: soup = ParserBeautifulSoup(b''.join(show_cells), ['lxml', 'html.parser']) else: # If RegEx fails, fall back to original r.content and use 'html.parser' soup = ParserBeautifulSoup(r.content, ['html.parser']) # populate the show ids show_ids = {} for show in soup.select('td.version > h3 > a[href^="/show/"]'): show_ids[sanitize(show.text)] = int(show['href'][6:]) logger.debug('Found %d show ids', len(show_ids)) return show_ids @region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME) def _search_show_id(self, series, year=None): """Search the show id from the `series` and `year`. :param str series: series of the episode. :param year: year of the series, if any. :type year: int :return: the show id, if found. :rtype: int """ # addic7ed doesn't support search with quotes series = series.replace('\'', ' ') # build the params series_year = '%s %d' % (series, year) if year is not None else series params = {'search': series_year, 'Submit': 'Search'} # make the search logger.info('Searching show ids with %r', params) r = self.session.get(self.server_url + 'srch.php', params=params, timeout=10) r.raise_for_status() soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) # get the suggestion suggestion = soup.select('span.titulo > a[href^="/show/"]') if not suggestion: logger.warning('Show id not found: no suggestion') return None if not sanitize(suggestion[0].i.text.replace( '\'', ' ')) == sanitize(series_year): logger.warning('Show id not found: suggestion does not match') return None show_id = int(suggestion[0]['href'][6:]) logger.debug('Found show id %d', show_id) return show_id def get_show_id(self, series, year=None, country_code=None): """Get the best matching show id for `series`, `year` and `country_code`. First search in the result of :meth:`_get_show_ids` and fallback on a search with :meth:`_search_show_id`. :param str series: series of the episode. :param year: year of the series, if any. :type year: int :param country_code: country code of the series, if any. :type country_code: str :return: the show id, if found. :rtype: int """ series_sanitized = sanitize(series).lower() show_ids = self._get_show_ids() show_id = None # attempt with country if not show_id and country_code: logger.debug('Getting show id with country') show_id = show_ids.get('%s %s' % (series_sanitized, country_code.lower())) # attempt with year if not show_id and year: logger.debug('Getting show id with year') show_id = show_ids.get('%s %d' % (series_sanitized, year)) # attempt clean if not show_id: logger.debug('Getting show id') show_id = show_ids.get(series_sanitized) # search as last resort if not show_id: logger.warning('Series %s not found in show ids', series) show_id = self._search_show_id(series) return show_id def query(self, show_id, series, season, year=None, country=None): # get the page of the season of the show logger.info('Getting the page of show id %d, season %d', show_id, season) r = self.session.get(self.server_url + 'show/%d' % show_id, params={'season': season}, timeout=10) r.raise_for_status() if not r.content: # Provider returns a status of 304 Not Modified with an empty content # raise_for_status won't raise exception for that status code logger.debug('No data returned from provider') return [] soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) # loop over subtitle rows match = series_year_re.match( soup.select('#header font')[0].text.strip()[:-10]) series = match.group('series') year = int(match.group('year')) if match.group('year') else None subtitles = [] for row in soup.select('tr.epeven'): cells = row('td') # ignore incomplete subtitles status = cells[5].text if status != 'Completed': logger.debug('Ignoring subtitle with status %s', status) continue # read the item language = Language.fromaddic7ed(cells[3].text) hearing_impaired = bool(cells[6].text) page_link = self.server_url + cells[2].a['href'][1:] season = int(cells[0].text) episode = int(cells[1].text) title = cells[2].text version = cells[4].text download_link = cells[9].a['href'][1:] subtitle = self.subtitle_class(language, hearing_impaired, page_link, series, season, episode, title, year, version, download_link) logger.debug('Found subtitle %r', subtitle) subtitles.append(subtitle) return subtitles def list_subtitles(self, video, languages): # lookup show_id titles = [video.series] + video.alternative_series show_id = None for title in titles: show_id = self.get_show_id(title, video.year) if show_id is not None: break # query for subtitles with the show_id if show_id is not None: subtitles = [ s for s in self.query(show_id, title, video.season, video.year) if s.language in languages and s.episode == video.episode ] if subtitles: return subtitles else: logger.error('No show id found for %r (%r)', video.series, {'year': video.year}) return [] def download_subtitle(self, subtitle): # download the subtitle logger.info('Downloading subtitle %r', subtitle) r = self.session.get(self.server_url + subtitle.download_link, headers={'Referer': subtitle.page_link}, timeout=10) r.raise_for_status() if not r.content: # Provider returns a status of 304 Not Modified with an empty content # raise_for_status won't raise exception for that status code logger.debug( 'Unable to download subtitle. No data returned from provider') return # detect download limit exceeded if r.headers['Content-Type'] == 'text/html': raise DownloadLimitExceeded subtitle.content = fix_line_ending(r.content)
def test_eq_with_country(self): self.assertTrue( Language('Portuguese (BR)') == Language('Portuguese - Brazil')) self.assertTrue(Language('English') == Language('en'))