def download_callback(self, menuitem, files): # scan videos videos = [] for f in files: # ignore non-writable locations if not f.can_write(): continue # directories if f.is_directory(): try: scanned_videos = scan_videos(f.get_location().get_path()) except: continue for video in scanned_videos: if check_video(video, languages=self.config.languages, age=self.config.age, undefined=self.config.single): video.subtitle_languages |= set(search_external_subtitles(video.name).values()) refine(video, episode_refiners=self.config.refiners, movie_refiners=self.config.refiners, embedded_subtitles=self.config.embedded_subtitles) videos.append(video) continue # other inputs try: video = scan_video(f.get_location().get_path()) except: continue if check_video(video, languages=self.config.languages, undefined=self.config.single): video.subtitle_languages |= set(search_external_subtitles(video.name).values()) refine(video, episode_refiners=self.config.refiners, movie_refiners=self.config.refiners, embedded_subtitles=self.config.embedded_subtitles) videos.append(video) # download best subtitles downloaded_subtitles = defaultdict(list) with AsyncProviderPool(providers=self.config.providers, provider_configs=self.config.provider_configs) as pool: for v in videos: scores = get_scores(v) subtitles = pool.download_best_subtitles( pool.list_subtitles(v, self.config.languages - v.subtitle_languages), v, self.config.languages, min_score=scores['hash'] * self.config.min_score / 100, hearing_impaired=self.config.hearing_impaired, only_one=self.config.single ) downloaded_subtitles[v] = subtitles # save subtitles for v, subtitles in downloaded_subtitles.items(): save_subtitles(v, subtitles, single=self.config.single)
def get_subtitles(self, entry): if (entry.get('subtitles', eval_lazy=False) or not ('location' in entry) or ('$RECYCLE.BIN' in entry['location']) or not os.path.exists(entry['location'])): return from subliminal import scan_video from subliminal.core import search_external_subtitles, refine try: video = scan_video(entry['location']) # grab external and internal subtitles subtitles = video.subtitle_languages refiner = ('metadata', ) refine(video, episode_refiners=refiner, movie_refiners=refiner) subtitles |= set( search_external_subtitles(entry['location']).values()) if subtitles: # convert to human-readable strings subtitles = [str(l) for l in subtitles] entry['subtitles'] = subtitles log.debug('Found subtitles %s for %s', '/'.join(subtitles), entry['title']) except Exception as e: log.error('Error checking local subtitles for %s: %s', entry['title'], e)
def subtitle(self, episodes): # Parse babelfish languages bb_lang = {Language.fromietf(l) for l in self.languages} # Create subliminal episode set sub_episodes = set() for episode in episodes: ep_path = os.path.join(episode['dir'], episode['filename']) sub_episode = Episode.fromguess(ep_path, episode) # Look for external subtitles (not done automatically, apparently) sub_episode.subtitle_languages |= set(search_external_subtitles(sub_episode.name).values()) sub_episodes.add(sub_episode) # download subtitles in the specified language subl_subtitles = download_best_subtitles(sub_episodes, bb_lang, providers=self.providers) for video, subtitles in subl_subtitles.items(): save_subtitles(video, subtitles) # save subtitle languages in episode dict
def get_subtitles(self, entry): if ( entry.get('subtitles', eval_lazy=False) or not ('location' in entry) or ('$RECYCLE.BIN' in entry['location']) or not os.path.exists(entry['location']) ): return from subliminal import scan_video from subliminal.core import search_external_subtitles, refine try: video = scan_video(entry['location']) # grab external and internal subtitles subtitles = video.subtitle_languages refiner = ('metadata',) refine(video, episode_refiners=refiner, movie_refiners=refiner) subtitles |= set(search_external_subtitles(entry['location']).values()) if subtitles: # convert to human-readable strings subtitles = [str(l) for l in subtitles] entry['subtitles'] = subtitles log.debug('Found subtitles %s for %s', '/'.join(subtitles), entry['title']) except Exception as e: log.error('Error checking local subtitles for %s: %s', entry['title'], e)
def get_video(tv_episode, video_path, subtitles_dir=None, subtitles=True, embedded_subtitles=None, release_name=None): """Return the subliminal video for the given path. The video_path is used as a key to cache the video to avoid scanning and parsing the video metadata all the time :param tv_episode: :type tv_episode: medusa.tv.Episode :param video_path: the video path :type video_path: str :param subtitles_dir: the subtitles directory :type subtitles_dir: str or None :param subtitles: True if existing external subtitles should be taken into account :type subtitles: bool or None :param embedded_subtitles: True if embedded subtitles should be taken into account :type embedded_subtitles: bool or None :param release_name: the release name :type release_name: str or None :return: video :rtype: subliminal.video.Video """ key = video_key.format(video_path=video_path) payload = {'subtitles_dir': subtitles_dir, 'subtitles': subtitles, 'embedded_subtitles': embedded_subtitles, 'release_name': release_name} cached_payload = memory_cache.get(key, expiration_time=VIDEO_EXPIRATION_TIME) if cached_payload != NO_VALUE and {k: v for k, v in iteritems(cached_payload) if k != 'video'} == payload: logger.debug(u'Found cached video information under key %s', key) return cached_payload['video'] video_is_mkv = video_path.endswith('.mkv') subtitles_dir = subtitles_dir or get_subtitles_dir(video_path) logger.debug(u'Scanning video %s...', video_path) try: video = scan_video(video_path) except ValueError as error: logger.warning(u'Unable to scan video: %s. Error: %r', video_path, error) else: if subtitles: video.subtitle_languages |= set(search_external_subtitles(video_path, directory=subtitles_dir).values()) if embedded_subtitles is None: embedded_subtitles = bool(not app.IGNORE_EMBEDDED_SUBS and video_is_mkv) refine(video, episode_refiners=episode_refiners, embedded_subtitles=embedded_subtitles, release_name=release_name, tv_episode=tv_episode) video.alternative_series = [alias.title for alias in tv_episode.series.aliases] payload['video'] = video memory_cache.set(key, payload) logger.debug(u'Video information cached under key %s', key) return video
def get_subtitles(self, entry): if entry.get('subtitles', eval_lazy=False) or not ('location' in entry) or \ ('$RECYCLE.BIN' in entry['location']) or not os.path.exists(entry['location']): return from subliminal.core import search_external_subtitles try: subtitles = list(search_external_subtitles(entry['location']).values()) if subtitles: entry['subtitles'] = subtitles log.debug('Found subtitles %s for %s', '/'.join(subtitles), entry['title']) except Exception as e: log.debug('Error checking local subtitles for %s: %s' % (entry['title'], e))
def get_video(tv_episode, video_path, subtitles_dir=None, subtitles=True, embedded_subtitles=None, release_name=None): """Return the subliminal video for the given path. The video_path is used as a key to cache the video to avoid scanning and parsing the video metadata all the time :param tv_episode: :type tv_episode: sickbeard.tv.TVEpisode :param video_path: the video path :type video_path: str :param subtitles_dir: the subtitles directory :type subtitles_dir: str or None :param subtitles: True if existing external subtitles should be taken into account :type subtitles: bool or None :param embedded_subtitles: True if embedded subtitles should be taken into account :type embedded_subtitles: bool or None :param release_name: the release name :type release_name: str or None :return: video :rtype: subliminal.video.Video """ key = video_key.format(video_path=video_path) payload = {'subtitles_dir': subtitles_dir, 'subtitles': subtitles, 'embedded_subtitles': embedded_subtitles, 'release_name': release_name} cached_payload = region.get(key, expiration_time=VIDEO_EXPIRATION_TIME) if cached_payload != NO_VALUE and {k: v for k, v in iteritems(cached_payload) if k != 'video'} == payload: logger.debug(u'Found cached video information under key %s', key) return cached_payload['video'] try: video_path = _encode(video_path) subtitles_dir = _encode(subtitles_dir or get_subtitles_dir(video_path)) logger.debug(u'Scanning video %s...', video_path) video = scan_video(video_path) # external subtitles if subtitles: video.subtitle_languages |= set(search_external_subtitles(video_path, directory=subtitles_dir).values()) if embedded_subtitles is None: embedded_subtitles = bool(not sickbeard.EMBEDDED_SUBTITLES_ALL and video_path.endswith('.mkv')) refine(video, episode_refiners=episode_refiners, embedded_subtitles=embedded_subtitles, release_name=release_name, tv_episode=tv_episode) payload['video'] = video region.set(key, payload) logger.debug(u'Video information cached under key %s', key) return video except Exception as error: logger.info(u'Exception: %s', error)
def test_search_external_subtitles_no_directory(movies, tmpdir, monkeypatch): video_name = os.path.split(movies['man_of_steel'].name)[1] video_root = os.path.splitext(video_name)[0] tmpdir.ensure(video_name) monkeypatch.chdir(str(tmpdir)) expected_subtitles = { video_name + '.srt': Language('und'), video_root + '.en.srt': Language('eng') } for path in expected_subtitles: tmpdir.ensure(path) subtitles = search_external_subtitles(video_name) assert subtitles == expected_subtitles
def test_search_external_subtitles_in_directory(episodes, tmpdir): video_name = episodes['marvels_agents_of_shield_s02e06'].name video_root = os.path.splitext(video_name)[0] tmpdir.ensure('tvshows', video_name) subtitles_directory = str(tmpdir.ensure('subtitles', dir=True)) expected_subtitles = { video_name + '.srt': Language('und'), video_root + '.en.srt': Language('eng') } tmpdir.ensure('tvshows', video_name + '.fr.srt') for path in expected_subtitles: tmpdir.ensure('subtitles', path) subtitles = search_external_subtitles(video_name, directory=subtitles_directory) assert subtitles == expected_subtitles
def run(self, scan_path, scan_age, languages, *args, **kwargs): if not os.path.isdir(scan_path): raise IOError('Path \'%s\' doesn\'t exist!' % scan_path) if not scan_age >= 1: raise ValueError('\'scan_age\' must by at least 1!') if not len(languages) >= 1: raise ValueError('\'languages\' list can\'t be empty!') __tree_dict = lambda: defaultdict(__tree_dict) result = __tree_dict() age = timedelta(weeks=scan_age) languages = set([Language(l) for l in languages]) scan_start = datetime.now() videos = [] ignored_videos = [] if not region.is_configured: region.configure('dogpile.cache.dbm', expiration_time=timedelta(days=30), arguments={'filename': 'subliminal.dbm', 'lock_factory': MutexLock}) # scan videos scanned_videos = scan_videos(scan_path, age=age) for video in scanned_videos: video.subtitle_languages |= set(search_external_subtitles(video.name).values()) if check_video(video, languages=languages, age=age, undefined=False): refine(video) if languages - video.subtitle_languages: videos.append(video) else: ignored_videos.append(video) else: ignored_videos.append(video) if videos: result['videos']['collected'] = [os.path.split(v.name)[1] for v in videos] if ignored_videos: result['videos']['ignored'] = [os.path.split(v.name)[1] for v in ignored_videos] scan_end = datetime.now() result['meta']['start'] = scan_start.isoformat() result['meta']['end'] = scan_end.isoformat() result['meta']['duration'] = str(scan_end - scan_start) return result
def test_search_external_subtitles_archive(movies, tmpdir): video_name = os.path.split(movies['interstellar'].name)[1] video_root = os.path.splitext(video_name)[0] video_path = str(tmpdir.ensure(video_name)) expected_subtitles = { video_name + '.srt': Language('und'), video_root + '.srt': Language('und'), video_root + '.en.srt': Language('eng'), video_name + '.fra.srt': Language('fra'), video_root + '.pt-BR.srt': Language('por', 'BR'), video_name + '.sr_cyrl.sub': Language('srp', script='Cyrl'), video_name + '.something.srt': Language('und') } tmpdir.ensure(os.path.split(movies['interstellar'].name)[1] + '.srt') for path in expected_subtitles: tmpdir.ensure(path) subtitles = search_external_subtitles(video_path) assert subtitles == expected_subtitles
def get_video(video_path, subtitles_path=None): if not subtitles_path: subtitles_path = get_subtitles_path(video_path) try: # Encode paths to UTF-8 to ensure subliminal support. video_path = video_path.encode('utf-8') subtitles_path = subtitles_path.encode('utf-8') except UnicodeEncodeError: # Fallback to system encoding. This should never happen. video_path = video_path.encode(sickbeard.SYS_ENCODING) subtitles_path = subtitles_path.encode(sickbeard.SYS_ENCODING) try: video = scan_video(video_path) video.subtitle_languages |= set(search_external_subtitles(video_path, directory=subtitles_path).values()) if not sickbeard.EMBEDDED_SUBTITLES_ALL and video_path.endswith('.mkv'): refine(video, embedded_subtitles=True) except Exception as error: logger.log(u'Exception: {}'.format(error), logger.DEBUG) return None return video
def download(obj, provider, language, age, directory, encoding, single, force, hearing_impaired, min_score, max_workers, archives, verbose, path): """Download best subtitles. PATH can be an directory containing videos, a video file path or a video file name. It can be used multiple times. If an existing subtitle is detected (external or embedded) in the correct language, the download is skipped for the associated video. """ # process parameters language = set(language) # scan videos videos = [] ignored_videos = [] errored_paths = [] with click.progressbar(path, label='Collecting videos', item_show_func=lambda p: p or '') as bar: for p in bar: logger.debug('Collecting path %s', p) # non-existing if not os.path.exists(p): try: video = Video.fromname(p) except: logger.exception('Unexpected error while collecting non-existing path %s', p) errored_paths.append(p) continue if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values()) refine(video, embedded_subtitles=not force) videos.append(video) continue # directories if os.path.isdir(p): try: scanned_videos = scan_videos(p, age=age, archives=archives, subtitles=not force, subtitles_dir=directory) except: logger.exception('Unexpected error while collecting directory path %s', p) errored_paths.append(p) continue for video in scanned_videos: if check_video(video, languages=language, age=age, undefined=single): if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values()) refine(video, embedded_subtitles=not force) videos.append(video) else: ignored_videos.append(video) continue # other inputs try: video = scan_video(p) except: logger.exception('Unexpected error while collecting path %s', p) errored_paths.append(p) continue if check_video(video, languages=language, age=age, undefined=single): if not force: video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values()) refine(video, embedded_subtitles=not force) videos.append(video) else: ignored_videos.append(video) # output errored paths if verbose > 0: for p in errored_paths: click.secho('%s errored' % p, fg='red') # output ignored videos if verbose > 1: for video in ignored_videos: click.secho('%s ignored - subtitles: %s / age: %d day%s' % ( os.path.split(video.name)[1], ', '.join(str(s) for s in video.subtitle_languages) or 'none', video.age.days, 's' if video.age.days > 1 else '' ), fg='yellow') # report collected videos click.echo('%s video%s collected / %s video%s ignored / %s error%s' % ( click.style(str(len(videos)), bold=True, fg='green' if videos else None), 's' if len(videos) > 1 else '', click.style(str(len(ignored_videos)), bold=True, fg='yellow' if ignored_videos else None), 's' if len(ignored_videos) > 1 else '', click.style(str(len(errored_paths)), bold=True, fg='red' if errored_paths else None), 's' if len(errored_paths) > 1 else '', )) # exit if no video collected if not videos: return # download best subtitles downloaded_subtitles = defaultdict(list) with AsyncProviderPool(max_workers=max_workers, providers=provider, provider_configs=obj['provider_configs']) as p: with click.progressbar(videos, label='Downloading subtitles', item_show_func=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar: for v in bar: scores = get_scores(v) subtitles = p.download_best_subtitles(p.list_subtitles(v, language - v.subtitle_languages), v, language, min_score=scores['hash'] * min_score / 100, hearing_impaired=hearing_impaired, only_one=single) downloaded_subtitles[v] = subtitles # TODO: warn about discarded providers # save subtitles total_subtitles = 0 for v, subtitles in downloaded_subtitles.items(): saved_subtitles = save_subtitles(v, subtitles, single=single, directory=directory, encoding=encoding) total_subtitles += len(saved_subtitles) if verbose > 0: click.echo('%s subtitle%s downloaded for %s' % (click.style(str(len(saved_subtitles)), bold=True), 's' if len(saved_subtitles) > 1 else '', os.path.split(v.name)[1])) if verbose > 1: for s in saved_subtitles: matches = s.get_matches(v) score = compute_score(s, v) # score color score_color = None scores = get_scores(v) if isinstance(v, Movie): if score < scores['title']: score_color = 'red' elif score < scores['title'] + scores['year'] + scores['release_group']: score_color = 'yellow' else: score_color = 'green' elif isinstance(v, Episode): if score < scores['series'] + scores['season'] + scores['episode']: score_color = 'red' elif score < scores['series'] + scores['season'] + scores['episode'] + scores['release_group']: score_color = 'yellow' else: score_color = 'green' # scale score from 0 to 100 taking out preferences scaled_score = score if s.hearing_impaired == hearing_impaired: scaled_score -= scores['hearing_impaired'] scaled_score *= 100 / scores['hash'] # echo some nice colored output click.echo(' - [{score}] {language} subtitle from {provider_name} (match on {matches})'.format( score=click.style('{:5.1f}'.format(scaled_score), fg=score_color, bold=score >= scores['hash']), language=s.language.name if s.language.country is None else '%s (%s)' % (s.language.name, s.language.country.name), provider_name=s.provider_name, matches=', '.join(sorted(matches, key=scores.get, reverse=True)) )) if verbose == 0: click.echo('Downloaded %s subtitle%s' % (click.style(str(total_subtitles), bold=True), 's' if total_subtitles > 1 else ''))
def download_callback(self, menuitem, files): # scan videos videos = [] for f in files: # ignore non-writable locations if not f.can_write(): continue # directories if f.is_directory(): try: scanned_videos = scan_videos(f.get_location().get_path()) except: continue for video in scanned_videos: if check_video(video, languages=self.config.languages, age=self.config.age, undefined=self.config.single): video.subtitle_languages |= set( search_external_subtitles(video.name).values()) refine( video, episode_refiners=self.config.refiners, movie_refiners=self.config.refiners, embedded_subtitles=self.config.embedded_subtitles) videos.append(video) continue # other inputs try: video = scan_video(f.get_location().get_path()) except: continue if check_video(video, languages=self.config.languages, undefined=self.config.single): video.subtitle_languages |= set( search_external_subtitles(video.name).values()) refine(video, episode_refiners=self.config.refiners, movie_refiners=self.config.refiners, embedded_subtitles=self.config.embedded_subtitles) videos.append(video) # download best subtitles downloaded_subtitles = defaultdict(list) with AsyncProviderPool( providers=self.config.providers, provider_configs=self.config.provider_configs) as pool: for v in videos: scores = get_scores(v) subtitles = pool.download_best_subtitles( pool.list_subtitles( v, self.config.languages - v.subtitle_languages), v, self.config.languages, min_score=scores['hash'] * self.config.min_score / 100, hearing_impaired=self.config.hearing_impaired, only_one=self.config.single) downloaded_subtitles[v] = subtitles # save subtitles for v, subtitles in downloaded_subtitles.items(): save_subtitles(v, subtitles, single=self.config.single)
def emit(self, task, config): if not config: return entries = [] with Session() as session: for sub_item in queue_get(session=session): if os.path.exists(sub_item.path): path = sub_item.path elif sub_item.alternate_path and os.path.exists( sub_item.alternate_path): path = sub_item.alternate_path elif not config['remove_not_found'] and \ sub_item.added + parse_timedelta('24 hours') > datetime.combine(date.today(), time()): log.warning( 'File %s was not found. Deleting after %s.' % (sub_item.path, str(sub_item.added + parse_timedelta('24 hours')))) continue else: log.error('File not found. Removing "%s" from queue.' % sub_item.title) session.delete(sub_item) continue if os.path.isdir(path): paths = os.listdir(path) if not paths: log.warning('Queued folder %s is empty.' % path) continue path_dir = path else: paths = [path] path_dir = os.path.dirname(path) primary = set() for language in sub_item.languages: primary.add(Language.fromietf(language.language)) for file in paths: entry = Entry() if not file.lower().endswith(VIDEO_EXTENSIONS): continue file = normalize_path(os.path.join(path_dir, file)) entry['url'] = urljoin('file:', request.pathname2url(file)) entry['location'] = file entry['title'] = os.path.splitext( os.path.basename(file))[0] # filename without ext entry['subtitle_languages'] = primary try: from subliminal.core import search_external_subtitles try: existing_subtitles = set( search_external_subtitles( normalize_path(file)).values()) if primary and not primary - existing_subtitles: log.debug( 'All subtitles already fetched for %s.' % entry['title']) sub_item.downloaded = True continue except ValueError as e: log.error( 'Invalid video file: %s. Removing %s from queue.' % (e, entry['title'])) session.delete(sub_item) continue except ImportError: log.debug( 'Falling back to simple check since Subliminal is not installed.' ) # use glob since subliminal is not there path_no_ext = os.path.splitext(normalize_path(file))[0] # can only check subtitles that have explicit language codes in the file name if primary: files = glob.glob(path_no_ext + "*") files = [item.lower() for item in files] for lang in primary: if not any('%s.%s' % (path_no_ext, lang) and f.lower( ).endswith(SUBTITLE_EXTENSIONS) for f in files): break else: log.debug( 'All subtitles already fetched for %s.' % entry['title']) sub_item.downloaded = True continue entry.on_complete(self.complete, path=path, task=task) entries.append(entry) log.debug('Emitting entry for %s.' % entry['title']) return entries
def get_video(tv_episode, video_path, subtitles_dir=None, subtitles=True, embedded_subtitles=None, release_name=None): """Return the subliminal video for the given path. The video_path is used as a key to cache the video to avoid scanning and parsing the video metadata all the time :param tv_episode: :type tv_episode: medusa.tv.TVEpisode :param video_path: the video path :type video_path: str :param subtitles_dir: the subtitles directory :type subtitles_dir: str or None :param subtitles: True if existing external subtitles should be taken into account :type subtitles: bool or None :param embedded_subtitles: True if embedded subtitles should be taken into account :type embedded_subtitles: bool or None :param release_name: the release name :type release_name: str or None :return: video :rtype: subliminal.video.Video """ key = video_key.format(video_path=video_path) payload = { 'subtitles_dir': subtitles_dir, 'subtitles': subtitles, 'embedded_subtitles': embedded_subtitles, 'release_name': release_name } cached_payload = region.get(key, expiration_time=VIDEO_EXPIRATION_TIME) if cached_payload != NO_VALUE and { k: v for k, v in iteritems(cached_payload) if k != 'video' } == payload: logger.debug(u'Found cached video information under key %s', key) return cached_payload['video'] try: video_path = _encode(video_path) subtitles_dir = _encode(subtitles_dir or get_subtitles_dir(video_path)) logger.debug(u'Scanning video %s...', video_path) video = scan_video(video_path) # external subtitles if subtitles: video.subtitle_languages |= set( search_external_subtitles(video_path, directory=subtitles_dir).values()) if embedded_subtitles is None: embedded_subtitles = bool(not app.EMBEDDED_SUBTITLES_ALL and video_path.endswith('.mkv')) refine(video, episode_refiners=episode_refiners, embedded_subtitles=embedded_subtitles, release_name=release_name, tv_episode=tv_episode) payload['video'] = video region.set(key, payload) logger.debug(u'Video information cached under key %s', key) return video except Exception as error: logger.info(u'Exception: %s', error)
def download(obj, provider, refiner, language, age, directory, encoding, single, force, hearing_impaired, min_score, max_workers, archives, verbose, path): """Download best subtitles. PATH can be an directory containing videos, a video file path or a video file name. It can be used multiple times. If an existing subtitle is detected (external or embedded) in the correct language, the download is skipped for the associated video. """ # process parameters language = set(language) # scan videos videos = [] ignored_videos = [] errored_paths = [] with click.progressbar(path, label='Collecting videos', item_show_func=lambda p: p or '') as bar: for p in bar: logger.debug('Collecting path %s', p) # non-existing if not os.path.exists(p): try: video = Video.fromname(p) except: logger.exception( 'Unexpected error while collecting non-existing path %s', p) errored_paths.append(p) continue if not force: video.subtitle_languages |= set( search_external_subtitles( video.name, directory=directory).values()) refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) videos.append(video) continue # directories if os.path.isdir(p): try: scanned_videos = scan_videos(p, age=age, archives=archives) except: logger.exception( 'Unexpected error while collecting directory path %s', p) errored_paths.append(p) continue for video in scanned_videos: if check_video(video, languages=language, age=age, undefined=single): if not force: video.subtitle_languages |= set( search_external_subtitles( video.name, directory=directory).values()) refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) videos.append(video) else: ignored_videos.append(video) continue # other inputs try: video = scan_video(p) except: logger.exception('Unexpected error while collecting path %s', p) errored_paths.append(p) continue if check_video(video, languages=language, age=age, undefined=single): if not force: video.subtitle_languages |= set( search_external_subtitles( video.name, directory=directory).values()) refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force) videos.append(video) else: ignored_videos.append(video) # output errored paths if verbose > 0: for p in errored_paths: click.secho('%s errored' % p, fg='red') # output ignored videos if verbose > 1: for video in ignored_videos: click.secho( '%s ignored - subtitles: %s / age: %d day%s' % (os.path.split(video.name)[1], ', '.join(str(s) for s in video.subtitle_languages) or 'none', video.age.days, 's' if video.age.days > 1 else ''), fg='yellow') # report collected videos click.echo('%s video%s collected / %s video%s ignored / %s error%s' % ( click.style( str(len(videos)), bold=True, fg='green' if videos else None), 's' if len(videos) > 1 else '', click.style(str(len(ignored_videos)), bold=True, fg='yellow' if ignored_videos else None), 's' if len(ignored_videos) > 1 else '', click.style(str(len(errored_paths)), bold=True, fg='red' if errored_paths else None), 's' if len(errored_paths) > 1 else '', )) # exit if no video collected if not videos: return # download best subtitles downloaded_subtitles = defaultdict(list) with AsyncProviderPool(max_workers=max_workers, providers=provider, provider_configs=obj['provider_configs']) as p: with click.progressbar( videos, label='Downloading subtitles', item_show_func=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar: for v in bar: scores = get_scores(v) subtitles = p.download_best_subtitles( p.list_subtitles(v, language - v.subtitle_languages), v, language, min_score=scores['hash'] * min_score / 100, hearing_impaired=hearing_impaired, only_one=single) downloaded_subtitles[v] = subtitles if p.discarded_providers: click.secho( 'Some providers have been discarded due to unexpected errors: %s' % ', '.join(p.discarded_providers), fg='yellow') # save subtitles total_subtitles = 0 for v, subtitles in downloaded_subtitles.items(): saved_subtitles = save_subtitles(v, subtitles, single=single, directory=directory, encoding=encoding) total_subtitles += len(saved_subtitles) if verbose > 0: click.echo( '%s subtitle%s downloaded for %s' % (click.style(str(len(saved_subtitles)), bold=True), 's' if len(saved_subtitles) > 1 else '', os.path.split(v.name)[1])) if verbose > 1: for s in saved_subtitles: matches = s.get_matches(v) score = compute_score(s, v) # score color score_color = None scores = get_scores(v) if isinstance(v, Movie): if score < scores['title']: score_color = 'red' elif score < scores['title'] + scores['year'] + scores[ 'release_group']: score_color = 'yellow' else: score_color = 'green' elif isinstance(v, Episode): if score < scores['series'] + scores['season'] + scores[ 'episode']: score_color = 'red' elif score < scores['series'] + scores['season'] + scores[ 'episode'] + scores['release_group']: score_color = 'yellow' else: score_color = 'green' # scale score from 0 to 100 taking out preferences scaled_score = score if s.hearing_impaired == hearing_impaired: scaled_score -= scores['hearing_impaired'] scaled_score *= 100 / scores['hash'] # echo some nice colored output click.echo( ' - [{score}] {language} subtitle from {provider_name} (match on {matches})' .format(score=click.style('{:5.1f}'.format(scaled_score), fg=score_color, bold=score >= scores['hash']), language=s.language.name if s.language.country is None else '%s (%s)' % (s.language.name, s.language.country.name), provider_name=s.provider_name, matches=', '.join( sorted(matches, key=scores.get, reverse=True)))) if verbose == 0: click.echo('Downloaded %s subtitle%s' % (click.style(str(total_subtitles), bold=True), 's' if total_subtitles > 1 else ''))
def on_task_output(self, task, config): """ Configuration:: subliminal: languages: List of languages (as IETF codes) in order of preference. At least one is required. alternatives: List of second-choice languages; subs will be downloaded but entries rejected. exact_match: Use file hash only to search for subs, otherwise Subliminal will try to guess by filename. providers: List of providers from where to download subtitles. single: Download subtitles in single mode (no language code added to subtitle filename). directory: Path to directory where to save the subtitles, default is next to the video. hearing_impaired: Prefer subtitles for the hearing impaired when available authentication: > Dictionary of configuration options for different providers. Keys correspond to provider names, and values are dictionaries, usually specifying `username` and `password`. """ if not task.accepted: logger.debug('nothing accepted, aborting') return import subliminal from babelfish import Language from dogpile.cache.exception import RegionAlreadyConfigured from subliminal import save_subtitles, scan_video from subliminal.cli import MutexLock from subliminal.core import ( ARCHIVE_EXTENSIONS, refine, scan_archive, search_external_subtitles, ) from subliminal.score import episode_scores, movie_scores from subliminal.video import VIDEO_EXTENSIONS try: subliminal.region.configure( 'dogpile.cache.dbm', arguments={ 'filename': os.path.join(tempfile.gettempdir(), 'cachefile.dbm'), 'lock_factory': MutexLock, }, ) except RegionAlreadyConfigured: pass # Let subliminal be more verbose if our logger is set to DEBUG if logger.level(task.manager.options.loglevel).no <= logger.level('DEBUG').no: logging.getLogger("subliminal").setLevel(logging.INFO) else: logging.getLogger("subliminal").setLevel(logging.CRITICAL) logging.getLogger("dogpile").setLevel(logging.CRITICAL) logging.getLogger("enzyme").setLevel(logging.WARNING) try: languages = set([Language.fromietf(s) for s in config.get('languages', [])]) alternative_languages = set( [Language.fromietf(s) for s in config.get('alternatives', [])] ) except ValueError as e: raise plugin.PluginError(e) # keep all downloaded subtitles and save to disk when done (no need to write every time) downloaded_subtitles = collections.defaultdict(list) providers_list = config.get('providers', None) provider_configs = config.get('authentication', None) # test if only one language was provided, if so we will download in single mode # (aka no language code added to subtitle filename) # unless we are forced not to by configuration # if we pass 'yes' for single in configuration but choose more than one language # we ignore the configuration and add the language code to the # potentially downloaded files single_mode = config.get('single', '') and len(languages | alternative_languages) <= 1 hearing_impaired = config.get('hearing_impaired', False) with subliminal.core.ProviderPool( providers=providers_list, provider_configs=provider_configs ) as provider_pool: for entry in task.accepted: if 'location' not in entry: logger.warning('Cannot act on entries that do not represent a local file.') continue if not os.path.exists(entry['location']): entry.fail('file not found: %s' % entry['location']) continue if '$RECYCLE.BIN' in entry['location']: # ignore deleted files in Windows shares continue try: entry_languages = set(entry.get('subtitle_languages', [])) or languages if entry['location'].endswith(VIDEO_EXTENSIONS): video = scan_video(entry['location']) elif entry['location'].endswith(ARCHIVE_EXTENSIONS): video = scan_archive(entry['location']) else: entry.reject( 'File extension is not a supported video or archive extension' ) continue # use metadata refiner to get mkv metadata refiner = ('metadata',) refine(video, episode_refiners=refiner, movie_refiners=refiner) existing_subtitles = set(search_external_subtitles(entry['location']).values()) video.subtitle_languages |= existing_subtitles if isinstance(video, subliminal.Episode): title = video.series hash_scores = episode_scores['hash'] else: title = video.title hash_scores = movie_scores['hash'] logger.info('Name computed for {} was {}', entry['location'], title) msc = hash_scores if config['exact_match'] else 0 if entry_languages.issubset(video.subtitle_languages): logger.debug( 'All preferred languages already exist for "{}"', entry['title'] ) entry['subtitles_missing'] = set() continue # subs for preferred lang(s) already exists else: # Gather the subtitles for the alternative languages too, to avoid needing to search the sites # again. They'll just be ignored if the main languages are found. all_subtitles = provider_pool.list_subtitles( video, entry_languages | alternative_languages ) try: subtitles = provider_pool.download_best_subtitles( all_subtitles, video, entry_languages, min_score=msc, hearing_impaired=hearing_impaired, ) except TypeError as e: logger.error( 'Downloading subtitles failed due to a bug in subliminal. Please seehttps://github.com/Diaoul/subliminal/issues/921. Error: {}', e, ) subtitles = [] if subtitles: downloaded_subtitles[video].extend(subtitles) logger.info('Subtitles found for {}', entry['location']) else: # only try to download for alternatives that aren't already downloaded subtitles = provider_pool.download_best_subtitles( all_subtitles, video, alternative_languages, min_score=msc, hearing_impaired=hearing_impaired, ) if subtitles: downloaded_subtitles[video].extend(subtitles) entry.reject('subtitles found for a second-choice language.') else: entry.reject('cannot find any subtitles for now.') downloaded_languages = set( [Language.fromietf(str(l.language)) for l in subtitles] ) if entry_languages: entry['subtitles_missing'] = entry_languages - downloaded_languages if len(entry['subtitles_missing']) > 0: entry.reject('Subtitles for all primary languages not found') except ValueError as e: logger.error('subliminal error: {}', e) entry.fail() if downloaded_subtitles: if task.options.test: logger.verbose('Test mode. Found subtitles:') # save subtitles to disk for video, subtitle in downloaded_subtitles.items(): if subtitle: _directory = config.get('directory') if _directory: _directory = os.path.expanduser(_directory) if task.options.test: logger.verbose( ' FOUND LANGUAGES {} for {}', [str(l.language) for l in subtitle], video.name, ) continue save_subtitles(video, subtitle, single=single_mode, directory=_directory)
def store_subtitles(file): # languages = [] actual_subtitles = [] if os.path.exists(file): if os.path.splitext(file)[1] == '.mkv': try: with open(file, 'rb') as f: mkv = enzyme.MKV(f) for subtitle_track in mkv.subtitle_tracks: try: if alpha2_from_alpha3(subtitle_track.language) != None: actual_subtitles.append([ str(alpha2_from_alpha3( subtitle_track.language)), None ]) except: pass except: pass brazilian_portuguese = [".pt-br", ".pob", "pb"] try: subtitles = core.search_external_subtitles(file) except: pass else: for subtitle, language in subtitles.iteritems(): if str(os.path.splitext(subtitle)[0]).lower().endswith( tuple(brazilian_portuguese)) is True: actual_subtitles.append([ str("pb"), path_replace_reverse( os.path.join(os.path.dirname(file), subtitle)) ]) elif str(language) != 'und': actual_subtitles.append([ str(language), path_replace_reverse( os.path.join(os.path.dirname(file), subtitle)) ]) else: with open( path_replace( os.path.join(os.path.dirname(file), subtitle)), 'r') as f: text = list(islice(f, 100)) text = ' '.join(text) encoding = UnicodeDammit(text) try: text = text.decode(encoding.original_encoding) except Exception as e: logging.exception( 'Error trying to detect character encoding for this subtitles file: ' + path_replace( os.path.join(os.path.dirname(file), subtitle)) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again.' ) else: detected_language = langdetect.detect(text) if len(detected_language) > 0: actual_subtitles.append([ str(detected_language), path_replace_reverse( os.path.join(os.path.dirname(file), subtitle)) ]) conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) c_db = conn_db.cursor() c_db.execute( "UPDATE table_episodes SET subtitles = ? WHERE path = ?", (str(actual_subtitles), path_replace_reverse(file))) conn_db.commit() c_db.close() return actual_subtitles
def run(self, scan_path, scan_age, languages, encoding, min_score, providers, provider_configs, max_workers, plex_url=None, plex_token=None, *args, **kwargs): if not os.path.isdir(scan_path): raise IOError('Path \'%s\' doesn\'t exist!' % scan_path) if not scan_age >= 1: raise ValueError('\'scan_age\' must by at least 1!') if not len(languages) >= 1: raise ValueError('\'languages\' list can\'t be empty!') if not providers: raise ValueError('\'providers\' argument can\'t be empty!') if not max_workers >= 1: raise ValueError('\'max_workers\' must be at least 1!') if not provider_configs: provider_configs = {} __tree_dict = lambda: defaultdict(__tree_dict) result = __tree_dict() encoding = codecs.lookup(encoding).name age = timedelta(weeks=scan_age) languages = set([Language(l) for l in languages]) plex = None if plex_url and plex_token: plex = PlexServer(plex_url, plex_token) scan_start = datetime.now() videos = [] ignored_videos = [] if not region.is_configured: region.configure('dogpile.cache.dbm', expiration_time=timedelta(days=30), arguments={ 'filename': 'subliminal.dbm', 'lock_factory': MutexLock }) # scan videos scanned_videos = scan_videos(scan_path, age=age) for video in scanned_videos: video.subtitle_languages |= set( search_external_subtitles(video.name).values()) if check_video(video, languages=languages, age=age, undefined=False): refine(video) if languages - video.subtitle_languages: videos.append(video) else: ignored_videos.append(video) else: ignored_videos.append(video) if videos: result['videos']['collected'] = [ os.path.split(v.name)[1] for v in videos ] if ignored_videos: result['videos']['ignored'] = len(ignored_videos) if videos: # download best subtitles downloaded_subtitles = defaultdict(list) with AsyncProviderPool(max_workers=max_workers, providers=providers, provider_configs=provider_configs) as p: for video in videos: scores = get_scores(video) subtitles_to_download = p.list_subtitles( video, languages - video.subtitle_languages) downloaded_subtitles[video] = p.download_best_subtitles( subtitles_to_download, video, languages, min_score=scores['hash'] * min_score / 100) if p.discarded_providers: result['providers']['discarded'] = list( p.discarded_providers) # filter subtitles with TinyDB('subtitle_db.json') as db: table = db.table('downloaded') query = Query() for video, subtitles in downloaded_subtitles.items(): discarded_subtitles = list() discarded_subtitles_info = list() for s in subtitles: subtitle_hash = hashlib.sha256(s.content).hexdigest() subtitle_file = get_subtitle_path( os.path.split(video.name)[1], s.language) dbo = {'hash': subtitle_hash, 'file': subtitle_file} if table.search((query.hash == subtitle_hash) & (query.file == subtitle_file)): discarded_subtitles.append(s) discarded_subtitles_info.append(dbo) else: table.insert(dbo) downloaded_subtitles[video] = [ x for x in subtitles if x not in discarded_subtitles ] if discarded_subtitles_info: result['subtitles'][ 'discarded'] = result['subtitles'].get( 'discarded', []) + discarded_subtitles_info downloaded_subtitles = { k: v for k, v in downloaded_subtitles.items() if v } # save subtitles saved_subtitles = {} for video, subtitles in downloaded_subtitles.items(): saved_subtitles[video] = save_subtitles(video, subtitles, directory=None, encoding=encoding) for key, group in groupby(saved_subtitles[video], lambda x: x.provider_name): subtitle_filenames = [ get_subtitle_path( os.path.split(video.name)[1], s.language) for s in list(group) ] result['subtitles'][key] = result['subtitles'].get( key, []) + subtitle_filenames result['subtitles']['total'] = sum( len(v) for v in saved_subtitles.values()) # refresh plex for video, subtitles in saved_subtitles.items(): if plex and subtitles: item_found = False for section in plex.library.sections(): try: if isinstance(section, MovieSection) and isinstance( video, Movie): results = section.search(title=video.title, year=video.year, libtype='movie', sort='addedAt:desc', maxresults=1) if not results: raise NotFound plex_item = results[0] elif isinstance(section, ShowSection) and isinstance( video, Episode): results = section.search(title=video.series, year=video.year, libtype='show', sort='addedAt:desc', maxresults=1) if not results: raise NotFound plex_item = results[0].episode( season=video.season, episode=video.episode) else: continue except NotFound: continue except BadRequest: continue if plex_item: plex_item.refresh() result['plex']['refreshed'] = result['plex'].get( 'refreshed', []) + [ '%s%s' % (repr(plex_item.section()), repr(video)) ] item_found = True if not item_found: result['plex']['failed'] = result['plex'].get( 'failed', []) + [repr(video)] # convert subtitles for video, subtitles in saved_subtitles.items(): target_format = aeidon.formats.SUBRIP for s in subtitles: subtitle_path = get_subtitle_path(video.name, s.language) source_format = aeidon.util.detect_format( subtitle_path, encoding) source_file = aeidon.files.new( source_format, subtitle_path, aeidon.encodings.detect_bom(subtitle_path) or encoding) if source_format != target_format: format_info = { 'file': get_subtitle_path( os.path.split(video.name)[1], s.language), 'from': source_format.label, 'to': target_format.label } result['subtitles'][ 'converted'] = result['subtitles'].get( 'converted', []) + [format_info] aeidon_subtitles = source_file.read() for f in [ aeidon.formats.SUBRIP, aeidon.formats.MICRODVD, aeidon.formats.MPL2 ]: markup = aeidon.markups.new(f) for s in aeidon_subtitles: s.main_text = markup.decode(s.main_text) markup = aeidon.markups.new(target_format) for s in aeidon_subtitles: s.main_text = markup.encode(s.main_text) target_file = aeidon.files.new(target_format, subtitle_path, encoding) target_file.write(aeidon_subtitles, aeidon.documents.MAIN) scan_end = datetime.now() result['meta']['start'] = scan_start.isoformat() result['meta']['end'] = scan_end.isoformat() result['meta']['duration'] = str(scan_end - scan_start) return result
def emit(self, task, config): if not config: return entries = [] with Session() as session: for sub_item in queue_get(session=session): if os.path.exists(sub_item.path): path = sub_item.path elif sub_item.alternate_path and os.path.exists(sub_item.alternate_path): path = sub_item.alternate_path elif not config['remove_not_found'] and \ sub_item.added + parse_timedelta('24 hours') > datetime.combine(date.today(), time()): log.warning('File %s was not found. Deleting after %s.' % (sub_item.path, str(sub_item.added + parse_timedelta('24 hours')))) continue else: log.error('File not found. Removing "%s" from queue.' % sub_item.title) session.delete(sub_item) continue if os.path.isdir(path): paths = os.listdir(path) if not paths: log.warning('Queued folder %s is empty.' % path) continue path_dir = path else: paths = [path] path_dir = os.path.dirname(path) primary = set() for language in sub_item.languages: primary.add(Language.fromietf(language.language)) for file in paths: entry = Entry() if not file.lower().endswith(VIDEO_EXTENSIONS): continue file = normalize_path(os.path.join(path_dir, file)) entry['url'] = urljoin('file:', request.pathname2url(file)) entry['location'] = file entry['title'] = os.path.splitext(os.path.basename(file))[0] # filename without ext entry['subtitle_languages'] = primary try: from subliminal.core import search_external_subtitles try: existing_subtitles = set(search_external_subtitles(normalize_path(file)).values()) if primary and not primary - existing_subtitles: log.debug('All subtitles already fetched for %s.' % entry['title']) sub_item.downloaded = True continue except ValueError as e: log.error('Invalid video file: %s. Removing %s from queue.' % (e, entry['title'])) session.delete(sub_item) continue except ImportError: log.debug('Falling back to simple check since Subliminal is not installed.') # use glob since subliminal is not there path_no_ext = os.path.splitext(normalize_path(file))[0] # can only check subtitles that have explicit language codes in the file name if primary: files = glob.glob(path_no_ext + "*") files = [item.lower() for item in files] for lang in primary: if not any('%s.%s' % (path_no_ext, lang) and f.lower().endswith(SUBTITLE_EXTENSIONS) for f in files): break else: log.debug('All subtitles already fetched for %s.' % entry['title']) sub_item.downloaded = True continue entry.on_complete(self.complete, path=path, task=task) entries.append(entry) log.debug('Emitting entry for %s.' % entry['title']) return entries
def on_task_output(self, task, config): """ Configuration:: subliminal: languages: List of languages (as IETF codes) in order of preference. At least one is required. alternatives: List of second-choice languages; subs will be downloaded but entries rejected. exact_match: Use file hash only to search for subs, otherwise Subliminal will try to guess by filename. providers: List of providers from where to download subtitles. single: Download subtitles in single mode (no language code added to subtitle filename). directory: Path to directory where to save the subtitles, default is next to the video. hearing_impaired: Prefer subtitles for the hearing impaired when available authentication: > Dictionary of configuration options for different providers. Keys correspond to provider names, and values are dictionaries, usually specifying `username` and `password`. """ if not task.accepted: log.debug('nothing accepted, aborting') return from babelfish import Language from dogpile.cache.exception import RegionAlreadyConfigured import subliminal from subliminal import scan_video, save_subtitles from subliminal.cli import MutexLock from subliminal.core import ARCHIVE_EXTENSIONS, scan_archive, refine, search_external_subtitles from subliminal.score import episode_scores, movie_scores from subliminal.video import VIDEO_EXTENSIONS try: subliminal.region.configure('dogpile.cache.dbm', arguments={ 'filename': os.path.join(tempfile.gettempdir(), 'cachefile.dbm'), 'lock_factory': MutexLock, }) except RegionAlreadyConfigured: pass # Let subliminal be more verbose if our logger is set to DEBUG if log.isEnabledFor(logging.DEBUG): logging.getLogger("subliminal").setLevel(logging.INFO) else: logging.getLogger("subliminal").setLevel(logging.CRITICAL) logging.getLogger("dogpile").setLevel(logging.CRITICAL) logging.getLogger("enzyme").setLevel(logging.WARNING) try: languages = set([Language.fromietf(s) for s in config.get('languages', [])]) alternative_languages = set([Language.fromietf(s) for s in config.get('alternatives', [])]) except ValueError as e: raise plugin.PluginError(e) # keep all downloaded subtitles and save to disk when done (no need to write every time) downloaded_subtitles = collections.defaultdict(list) providers_list = config.get('providers', None) provider_configs = config.get('authentication', None) # test if only one language was provided, if so we will download in single mode # (aka no language code added to subtitle filename) # unless we are forced not to by configuration # if we pass 'yes' for single in configuration but choose more than one language # we ignore the configuration and add the language code to the # potentially downloaded files single_mode = config.get('single', '') and len(languages | alternative_languages) <= 1 hearing_impaired = config.get('hearing_impaired', False) with subliminal.core.ProviderPool(providers=providers_list, provider_configs=provider_configs) as provider_pool: for entry in task.accepted: if 'location' not in entry: log.warning('Cannot act on entries that do not represent a local file.') continue if not os.path.exists(entry['location']): entry.fail('file not found: %s' % entry['location']) continue if '$RECYCLE.BIN' in entry['location']: # ignore deleted files in Windows shares continue try: entry_languages = set(entry.get('subtitle_languages', [])) or languages if entry['location'].endswith(VIDEO_EXTENSIONS): video = scan_video(entry['location']) elif entry['location'].endswith(ARCHIVE_EXTENSIONS): video = scan_archive(entry['location']) else: entry.reject('File extension is not a supported video or archive extension') continue # use metadata refiner to get mkv metadata refiner = ('metadata',) refine(video, episode_refiners=refiner, movie_refiners=refiner) existing_subtitles = set(search_external_subtitles(entry['location']).values()) video.subtitle_languages |= existing_subtitles if isinstance(video, subliminal.Episode): title = video.series hash_scores = episode_scores['hash'] else: title = video.title hash_scores = movie_scores['hash'] log.info('Name computed for %s was %s', entry['location'], title) msc = hash_scores if config['exact_match'] else 0 if entry_languages.issubset(video.subtitle_languages): log.debug('All preferred languages already exist for "%s"', entry['title']) entry['subtitles_missing'] = set() continue # subs for preferred lang(s) already exists else: # Gather the subtitles for the alternative languages too, to avoid needing to search the sites # again. They'll just be ignored if the main languages are found. all_subtitles = provider_pool.list_subtitles(video, entry_languages | alternative_languages) subtitles = provider_pool.download_best_subtitles(all_subtitles, video, entry_languages, min_score=msc, hearing_impaired=hearing_impaired) if subtitles: downloaded_subtitles[video].extend(subtitles) log.info('Subtitles found for %s', entry['location']) else: # only try to download for alternatives that aren't alread downloaded subtitles = provider_pool.download_best_subtitles(all_subtitles, video, alternative_languages, min_score=msc, hearing_impaired=hearing_impaired) if subtitles: downloaded_subtitles[video].extend(subtitles) entry.reject('subtitles found for a second-choice language.') else: entry.reject('cannot find any subtitles for now.') downloaded_languages = set([Language.fromietf(str(l.language)) for l in subtitles]) if entry_languages: entry['subtitles_missing'] = entry_languages - downloaded_languages if len(entry['subtitles_missing']) > 0: entry.reject('Subtitles for all primary languages not found') except ValueError as e: log.error('subliminal error: %s', e) entry.fail() if downloaded_subtitles: if task.options.test: log.verbose('Test mode. Found subtitles:') # save subtitles to disk for video, subtitle in downloaded_subtitles.items(): if subtitle: _directory = config.get('directory') if _directory: _directory = os.path.expanduser(_directory) if task.options.test: log.verbose(' FOUND LANGUAGES %s for %s', [str(l.language) for l in subtitle], video.name) continue save_subtitles(video, subtitle, single=single_mode, directory=_directory)
def store_subtitles(file): logging.debug('BAZARR started subtitles indexing for this file: ' + file) actual_subtitles = [] if os.path.exists(file): if os.path.splitext(file)[1] == '.mkv': logging.debug("BAZARR is trying to index embedded subtitles.") try: with open(file, 'rb') as f: mkv = enzyme.MKV(f) for subtitle_track in mkv.subtitle_tracks: try: if alpha2_from_alpha3(subtitle_track.language) != None: lang = str( alpha2_from_alpha3(subtitle_track.language)) logging.debug( "BAZARR embedded subtitles detected: " + lang) actual_subtitles.append([lang, None]) except: logging.debug( "BAZARR unable to index this unrecognized language: " + subtitle_track.language) pass except Exception as e: logging.exception( "BAZARR error when trying to analyze this mkv file: " + file) pass else: logging.debug("BAZARR This file isn't an .mkv file.") brazilian_portuguese = [".pt-br", ".pob", "pb"] try: subtitles = core.search_external_subtitles(file) except Exception as e: logging.exception("BAZARR unable to index external subtitles.") pass else: for subtitle, language in subtitles.iteritems(): if str(os.path.splitext(subtitle)[0]).lower().endswith( tuple(brazilian_portuguese)) is True: logging.debug("BAZARR external subtitles detected: " + "pb") actual_subtitles.append([ str("pb"), path_replace_reverse( os.path.join(os.path.dirname(file), subtitle)) ]) elif str(language) != 'und': logging.debug("BAZARR external subtitles detected: " + str(language)) actual_subtitles.append([ str(language), path_replace_reverse( os.path.join(os.path.dirname(file), subtitle)) ]) else: if os.path.splitext(subtitle)[1] != ".sub": logging.debug( "BAZARR falling back to file content analysis to detect language." ) with open( path_replace( os.path.join(os.path.dirname(file), subtitle)), 'r') as f: text = list(islice(f, 100)) text = ' '.join(text) encoding = UnicodeDammit(text) try: text = text.decode(encoding.original_encoding) detected_language = langdetect.detect(text) except Exception as e: logging.exception( 'BAZARR Error trying to detect language for this subtitles file: ' + path_replace( os.path.join(os.path.dirname(file), subtitle)) + ' You should try to delete this subtitles file manually and ask Bazarr to download it again.' ) else: if len(detected_language) > 0: logging.debug( "BAZARR external subtitles detected and analysis guessed this language: " + str(detected_language)) actual_subtitles.append([ str(detected_language), path_replace_reverse( os.path.join( os.path.dirname(file), subtitle)) ]) conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30) c_db = conn_db.cursor() logging.debug("BAZARR storing those languages to DB: " + str(actual_subtitles)) c_db.execute("UPDATE table_episodes SET subtitles = ? WHERE path = ?", (str(actual_subtitles), path_replace_reverse(file))) conn_db.commit() c_db.close() else: logging.debug( "BAZARR this file doesn't seems to exist or isn't accessible.") logging.debug('BAZARR ended subtitles indexing for this file: ' + file) return actual_subtitles
def get_video(tv_episode, video_path, subtitles_dir=None, subtitles=True, embedded_subtitles=None, release_name=None): """Return the subliminal video for the given path. The video_path is used as a key to cache the video to avoid scanning and parsing the video metadata all the time :param tv_episode: :type tv_episode: medusa.tv.Episode :param video_path: the video path :type video_path: str :param subtitles_dir: the subtitles directory :type subtitles_dir: str or None :param subtitles: True if existing external subtitles should be taken into account :type subtitles: bool or None :param embedded_subtitles: True if embedded subtitles should be taken into account :type embedded_subtitles: bool or None :param release_name: the release name :type release_name: str or None :return: video :rtype: subliminal.video.Video """ key = video_key.format(video_path=video_path) payload = {'subtitles_dir': subtitles_dir, 'subtitles': subtitles, 'embedded_subtitles': embedded_subtitles, 'release_name': release_name} cached_payload = memory_cache.get(key, expiration_time=VIDEO_EXPIRATION_TIME) if cached_payload != NO_VALUE and {k: v for k, v in iteritems(cached_payload) if k != 'video'} == payload: logger.debug(u'Found cached video information under key %s', key) return cached_payload['video'] video_is_mkv = video_path.endswith('.mkv') subtitles_dir = subtitles_dir or get_subtitles_dir(video_path) logger.debug(u'Scanning video %s...', video_path) try: video = scan_video(video_path) except ValueError as error: logger.warning(u'Unable to scan video: %s. Error: %r', video_path, error) else: # Add hash of our custom provider Itasa video.size = os.path.getsize(video_path) if video.size > 10485760: video.hashes['itasa'] = hash_itasa(video_path) # external subtitles if subtitles: video.subtitle_languages |= set(search_external_subtitles(video_path, directory=subtitles_dir).values()) if embedded_subtitles is None: embedded_subtitles = bool(not app.IGNORE_EMBEDDED_SUBS and video_is_mkv) refine(video, episode_refiners=episode_refiners, embedded_subtitles=embedded_subtitles, release_name=release_name, tv_episode=tv_episode) video.alternative_series = list(tv_episode.series.aliases) payload['video'] = video memory_cache.set(key, payload) logger.debug(u'Video information cached under key %s', key) return video