def download_callback(self, menuitem, files):
# scan videos
videos = []
for f in files:
# ignore non-writable locations
if not f.can_write():
continue
# directories
if f.is_directory():
try:
scanned_videos = scan_videos(f.get_location().get_path())
except:
continue
for video in scanned_videos:
if check_video(video, languages=self.config.languages, age=self.config.age,
undefined=self.config.single):
video.subtitle_languages |= set(search_external_subtitles(video.name).values())
refine(video, episode_refiners=self.config.refiners, movie_refiners=self.config.refiners,
embedded_subtitles=self.config.embedded_subtitles)
videos.append(video)
continue
# other inputs
try:
video = scan_video(f.get_location().get_path())
except:
continue
if check_video(video, languages=self.config.languages, undefined=self.config.single):
video.subtitle_languages |= set(search_external_subtitles(video.name).values())
refine(video, episode_refiners=self.config.refiners, movie_refiners=self.config.refiners,
embedded_subtitles=self.config.embedded_subtitles)
videos.append(video)
# download best subtitles
downloaded_subtitles = defaultdict(list)
with AsyncProviderPool(providers=self.config.providers, provider_configs=self.config.provider_configs) as pool:
for v in videos:
scores = get_scores(v)
subtitles = pool.download_best_subtitles(
pool.list_subtitles(v, self.config.languages - v.subtitle_languages),
v, self.config.languages, min_score=scores['hash'] * self.config.min_score / 100,
hearing_impaired=self.config.hearing_impaired, only_one=self.config.single
)
downloaded_subtitles[v] = subtitles
# save subtitles
for v, subtitles in downloaded_subtitles.items():
save_subtitles(v, subtitles, single=self.config.single)
def get_subtitles(self, entry):
if (entry.get('subtitles', eval_lazy=False)
or not ('location' in entry)
or ('$RECYCLE.BIN' in entry['location'])
or not os.path.exists(entry['location'])):
return
from subliminal import scan_video
from subliminal.core import search_external_subtitles, refine
try:
video = scan_video(entry['location'])
# grab external and internal subtitles
subtitles = video.subtitle_languages
refiner = ('metadata', )
refine(video, episode_refiners=refiner, movie_refiners=refiner)
subtitles |= set(
search_external_subtitles(entry['location']).values())
if subtitles:
# convert to human-readable strings
subtitles = [str(l) for l in subtitles]
entry['subtitles'] = subtitles
log.debug('Found subtitles %s for %s', '/'.join(subtitles),
entry['title'])
except Exception as e:
log.error('Error checking local subtitles for %s: %s',
entry['title'], e)
def subtitle(self, episodes):
# Parse babelfish languages
bb_lang = {Language.fromietf(l) for l in self.languages}
# Create subliminal episode set
sub_episodes = set()
for episode in episodes:
ep_path = os.path.join(episode['dir'], episode['filename'])
sub_episode = Episode.fromguess(ep_path, episode)
# Look for external subtitles (not done automatically, apparently)
sub_episode.subtitle_languages |= set(search_external_subtitles(sub_episode.name).values())
sub_episodes.add(sub_episode)
# download subtitles in the specified language
subl_subtitles = download_best_subtitles(sub_episodes, bb_lang, providers=self.providers)
for video, subtitles in subl_subtitles.items():
save_subtitles(video, subtitles)
# save subtitle languages in episode dict
def get_subtitles(self, entry):
if (
entry.get('subtitles', eval_lazy=False)
or not ('location' in entry)
or ('$RECYCLE.BIN' in entry['location'])
or not os.path.exists(entry['location'])
):
return
from subliminal import scan_video
from subliminal.core import search_external_subtitles, refine
try:
video = scan_video(entry['location'])
# grab external and internal subtitles
subtitles = video.subtitle_languages
refiner = ('metadata',)
refine(video, episode_refiners=refiner, movie_refiners=refiner)
subtitles |= set(search_external_subtitles(entry['location']).values())
if subtitles:
# convert to human-readable strings
subtitles = [str(l) for l in subtitles]
entry['subtitles'] = subtitles
log.debug('Found subtitles %s for %s', '/'.join(subtitles), entry['title'])
except Exception as e:
log.error('Error checking local subtitles for %s: %s', entry['title'], e)
def get_video(tv_episode, video_path, subtitles_dir=None, subtitles=True, embedded_subtitles=None, release_name=None):
"""Return the subliminal video for the given path.
The video_path is used as a key to cache the video to avoid
scanning and parsing the video metadata all the time
:param tv_episode:
:type tv_episode: medusa.tv.Episode
:param video_path: the video path
:type video_path: str
:param subtitles_dir: the subtitles directory
:type subtitles_dir: str or None
:param subtitles: True if existing external subtitles should be taken into account
:type subtitles: bool or None
:param embedded_subtitles: True if embedded subtitles should be taken into account
:type embedded_subtitles: bool or None
:param release_name: the release name
:type release_name: str or None
:return: video
:rtype: subliminal.video.Video
"""
key = video_key.format(video_path=video_path)
payload = {'subtitles_dir': subtitles_dir, 'subtitles': subtitles, 'embedded_subtitles': embedded_subtitles,
'release_name': release_name}
cached_payload = memory_cache.get(key, expiration_time=VIDEO_EXPIRATION_TIME)
if cached_payload != NO_VALUE and {k: v for k, v in iteritems(cached_payload) if k != 'video'} == payload:
logger.debug(u'Found cached video information under key %s', key)
return cached_payload['video']
video_is_mkv = video_path.endswith('.mkv')
subtitles_dir = subtitles_dir or get_subtitles_dir(video_path)
logger.debug(u'Scanning video %s...', video_path)
try:
video = scan_video(video_path)
except ValueError as error:
logger.warning(u'Unable to scan video: %s. Error: %r', video_path, error)
else:
if subtitles:
video.subtitle_languages |= set(search_external_subtitles(video_path, directory=subtitles_dir).values())
if embedded_subtitles is None:
embedded_subtitles = bool(not app.IGNORE_EMBEDDED_SUBS and video_is_mkv)
refine(video, episode_refiners=episode_refiners, embedded_subtitles=embedded_subtitles,
release_name=release_name, tv_episode=tv_episode)
video.alternative_series = [alias.title for alias in tv_episode.series.aliases]
payload['video'] = video
memory_cache.set(key, payload)
logger.debug(u'Video information cached under key %s', key)
return video
def get_subtitles(self, entry):
if entry.get('subtitles', eval_lazy=False) or not ('location' in entry) or \
('$RECYCLE.BIN' in entry['location']) or not os.path.exists(entry['location']):
return
from subliminal.core import search_external_subtitles
try:
subtitles = list(search_external_subtitles(entry['location']).values())
if subtitles:
entry['subtitles'] = subtitles
log.debug('Found subtitles %s for %s', '/'.join(subtitles), entry['title'])
except Exception as e:
log.debug('Error checking local subtitles for %s: %s' % (entry['title'], e))
def get_video(tv_episode, video_path, subtitles_dir=None, subtitles=True, embedded_subtitles=None, release_name=None):
"""Return the subliminal video for the given path.
The video_path is used as a key to cache the video to avoid
scanning and parsing the video metadata all the time
:param tv_episode:
:type tv_episode: sickbeard.tv.TVEpisode
:param video_path: the video path
:type video_path: str
:param subtitles_dir: the subtitles directory
:type subtitles_dir: str or None
:param subtitles: True if existing external subtitles should be taken into account
:type subtitles: bool or None
:param embedded_subtitles: True if embedded subtitles should be taken into account
:type embedded_subtitles: bool or None
:param release_name: the release name
:type release_name: str or None
:return: video
:rtype: subliminal.video.Video
"""
key = video_key.format(video_path=video_path)
payload = {'subtitles_dir': subtitles_dir, 'subtitles': subtitles, 'embedded_subtitles': embedded_subtitles,
'release_name': release_name}
cached_payload = region.get(key, expiration_time=VIDEO_EXPIRATION_TIME)
if cached_payload != NO_VALUE and {k: v for k, v in iteritems(cached_payload) if k != 'video'} == payload:
logger.debug(u'Found cached video information under key %s', key)
return cached_payload['video']
try:
video_path = _encode(video_path)
subtitles_dir = _encode(subtitles_dir or get_subtitles_dir(video_path))
logger.debug(u'Scanning video %s...', video_path)
video = scan_video(video_path)
# external subtitles
if subtitles:
video.subtitle_languages |= set(search_external_subtitles(video_path, directory=subtitles_dir).values())
if embedded_subtitles is None:
embedded_subtitles = bool(not sickbeard.EMBEDDED_SUBTITLES_ALL and video_path.endswith('.mkv'))
refine(video, episode_refiners=episode_refiners, embedded_subtitles=embedded_subtitles,
release_name=release_name, tv_episode=tv_episode)
payload['video'] = video
region.set(key, payload)
logger.debug(u'Video information cached under key %s', key)
return video
except Exception as error:
logger.info(u'Exception: %s', error)
def test_search_external_subtitles_no_directory(movies, tmpdir, monkeypatch):
video_name = os.path.split(movies['man_of_steel'].name)[1]
video_root = os.path.splitext(video_name)[0]
tmpdir.ensure(video_name)
monkeypatch.chdir(str(tmpdir))
expected_subtitles = {
video_name + '.srt': Language('und'),
video_root + '.en.srt': Language('eng')
}
for path in expected_subtitles:
tmpdir.ensure(path)
subtitles = search_external_subtitles(video_name)
assert subtitles == expected_subtitles
def test_search_external_subtitles_no_directory(movies, tmpdir, monkeypatch):
video_name = os.path.split(movies['man_of_steel'].name)[1]
video_root = os.path.splitext(video_name)[0]
tmpdir.ensure(video_name)
monkeypatch.chdir(str(tmpdir))
expected_subtitles = {
video_name + '.srt': Language('und'),
video_root + '.en.srt': Language('eng')
}
for path in expected_subtitles:
tmpdir.ensure(path)
subtitles = search_external_subtitles(video_name)
assert subtitles == expected_subtitles
def test_search_external_subtitles_in_directory(episodes, tmpdir):
video_name = episodes['marvels_agents_of_shield_s02e06'].name
video_root = os.path.splitext(video_name)[0]
tmpdir.ensure('tvshows', video_name)
subtitles_directory = str(tmpdir.ensure('subtitles', dir=True))
expected_subtitles = {
video_name + '.srt': Language('und'),
video_root + '.en.srt': Language('eng')
}
tmpdir.ensure('tvshows', video_name + '.fr.srt')
for path in expected_subtitles:
tmpdir.ensure('subtitles', path)
subtitles = search_external_subtitles(video_name, directory=subtitles_directory)
assert subtitles == expected_subtitles
def test_search_external_subtitles_in_directory(episodes, tmpdir):
video_name = episodes['marvels_agents_of_shield_s02e06'].name
video_root = os.path.splitext(video_name)[0]
tmpdir.ensure('tvshows', video_name)
subtitles_directory = str(tmpdir.ensure('subtitles', dir=True))
expected_subtitles = {
video_name + '.srt': Language('und'),
video_root + '.en.srt': Language('eng')
}
tmpdir.ensure('tvshows', video_name + '.fr.srt')
for path in expected_subtitles:
tmpdir.ensure('subtitles', path)
subtitles = search_external_subtitles(video_name, directory=subtitles_directory)
assert subtitles == expected_subtitles
def run(self, scan_path, scan_age, languages, *args, **kwargs):
if not os.path.isdir(scan_path):
raise IOError('Path \'%s\' doesn\'t exist!' % scan_path)
if not scan_age >= 1:
raise ValueError('\'scan_age\' must by at least 1!')
if not len(languages) >= 1:
raise ValueError('\'languages\' list can\'t be empty!')
__tree_dict = lambda: defaultdict(__tree_dict)
result = __tree_dict()
age = timedelta(weeks=scan_age)
languages = set([Language(l) for l in languages])
scan_start = datetime.now()
videos = []
ignored_videos = []
if not region.is_configured:
region.configure('dogpile.cache.dbm', expiration_time=timedelta(days=30), arguments={'filename': 'subliminal.dbm', 'lock_factory': MutexLock})
# scan videos
scanned_videos = scan_videos(scan_path, age=age)
for video in scanned_videos:
video.subtitle_languages |= set(search_external_subtitles(video.name).values())
if check_video(video, languages=languages, age=age, undefined=False):
refine(video)
if languages - video.subtitle_languages:
videos.append(video)
else:
ignored_videos.append(video)
else:
ignored_videos.append(video)
if videos:
result['videos']['collected'] = [os.path.split(v.name)[1] for v in videos]
if ignored_videos:
result['videos']['ignored'] = [os.path.split(v.name)[1] for v in ignored_videos]
scan_end = datetime.now()
result['meta']['start'] = scan_start.isoformat()
result['meta']['end'] = scan_end.isoformat()
result['meta']['duration'] = str(scan_end - scan_start)
return result
def test_search_external_subtitles_archive(movies, tmpdir):
video_name = os.path.split(movies['interstellar'].name)[1]
video_root = os.path.splitext(video_name)[0]
video_path = str(tmpdir.ensure(video_name))
expected_subtitles = {
video_name + '.srt': Language('und'),
video_root + '.srt': Language('und'),
video_root + '.en.srt': Language('eng'),
video_name + '.fra.srt': Language('fra'),
video_root + '.pt-BR.srt': Language('por', 'BR'),
video_name + '.sr_cyrl.sub': Language('srp', script='Cyrl'),
video_name + '.something.srt': Language('und')
}
tmpdir.ensure(os.path.split(movies['interstellar'].name)[1] + '.srt')
for path in expected_subtitles:
tmpdir.ensure(path)
subtitles = search_external_subtitles(video_path)
assert subtitles == expected_subtitles
def test_search_external_subtitles_archive(movies, tmpdir):
video_name = os.path.split(movies['interstellar'].name)[1]
video_root = os.path.splitext(video_name)[0]
video_path = str(tmpdir.ensure(video_name))
expected_subtitles = {
video_name + '.srt': Language('und'),
video_root + '.srt': Language('und'),
video_root + '.en.srt': Language('eng'),
video_name + '.fra.srt': Language('fra'),
video_root + '.pt-BR.srt': Language('por', 'BR'),
video_name + '.sr_cyrl.sub': Language('srp', script='Cyrl'),
video_name + '.something.srt': Language('und')
}
tmpdir.ensure(os.path.split(movies['interstellar'].name)[1] + '.srt')
for path in expected_subtitles:
tmpdir.ensure(path)
subtitles = search_external_subtitles(video_path)
assert subtitles == expected_subtitles
def get_video(video_path, subtitles_path=None):
if not subtitles_path:
subtitles_path = get_subtitles_path(video_path)
try:
# Encode paths to UTF-8 to ensure subliminal support.
video_path = video_path.encode('utf-8')
subtitles_path = subtitles_path.encode('utf-8')
except UnicodeEncodeError:
# Fallback to system encoding. This should never happen.
video_path = video_path.encode(sickbeard.SYS_ENCODING)
subtitles_path = subtitles_path.encode(sickbeard.SYS_ENCODING)
try:
video = scan_video(video_path)
video.subtitle_languages |= set(search_external_subtitles(video_path, directory=subtitles_path).values())
if not sickbeard.EMBEDDED_SUBTITLES_ALL and video_path.endswith('.mkv'):
refine(video, embedded_subtitles=True)
except Exception as error:
logger.log(u'Exception: {}'.format(error), logger.DEBUG)
return None
return video
def download(obj, provider, language, age, directory, encoding, single, force, hearing_impaired,
min_score, max_workers, archives, verbose, path):
"""Download best subtitles.
PATH can be an directory containing videos, a video file path or a video file name. It can be used multiple times.
If an existing subtitle is detected (external or embedded) in the correct language, the download is skipped for
the associated video.
"""
# process parameters
language = set(language)
# scan videos
videos = []
ignored_videos = []
errored_paths = []
with click.progressbar(path, label='Collecting videos', item_show_func=lambda p: p or '') as bar:
for p in bar:
logger.debug('Collecting path %s', p)
# non-existing
if not os.path.exists(p):
try:
video = Video.fromname(p)
except:
logger.exception('Unexpected error while collecting non-existing path %s', p)
errored_paths.append(p)
continue
if not force:
video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values())
refine(video, embedded_subtitles=not force)
videos.append(video)
continue
# directories
if os.path.isdir(p):
try:
scanned_videos = scan_videos(p, age=age, archives=archives, subtitles=not force,
subtitles_dir=directory)
except:
logger.exception('Unexpected error while collecting directory path %s', p)
errored_paths.append(p)
continue
for video in scanned_videos:
if check_video(video, languages=language, age=age, undefined=single):
if not force:
video.subtitle_languages |= set(search_external_subtitles(video.name,
directory=directory).values())
refine(video, embedded_subtitles=not force)
videos.append(video)
else:
ignored_videos.append(video)
continue
# other inputs
try:
video = scan_video(p)
except:
logger.exception('Unexpected error while collecting path %s', p)
errored_paths.append(p)
continue
if check_video(video, languages=language, age=age, undefined=single):
if not force:
video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values())
refine(video, embedded_subtitles=not force)
videos.append(video)
else:
ignored_videos.append(video)
# output errored paths
if verbose > 0:
for p in errored_paths:
click.secho('%s errored' % p, fg='red')
# output ignored videos
if verbose > 1:
for video in ignored_videos:
click.secho('%s ignored - subtitles: %s / age: %d day%s' % (
os.path.split(video.name)[1],
', '.join(str(s) for s in video.subtitle_languages) or 'none',
video.age.days,
's' if video.age.days > 1 else ''
), fg='yellow')
# report collected videos
click.echo('%s video%s collected / %s video%s ignored / %s error%s' % (
click.style(str(len(videos)), bold=True, fg='green' if videos else None),
's' if len(videos) > 1 else '',
click.style(str(len(ignored_videos)), bold=True, fg='yellow' if ignored_videos else None),
's' if len(ignored_videos) > 1 else '',
click.style(str(len(errored_paths)), bold=True, fg='red' if errored_paths else None),
's' if len(errored_paths) > 1 else '',
))
# exit if no video collected
if not videos:
return
# download best subtitles
downloaded_subtitles = defaultdict(list)
with AsyncProviderPool(max_workers=max_workers, providers=provider, provider_configs=obj['provider_configs']) as p:
with click.progressbar(videos, label='Downloading subtitles',
item_show_func=lambda v: os.path.split(v.name)[1] if v is not None else '') as bar:
for v in bar:
scores = get_scores(v)
subtitles = p.download_best_subtitles(p.list_subtitles(v, language - v.subtitle_languages),
v, language, min_score=scores['hash'] * min_score / 100,
hearing_impaired=hearing_impaired, only_one=single)
downloaded_subtitles[v] = subtitles
# TODO: warn about discarded providers
# save subtitles
total_subtitles = 0
for v, subtitles in downloaded_subtitles.items():
saved_subtitles = save_subtitles(v, subtitles, single=single, directory=directory, encoding=encoding)
total_subtitles += len(saved_subtitles)
if verbose > 0:
click.echo('%s subtitle%s downloaded for %s' % (click.style(str(len(saved_subtitles)), bold=True),
's' if len(saved_subtitles) > 1 else '',
os.path.split(v.name)[1]))
if verbose > 1:
for s in saved_subtitles:
matches = s.get_matches(v)
score = compute_score(s, v)
# score color
score_color = None
scores = get_scores(v)
if isinstance(v, Movie):
if score < scores['title']:
score_color = 'red'
elif score < scores['title'] + scores['year'] + scores['release_group']:
score_color = 'yellow'
else:
score_color = 'green'
elif isinstance(v, Episode):
if score < scores['series'] + scores['season'] + scores['episode']:
score_color = 'red'
elif score < scores['series'] + scores['season'] + scores['episode'] + scores['release_group']:
score_color = 'yellow'
else:
score_color = 'green'
# scale score from 0 to 100 taking out preferences
scaled_score = score
if s.hearing_impaired == hearing_impaired:
scaled_score -= scores['hearing_impaired']
scaled_score *= 100 / scores['hash']
# echo some nice colored output
click.echo(' - [{score}] {language} subtitle from {provider_name} (match on {matches})'.format(
score=click.style('{:5.1f}'.format(scaled_score), fg=score_color, bold=score >= scores['hash']),
language=s.language.name if s.language.country is None else '%s (%s)' % (s.language.name,
s.language.country.name),
provider_name=s.provider_name,
matches=', '.join(sorted(matches, key=scores.get, reverse=True))
))
if verbose == 0:
click.echo('Downloaded %s subtitle%s' % (click.style(str(total_subtitles), bold=True),
's' if total_subtitles > 1 else ''))
def download_callback(self, menuitem, files):
# scan videos
videos = []
for f in files:
# ignore non-writable locations
if not f.can_write():
continue
# directories
if f.is_directory():
try:
scanned_videos = scan_videos(f.get_location().get_path())
except:
continue
for video in scanned_videos:
if check_video(video,
languages=self.config.languages,
age=self.config.age,
undefined=self.config.single):
video.subtitle_languages |= set(
search_external_subtitles(video.name).values())
refine(
video,
episode_refiners=self.config.refiners,
movie_refiners=self.config.refiners,
embedded_subtitles=self.config.embedded_subtitles)
videos.append(video)
continue
# other inputs
try:
video = scan_video(f.get_location().get_path())
except:
continue
if check_video(video,
languages=self.config.languages,
undefined=self.config.single):
video.subtitle_languages |= set(
search_external_subtitles(video.name).values())
refine(video,
episode_refiners=self.config.refiners,
movie_refiners=self.config.refiners,
embedded_subtitles=self.config.embedded_subtitles)
videos.append(video)
# download best subtitles
downloaded_subtitles = defaultdict(list)
with AsyncProviderPool(
providers=self.config.providers,
provider_configs=self.config.provider_configs) as pool:
for v in videos:
scores = get_scores(v)
subtitles = pool.download_best_subtitles(
pool.list_subtitles(
v, self.config.languages - v.subtitle_languages),
v,
self.config.languages,
min_score=scores['hash'] * self.config.min_score / 100,
hearing_impaired=self.config.hearing_impaired,
only_one=self.config.single)
downloaded_subtitles[v] = subtitles
# save subtitles
for v, subtitles in downloaded_subtitles.items():
save_subtitles(v, subtitles, single=self.config.single)
def emit(self, task, config):
if not config:
return
entries = []
with Session() as session:
for sub_item in queue_get(session=session):
if os.path.exists(sub_item.path):
path = sub_item.path
elif sub_item.alternate_path and os.path.exists(
sub_item.alternate_path):
path = sub_item.alternate_path
elif not config['remove_not_found'] and \
sub_item.added + parse_timedelta('24 hours') > datetime.combine(date.today(), time()):
log.warning(
'File %s was not found. Deleting after %s.' %
(sub_item.path,
str(sub_item.added + parse_timedelta('24 hours'))))
continue
else:
log.error('File not found. Removing "%s" from queue.' %
sub_item.title)
session.delete(sub_item)
continue
if os.path.isdir(path):
paths = os.listdir(path)
if not paths:
log.warning('Queued folder %s is empty.' % path)
continue
path_dir = path
else:
paths = [path]
path_dir = os.path.dirname(path)
primary = set()
for language in sub_item.languages:
primary.add(Language.fromietf(language.language))
for file in paths:
entry = Entry()
if not file.lower().endswith(VIDEO_EXTENSIONS):
continue
file = normalize_path(os.path.join(path_dir, file))
entry['url'] = urljoin('file:', request.pathname2url(file))
entry['location'] = file
entry['title'] = os.path.splitext(
os.path.basename(file))[0] # filename without ext
entry['subtitle_languages'] = primary
try:
from subliminal.core import search_external_subtitles
try:
existing_subtitles = set(
search_external_subtitles(
normalize_path(file)).values())
if primary and not primary - existing_subtitles:
log.debug(
'All subtitles already fetched for %s.' %
entry['title'])
sub_item.downloaded = True
continue
except ValueError as e:
log.error(
'Invalid video file: %s. Removing %s from queue.'
% (e, entry['title']))
session.delete(sub_item)
continue
except ImportError:
log.debug(
'Falling back to simple check since Subliminal is not installed.'
)
# use glob since subliminal is not there
path_no_ext = os.path.splitext(normalize_path(file))[0]
# can only check subtitles that have explicit language codes in the file name
if primary:
files = glob.glob(path_no_ext + "*")
files = [item.lower() for item in files]
for lang in primary:
if not any('%s.%s' %
(path_no_ext, lang) and f.lower(
).endswith(SUBTITLE_EXTENSIONS)
for f in files):
break
else:
log.debug(
'All subtitles already fetched for %s.' %
entry['title'])
sub_item.downloaded = True
continue
entry.on_complete(self.complete, path=path, task=task)
entries.append(entry)
log.debug('Emitting entry for %s.' % entry['title'])
return entries
def get_video(tv_episode,
video_path,
subtitles_dir=None,
subtitles=True,
embedded_subtitles=None,
release_name=None):
"""Return the subliminal video for the given path.
The video_path is used as a key to cache the video to avoid
scanning and parsing the video metadata all the time
:param tv_episode:
:type tv_episode: medusa.tv.TVEpisode
:param video_path: the video path
:type video_path: str
:param subtitles_dir: the subtitles directory
:type subtitles_dir: str or None
:param subtitles: True if existing external subtitles should be taken into account
:type subtitles: bool or None
:param embedded_subtitles: True if embedded subtitles should be taken into account
:type embedded_subtitles: bool or None
:param release_name: the release name
:type release_name: str or None
:return: video
:rtype: subliminal.video.Video
"""
key = video_key.format(video_path=video_path)
payload = {
'subtitles_dir': subtitles_dir,
'subtitles': subtitles,
'embedded_subtitles': embedded_subtitles,
'release_name': release_name
}
cached_payload = region.get(key, expiration_time=VIDEO_EXPIRATION_TIME)
if cached_payload != NO_VALUE and {
k: v
for k, v in iteritems(cached_payload) if k != 'video'
} == payload:
logger.debug(u'Found cached video information under key %s', key)
return cached_payload['video']
try:
video_path = _encode(video_path)
subtitles_dir = _encode(subtitles_dir or get_subtitles_dir(video_path))
logger.debug(u'Scanning video %s...', video_path)
video = scan_video(video_path)
# external subtitles
if subtitles:
video.subtitle_languages |= set(
search_external_subtitles(video_path,
directory=subtitles_dir).values())
if embedded_subtitles is None:
embedded_subtitles = bool(not app.EMBEDDED_SUBTITLES_ALL
and video_path.endswith('.mkv'))
refine(video,
episode_refiners=episode_refiners,
embedded_subtitles=embedded_subtitles,
release_name=release_name,
tv_episode=tv_episode)
payload['video'] = video
region.set(key, payload)
logger.debug(u'Video information cached under key %s', key)
return video
except Exception as error:
logger.info(u'Exception: %s', error)
def download(obj, provider, refiner, language, age, directory, encoding,
single, force, hearing_impaired, min_score, max_workers, archives,
verbose, path):
"""Download best subtitles.
PATH can be an directory containing videos, a video file path or a video file name. It can be used multiple times.
If an existing subtitle is detected (external or embedded) in the correct language, the download is skipped for
the associated video.
"""
# process parameters
language = set(language)
# scan videos
videos = []
ignored_videos = []
errored_paths = []
with click.progressbar(path,
label='Collecting videos',
item_show_func=lambda p: p or '') as bar:
for p in bar:
logger.debug('Collecting path %s', p)
# non-existing
if not os.path.exists(p):
try:
video = Video.fromname(p)
except:
logger.exception(
'Unexpected error while collecting non-existing path %s',
p)
errored_paths.append(p)
continue
if not force:
video.subtitle_languages |= set(
search_external_subtitles(
video.name, directory=directory).values())
refine(video,
episode_refiners=refiner,
movie_refiners=refiner,
embedded_subtitles=not force)
videos.append(video)
continue
# directories
if os.path.isdir(p):
try:
scanned_videos = scan_videos(p, age=age, archives=archives)
except:
logger.exception(
'Unexpected error while collecting directory path %s',
p)
errored_paths.append(p)
continue
for video in scanned_videos:
if check_video(video,
languages=language,
age=age,
undefined=single):
if not force:
video.subtitle_languages |= set(
search_external_subtitles(
video.name, directory=directory).values())
refine(video,
episode_refiners=refiner,
movie_refiners=refiner,
embedded_subtitles=not force)
videos.append(video)
else:
ignored_videos.append(video)
continue
# other inputs
try:
video = scan_video(p)
except:
logger.exception('Unexpected error while collecting path %s',
p)
errored_paths.append(p)
continue
if check_video(video,
languages=language,
age=age,
undefined=single):
if not force:
video.subtitle_languages |= set(
search_external_subtitles(
video.name, directory=directory).values())
refine(video,
episode_refiners=refiner,
movie_refiners=refiner,
embedded_subtitles=not force)
videos.append(video)
else:
ignored_videos.append(video)
# output errored paths
if verbose > 0:
for p in errored_paths:
click.secho('%s errored' % p, fg='red')
# output ignored videos
if verbose > 1:
for video in ignored_videos:
click.secho(
'%s ignored - subtitles: %s / age: %d day%s' %
(os.path.split(video.name)[1],
', '.join(str(s) for s in video.subtitle_languages)
or 'none', video.age.days, 's' if video.age.days > 1 else ''),
fg='yellow')
# report collected videos
click.echo('%s video%s collected / %s video%s ignored / %s error%s' % (
click.style(
str(len(videos)), bold=True, fg='green' if videos else None),
's' if len(videos) > 1 else '',
click.style(str(len(ignored_videos)),
bold=True,
fg='yellow' if ignored_videos else None),
's' if len(ignored_videos) > 1 else '',
click.style(str(len(errored_paths)),
bold=True,
fg='red' if errored_paths else None),
's' if len(errored_paths) > 1 else '',
))
# exit if no video collected
if not videos:
return
# download best subtitles
downloaded_subtitles = defaultdict(list)
with AsyncProviderPool(max_workers=max_workers,
providers=provider,
provider_configs=obj['provider_configs']) as p:
with click.progressbar(
videos,
label='Downloading subtitles',
item_show_func=lambda v: os.path.split(v.name)[1]
if v is not None else '') as bar:
for v in bar:
scores = get_scores(v)
subtitles = p.download_best_subtitles(
p.list_subtitles(v, language - v.subtitle_languages),
v,
language,
min_score=scores['hash'] * min_score / 100,
hearing_impaired=hearing_impaired,
only_one=single)
downloaded_subtitles[v] = subtitles
if p.discarded_providers:
click.secho(
'Some providers have been discarded due to unexpected errors: %s'
% ', '.join(p.discarded_providers),
fg='yellow')
# save subtitles
total_subtitles = 0
for v, subtitles in downloaded_subtitles.items():
saved_subtitles = save_subtitles(v,
subtitles,
single=single,
directory=directory,
encoding=encoding)
total_subtitles += len(saved_subtitles)
if verbose > 0:
click.echo(
'%s subtitle%s downloaded for %s' %
(click.style(str(len(saved_subtitles)), bold=True), 's' if
len(saved_subtitles) > 1 else '', os.path.split(v.name)[1]))
if verbose > 1:
for s in saved_subtitles:
matches = s.get_matches(v)
score = compute_score(s, v)
# score color
score_color = None
scores = get_scores(v)
if isinstance(v, Movie):
if score < scores['title']:
score_color = 'red'
elif score < scores['title'] + scores['year'] + scores[
'release_group']:
score_color = 'yellow'
else:
score_color = 'green'
elif isinstance(v, Episode):
if score < scores['series'] + scores['season'] + scores[
'episode']:
score_color = 'red'
elif score < scores['series'] + scores['season'] + scores[
'episode'] + scores['release_group']:
score_color = 'yellow'
else:
score_color = 'green'
# scale score from 0 to 100 taking out preferences
scaled_score = score
if s.hearing_impaired == hearing_impaired:
scaled_score -= scores['hearing_impaired']
scaled_score *= 100 / scores['hash']
# echo some nice colored output
click.echo(
' - [{score}] {language} subtitle from {provider_name} (match on {matches})'
.format(score=click.style('{:5.1f}'.format(scaled_score),
fg=score_color,
bold=score >= scores['hash']),
language=s.language.name
if s.language.country is None else '%s (%s)' %
(s.language.name, s.language.country.name),
provider_name=s.provider_name,
matches=', '.join(
sorted(matches, key=scores.get,
reverse=True))))
if verbose == 0:
click.echo('Downloaded %s subtitle%s' %
(click.style(str(total_subtitles), bold=True),
's' if total_subtitles > 1 else ''))
def on_task_output(self, task, config):
"""
Configuration::
subliminal:
languages: List of languages (as IETF codes) in order of preference. At least one is required.
alternatives: List of second-choice languages; subs will be downloaded but entries rejected.
exact_match: Use file hash only to search for subs, otherwise Subliminal will try to guess by filename.
providers: List of providers from where to download subtitles.
single: Download subtitles in single mode (no language code added to subtitle filename).
directory: Path to directory where to save the subtitles, default is next to the video.
hearing_impaired: Prefer subtitles for the hearing impaired when available
authentication: >
Dictionary of configuration options for different providers.
Keys correspond to provider names, and values are dictionaries, usually specifying `username` and
`password`.
"""
if not task.accepted:
logger.debug('nothing accepted, aborting')
return
import subliminal
from babelfish import Language
from dogpile.cache.exception import RegionAlreadyConfigured
from subliminal import save_subtitles, scan_video
from subliminal.cli import MutexLock
from subliminal.core import (
ARCHIVE_EXTENSIONS,
refine,
scan_archive,
search_external_subtitles,
)
from subliminal.score import episode_scores, movie_scores
from subliminal.video import VIDEO_EXTENSIONS
try:
subliminal.region.configure(
'dogpile.cache.dbm',
arguments={
'filename': os.path.join(tempfile.gettempdir(), 'cachefile.dbm'),
'lock_factory': MutexLock,
},
)
except RegionAlreadyConfigured:
pass
# Let subliminal be more verbose if our logger is set to DEBUG
if logger.level(task.manager.options.loglevel).no <= logger.level('DEBUG').no:
logging.getLogger("subliminal").setLevel(logging.INFO)
else:
logging.getLogger("subliminal").setLevel(logging.CRITICAL)
logging.getLogger("dogpile").setLevel(logging.CRITICAL)
logging.getLogger("enzyme").setLevel(logging.WARNING)
try:
languages = set([Language.fromietf(s) for s in config.get('languages', [])])
alternative_languages = set(
[Language.fromietf(s) for s in config.get('alternatives', [])]
)
except ValueError as e:
raise plugin.PluginError(e)
# keep all downloaded subtitles and save to disk when done (no need to write every time)
downloaded_subtitles = collections.defaultdict(list)
providers_list = config.get('providers', None)
provider_configs = config.get('authentication', None)
# test if only one language was provided, if so we will download in single mode
# (aka no language code added to subtitle filename)
# unless we are forced not to by configuration
# if we pass 'yes' for single in configuration but choose more than one language
# we ignore the configuration and add the language code to the
# potentially downloaded files
single_mode = config.get('single', '') and len(languages | alternative_languages) <= 1
hearing_impaired = config.get('hearing_impaired', False)
with subliminal.core.ProviderPool(
providers=providers_list, provider_configs=provider_configs
) as provider_pool:
for entry in task.accepted:
if 'location' not in entry:
logger.warning('Cannot act on entries that do not represent a local file.')
continue
if not os.path.exists(entry['location']):
entry.fail('file not found: %s' % entry['location'])
continue
if '$RECYCLE.BIN' in entry['location']: # ignore deleted files in Windows shares
continue
try:
entry_languages = set(entry.get('subtitle_languages', [])) or languages
if entry['location'].endswith(VIDEO_EXTENSIONS):
video = scan_video(entry['location'])
elif entry['location'].endswith(ARCHIVE_EXTENSIONS):
video = scan_archive(entry['location'])
else:
entry.reject(
'File extension is not a supported video or archive extension'
)
continue
# use metadata refiner to get mkv metadata
refiner = ('metadata',)
refine(video, episode_refiners=refiner, movie_refiners=refiner)
existing_subtitles = set(search_external_subtitles(entry['location']).values())
video.subtitle_languages |= existing_subtitles
if isinstance(video, subliminal.Episode):
title = video.series
hash_scores = episode_scores['hash']
else:
title = video.title
hash_scores = movie_scores['hash']
logger.info('Name computed for {} was {}', entry['location'], title)
msc = hash_scores if config['exact_match'] else 0
if entry_languages.issubset(video.subtitle_languages):
logger.debug(
'All preferred languages already exist for "{}"', entry['title']
)
entry['subtitles_missing'] = set()
continue # subs for preferred lang(s) already exists
else:
# Gather the subtitles for the alternative languages too, to avoid needing to search the sites
# again. They'll just be ignored if the main languages are found.
all_subtitles = provider_pool.list_subtitles(
video, entry_languages | alternative_languages
)
try:
subtitles = provider_pool.download_best_subtitles(
all_subtitles,
video,
entry_languages,
min_score=msc,
hearing_impaired=hearing_impaired,
)
except TypeError as e:
logger.error(
'Downloading subtitles failed due to a bug in subliminal. Please seehttps://github.com/Diaoul/subliminal/issues/921. Error: {}',
e,
)
subtitles = []
if subtitles:
downloaded_subtitles[video].extend(subtitles)
logger.info('Subtitles found for {}', entry['location'])
else:
# only try to download for alternatives that aren't already downloaded
subtitles = provider_pool.download_best_subtitles(
all_subtitles,
video,
alternative_languages,
min_score=msc,
hearing_impaired=hearing_impaired,
)
if subtitles:
downloaded_subtitles[video].extend(subtitles)
entry.reject('subtitles found for a second-choice language.')
else:
entry.reject('cannot find any subtitles for now.')
downloaded_languages = set(
[Language.fromietf(str(l.language)) for l in subtitles]
)
if entry_languages:
entry['subtitles_missing'] = entry_languages - downloaded_languages
if len(entry['subtitles_missing']) > 0:
entry.reject('Subtitles for all primary languages not found')
except ValueError as e:
logger.error('subliminal error: {}', e)
entry.fail()
if downloaded_subtitles:
if task.options.test:
logger.verbose('Test mode. Found subtitles:')
# save subtitles to disk
for video, subtitle in downloaded_subtitles.items():
if subtitle:
_directory = config.get('directory')
if _directory:
_directory = os.path.expanduser(_directory)
if task.options.test:
logger.verbose(
' FOUND LANGUAGES {} for {}',
[str(l.language) for l in subtitle],
video.name,
)
continue
save_subtitles(video, subtitle, single=single_mode, directory=_directory)
def store_subtitles(file):
# languages = []
actual_subtitles = []
if os.path.exists(file):
if os.path.splitext(file)[1] == '.mkv':
try:
with open(file, 'rb') as f:
mkv = enzyme.MKV(f)
for subtitle_track in mkv.subtitle_tracks:
try:
if alpha2_from_alpha3(subtitle_track.language) != None:
actual_subtitles.append([
str(alpha2_from_alpha3(
subtitle_track.language)), None
])
except:
pass
except:
pass
brazilian_portuguese = [".pt-br", ".pob", "pb"]
try:
subtitles = core.search_external_subtitles(file)
except:
pass
else:
for subtitle, language in subtitles.iteritems():
if str(os.path.splitext(subtitle)[0]).lower().endswith(
tuple(brazilian_portuguese)) is True:
actual_subtitles.append([
str("pb"),
path_replace_reverse(
os.path.join(os.path.dirname(file), subtitle))
])
elif str(language) != 'und':
actual_subtitles.append([
str(language),
path_replace_reverse(
os.path.join(os.path.dirname(file), subtitle))
])
else:
with open(
path_replace(
os.path.join(os.path.dirname(file), subtitle)),
'r') as f:
text = list(islice(f, 100))
text = ' '.join(text)
encoding = UnicodeDammit(text)
try:
text = text.decode(encoding.original_encoding)
except Exception as e:
logging.exception(
'Error trying to detect character encoding for this subtitles file: '
+ path_replace(
os.path.join(os.path.dirname(file),
subtitle)) +
' You should try to delete this subtitles file manually and ask Bazarr to download it again.'
)
else:
detected_language = langdetect.detect(text)
if len(detected_language) > 0:
actual_subtitles.append([
str(detected_language),
path_replace_reverse(
os.path.join(os.path.dirname(file),
subtitle))
])
conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
c_db.execute(
"UPDATE table_episodes SET subtitles = ? WHERE path = ?",
(str(actual_subtitles), path_replace_reverse(file)))
conn_db.commit()
c_db.close()
return actual_subtitles
def run(self,
scan_path,
scan_age,
languages,
encoding,
min_score,
providers,
provider_configs,
max_workers,
plex_url=None,
plex_token=None,
*args,
**kwargs):
if not os.path.isdir(scan_path):
raise IOError('Path \'%s\' doesn\'t exist!' % scan_path)
if not scan_age >= 1:
raise ValueError('\'scan_age\' must by at least 1!')
if not len(languages) >= 1:
raise ValueError('\'languages\' list can\'t be empty!')
if not providers:
raise ValueError('\'providers\' argument can\'t be empty!')
if not max_workers >= 1:
raise ValueError('\'max_workers\' must be at least 1!')
if not provider_configs:
provider_configs = {}
__tree_dict = lambda: defaultdict(__tree_dict)
result = __tree_dict()
encoding = codecs.lookup(encoding).name
age = timedelta(weeks=scan_age)
languages = set([Language(l) for l in languages])
plex = None
if plex_url and plex_token:
plex = PlexServer(plex_url, plex_token)
scan_start = datetime.now()
videos = []
ignored_videos = []
if not region.is_configured:
region.configure('dogpile.cache.dbm',
expiration_time=timedelta(days=30),
arguments={
'filename': 'subliminal.dbm',
'lock_factory': MutexLock
})
# scan videos
scanned_videos = scan_videos(scan_path, age=age)
for video in scanned_videos:
video.subtitle_languages |= set(
search_external_subtitles(video.name).values())
if check_video(video,
languages=languages,
age=age,
undefined=False):
refine(video)
if languages - video.subtitle_languages:
videos.append(video)
else:
ignored_videos.append(video)
else:
ignored_videos.append(video)
if videos:
result['videos']['collected'] = [
os.path.split(v.name)[1] for v in videos
]
if ignored_videos:
result['videos']['ignored'] = len(ignored_videos)
if videos:
# download best subtitles
downloaded_subtitles = defaultdict(list)
with AsyncProviderPool(max_workers=max_workers,
providers=providers,
provider_configs=provider_configs) as p:
for video in videos:
scores = get_scores(video)
subtitles_to_download = p.list_subtitles(
video, languages - video.subtitle_languages)
downloaded_subtitles[video] = p.download_best_subtitles(
subtitles_to_download,
video,
languages,
min_score=scores['hash'] * min_score / 100)
if p.discarded_providers:
result['providers']['discarded'] = list(
p.discarded_providers)
# filter subtitles
with TinyDB('subtitle_db.json') as db:
table = db.table('downloaded')
query = Query()
for video, subtitles in downloaded_subtitles.items():
discarded_subtitles = list()
discarded_subtitles_info = list()
for s in subtitles:
subtitle_hash = hashlib.sha256(s.content).hexdigest()
subtitle_file = get_subtitle_path(
os.path.split(video.name)[1], s.language)
dbo = {'hash': subtitle_hash, 'file': subtitle_file}
if table.search((query.hash == subtitle_hash)
& (query.file == subtitle_file)):
discarded_subtitles.append(s)
discarded_subtitles_info.append(dbo)
else:
table.insert(dbo)
downloaded_subtitles[video] = [
x for x in subtitles if x not in discarded_subtitles
]
if discarded_subtitles_info:
result['subtitles'][
'discarded'] = result['subtitles'].get(
'discarded', []) + discarded_subtitles_info
downloaded_subtitles = {
k: v
for k, v in downloaded_subtitles.items() if v
}
# save subtitles
saved_subtitles = {}
for video, subtitles in downloaded_subtitles.items():
saved_subtitles[video] = save_subtitles(video,
subtitles,
directory=None,
encoding=encoding)
for key, group in groupby(saved_subtitles[video],
lambda x: x.provider_name):
subtitle_filenames = [
get_subtitle_path(
os.path.split(video.name)[1], s.language)
for s in list(group)
]
result['subtitles'][key] = result['subtitles'].get(
key, []) + subtitle_filenames
result['subtitles']['total'] = sum(
len(v) for v in saved_subtitles.values())
# refresh plex
for video, subtitles in saved_subtitles.items():
if plex and subtitles:
item_found = False
for section in plex.library.sections():
try:
if isinstance(section,
MovieSection) and isinstance(
video, Movie):
results = section.search(title=video.title,
year=video.year,
libtype='movie',
sort='addedAt:desc',
maxresults=1)
if not results:
raise NotFound
plex_item = results[0]
elif isinstance(section,
ShowSection) and isinstance(
video, Episode):
results = section.search(title=video.series,
year=video.year,
libtype='show',
sort='addedAt:desc',
maxresults=1)
if not results:
raise NotFound
plex_item = results[0].episode(
season=video.season, episode=video.episode)
else:
continue
except NotFound:
continue
except BadRequest:
continue
if plex_item:
plex_item.refresh()
result['plex']['refreshed'] = result['plex'].get(
'refreshed', []) + [
'%s%s' %
(repr(plex_item.section()), repr(video))
]
item_found = True
if not item_found:
result['plex']['failed'] = result['plex'].get(
'failed', []) + [repr(video)]
# convert subtitles
for video, subtitles in saved_subtitles.items():
target_format = aeidon.formats.SUBRIP
for s in subtitles:
subtitle_path = get_subtitle_path(video.name, s.language)
source_format = aeidon.util.detect_format(
subtitle_path, encoding)
source_file = aeidon.files.new(
source_format, subtitle_path,
aeidon.encodings.detect_bom(subtitle_path) or encoding)
if source_format != target_format:
format_info = {
'file':
get_subtitle_path(
os.path.split(video.name)[1], s.language),
'from':
source_format.label,
'to':
target_format.label
}
result['subtitles'][
'converted'] = result['subtitles'].get(
'converted', []) + [format_info]
aeidon_subtitles = source_file.read()
for f in [
aeidon.formats.SUBRIP, aeidon.formats.MICRODVD,
aeidon.formats.MPL2
]:
markup = aeidon.markups.new(f)
for s in aeidon_subtitles:
s.main_text = markup.decode(s.main_text)
markup = aeidon.markups.new(target_format)
for s in aeidon_subtitles:
s.main_text = markup.encode(s.main_text)
target_file = aeidon.files.new(target_format,
subtitle_path, encoding)
target_file.write(aeidon_subtitles, aeidon.documents.MAIN)
scan_end = datetime.now()
result['meta']['start'] = scan_start.isoformat()
result['meta']['end'] = scan_end.isoformat()
result['meta']['duration'] = str(scan_end - scan_start)
return result
def emit(self, task, config):
if not config:
return
entries = []
with Session() as session:
for sub_item in queue_get(session=session):
if os.path.exists(sub_item.path):
path = sub_item.path
elif sub_item.alternate_path and os.path.exists(sub_item.alternate_path):
path = sub_item.alternate_path
elif not config['remove_not_found'] and \
sub_item.added + parse_timedelta('24 hours') > datetime.combine(date.today(), time()):
log.warning('File %s was not found. Deleting after %s.' %
(sub_item.path, str(sub_item.added + parse_timedelta('24 hours'))))
continue
else:
log.error('File not found. Removing "%s" from queue.' % sub_item.title)
session.delete(sub_item)
continue
if os.path.isdir(path):
paths = os.listdir(path)
if not paths:
log.warning('Queued folder %s is empty.' % path)
continue
path_dir = path
else:
paths = [path]
path_dir = os.path.dirname(path)
primary = set()
for language in sub_item.languages:
primary.add(Language.fromietf(language.language))
for file in paths:
entry = Entry()
if not file.lower().endswith(VIDEO_EXTENSIONS):
continue
file = normalize_path(os.path.join(path_dir, file))
entry['url'] = urljoin('file:', request.pathname2url(file))
entry['location'] = file
entry['title'] = os.path.splitext(os.path.basename(file))[0] # filename without ext
entry['subtitle_languages'] = primary
try:
from subliminal.core import search_external_subtitles
try:
existing_subtitles = set(search_external_subtitles(normalize_path(file)).values())
if primary and not primary - existing_subtitles:
log.debug('All subtitles already fetched for %s.' % entry['title'])
sub_item.downloaded = True
continue
except ValueError as e:
log.error('Invalid video file: %s. Removing %s from queue.' % (e, entry['title']))
session.delete(sub_item)
continue
except ImportError:
log.debug('Falling back to simple check since Subliminal is not installed.')
# use glob since subliminal is not there
path_no_ext = os.path.splitext(normalize_path(file))[0]
# can only check subtitles that have explicit language codes in the file name
if primary:
files = glob.glob(path_no_ext + "*")
files = [item.lower() for item in files]
for lang in primary:
if not any('%s.%s' % (path_no_ext, lang) and
f.lower().endswith(SUBTITLE_EXTENSIONS) for f in files):
break
else:
log.debug('All subtitles already fetched for %s.' % entry['title'])
sub_item.downloaded = True
continue
entry.on_complete(self.complete, path=path, task=task)
entries.append(entry)
log.debug('Emitting entry for %s.' % entry['title'])
return entries
def on_task_output(self, task, config):
"""
Configuration::
subliminal:
languages: List of languages (as IETF codes) in order of preference. At least one is required.
alternatives: List of second-choice languages; subs will be downloaded but entries rejected.
exact_match: Use file hash only to search for subs, otherwise Subliminal will try to guess by filename.
providers: List of providers from where to download subtitles.
single: Download subtitles in single mode (no language code added to subtitle filename).
directory: Path to directory where to save the subtitles, default is next to the video.
hearing_impaired: Prefer subtitles for the hearing impaired when available
authentication: >
Dictionary of configuration options for different providers.
Keys correspond to provider names, and values are dictionaries, usually specifying `username` and
`password`.
"""
if not task.accepted:
log.debug('nothing accepted, aborting')
return
from babelfish import Language
from dogpile.cache.exception import RegionAlreadyConfigured
import subliminal
from subliminal import scan_video, save_subtitles
from subliminal.cli import MutexLock
from subliminal.core import ARCHIVE_EXTENSIONS, scan_archive, refine, search_external_subtitles
from subliminal.score import episode_scores, movie_scores
from subliminal.video import VIDEO_EXTENSIONS
try:
subliminal.region.configure('dogpile.cache.dbm',
arguments={
'filename': os.path.join(tempfile.gettempdir(), 'cachefile.dbm'),
'lock_factory': MutexLock,
})
except RegionAlreadyConfigured:
pass
# Let subliminal be more verbose if our logger is set to DEBUG
if log.isEnabledFor(logging.DEBUG):
logging.getLogger("subliminal").setLevel(logging.INFO)
else:
logging.getLogger("subliminal").setLevel(logging.CRITICAL)
logging.getLogger("dogpile").setLevel(logging.CRITICAL)
logging.getLogger("enzyme").setLevel(logging.WARNING)
try:
languages = set([Language.fromietf(s) for s in config.get('languages', [])])
alternative_languages = set([Language.fromietf(s) for s in config.get('alternatives', [])])
except ValueError as e:
raise plugin.PluginError(e)
# keep all downloaded subtitles and save to disk when done (no need to write every time)
downloaded_subtitles = collections.defaultdict(list)
providers_list = config.get('providers', None)
provider_configs = config.get('authentication', None)
# test if only one language was provided, if so we will download in single mode
# (aka no language code added to subtitle filename)
# unless we are forced not to by configuration
# if we pass 'yes' for single in configuration but choose more than one language
# we ignore the configuration and add the language code to the
# potentially downloaded files
single_mode = config.get('single', '') and len(languages | alternative_languages) <= 1
hearing_impaired = config.get('hearing_impaired', False)
with subliminal.core.ProviderPool(providers=providers_list, provider_configs=provider_configs) as provider_pool:
for entry in task.accepted:
if 'location' not in entry:
log.warning('Cannot act on entries that do not represent a local file.')
continue
if not os.path.exists(entry['location']):
entry.fail('file not found: %s' % entry['location'])
continue
if '$RECYCLE.BIN' in entry['location']: # ignore deleted files in Windows shares
continue
try:
entry_languages = set(entry.get('subtitle_languages', [])) or languages
if entry['location'].endswith(VIDEO_EXTENSIONS):
video = scan_video(entry['location'])
elif entry['location'].endswith(ARCHIVE_EXTENSIONS):
video = scan_archive(entry['location'])
else:
entry.reject('File extension is not a supported video or archive extension')
continue
# use metadata refiner to get mkv metadata
refiner = ('metadata',)
refine(video, episode_refiners=refiner, movie_refiners=refiner)
existing_subtitles = set(search_external_subtitles(entry['location']).values())
video.subtitle_languages |= existing_subtitles
if isinstance(video, subliminal.Episode):
title = video.series
hash_scores = episode_scores['hash']
else:
title = video.title
hash_scores = movie_scores['hash']
log.info('Name computed for %s was %s', entry['location'], title)
msc = hash_scores if config['exact_match'] else 0
if entry_languages.issubset(video.subtitle_languages):
log.debug('All preferred languages already exist for "%s"', entry['title'])
entry['subtitles_missing'] = set()
continue # subs for preferred lang(s) already exists
else:
# Gather the subtitles for the alternative languages too, to avoid needing to search the sites
# again. They'll just be ignored if the main languages are found.
all_subtitles = provider_pool.list_subtitles(video, entry_languages | alternative_languages)
subtitles = provider_pool.download_best_subtitles(all_subtitles, video, entry_languages,
min_score=msc,
hearing_impaired=hearing_impaired)
if subtitles:
downloaded_subtitles[video].extend(subtitles)
log.info('Subtitles found for %s', entry['location'])
else:
# only try to download for alternatives that aren't alread downloaded
subtitles = provider_pool.download_best_subtitles(all_subtitles, video,
alternative_languages, min_score=msc,
hearing_impaired=hearing_impaired)
if subtitles:
downloaded_subtitles[video].extend(subtitles)
entry.reject('subtitles found for a second-choice language.')
else:
entry.reject('cannot find any subtitles for now.')
downloaded_languages = set([Language.fromietf(str(l.language))
for l in subtitles])
if entry_languages:
entry['subtitles_missing'] = entry_languages - downloaded_languages
if len(entry['subtitles_missing']) > 0:
entry.reject('Subtitles for all primary languages not found')
except ValueError as e:
log.error('subliminal error: %s', e)
entry.fail()
if downloaded_subtitles:
if task.options.test:
log.verbose('Test mode. Found subtitles:')
# save subtitles to disk
for video, subtitle in downloaded_subtitles.items():
if subtitle:
_directory = config.get('directory')
if _directory:
_directory = os.path.expanduser(_directory)
if task.options.test:
log.verbose(' FOUND LANGUAGES %s for %s', [str(l.language) for l in subtitle], video.name)
continue
save_subtitles(video, subtitle, single=single_mode, directory=_directory)
def store_subtitles(file):
logging.debug('BAZARR started subtitles indexing for this file: ' + file)
actual_subtitles = []
if os.path.exists(file):
if os.path.splitext(file)[1] == '.mkv':
logging.debug("BAZARR is trying to index embedded subtitles.")
try:
with open(file, 'rb') as f:
mkv = enzyme.MKV(f)
for subtitle_track in mkv.subtitle_tracks:
try:
if alpha2_from_alpha3(subtitle_track.language) != None:
lang = str(
alpha2_from_alpha3(subtitle_track.language))
logging.debug(
"BAZARR embedded subtitles detected: " + lang)
actual_subtitles.append([lang, None])
except:
logging.debug(
"BAZARR unable to index this unrecognized language: "
+ subtitle_track.language)
pass
except Exception as e:
logging.exception(
"BAZARR error when trying to analyze this mkv file: " +
file)
pass
else:
logging.debug("BAZARR This file isn't an .mkv file.")
brazilian_portuguese = [".pt-br", ".pob", "pb"]
try:
subtitles = core.search_external_subtitles(file)
except Exception as e:
logging.exception("BAZARR unable to index external subtitles.")
pass
else:
for subtitle, language in subtitles.iteritems():
if str(os.path.splitext(subtitle)[0]).lower().endswith(
tuple(brazilian_portuguese)) is True:
logging.debug("BAZARR external subtitles detected: " +
"pb")
actual_subtitles.append([
str("pb"),
path_replace_reverse(
os.path.join(os.path.dirname(file), subtitle))
])
elif str(language) != 'und':
logging.debug("BAZARR external subtitles detected: " +
str(language))
actual_subtitles.append([
str(language),
path_replace_reverse(
os.path.join(os.path.dirname(file), subtitle))
])
else:
if os.path.splitext(subtitle)[1] != ".sub":
logging.debug(
"BAZARR falling back to file content analysis to detect language."
)
with open(
path_replace(
os.path.join(os.path.dirname(file),
subtitle)), 'r') as f:
text = list(islice(f, 100))
text = ' '.join(text)
encoding = UnicodeDammit(text)
try:
text = text.decode(encoding.original_encoding)
detected_language = langdetect.detect(text)
except Exception as e:
logging.exception(
'BAZARR Error trying to detect language for this subtitles file: '
+ path_replace(
os.path.join(os.path.dirname(file),
subtitle)) +
' You should try to delete this subtitles file manually and ask Bazarr to download it again.'
)
else:
if len(detected_language) > 0:
logging.debug(
"BAZARR external subtitles detected and analysis guessed this language: "
+ str(detected_language))
actual_subtitles.append([
str(detected_language),
path_replace_reverse(
os.path.join(
os.path.dirname(file),
subtitle))
])
conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
logging.debug("BAZARR storing those languages to DB: " +
str(actual_subtitles))
c_db.execute("UPDATE table_episodes SET subtitles = ? WHERE path = ?",
(str(actual_subtitles), path_replace_reverse(file)))
conn_db.commit()
c_db.close()
else:
logging.debug(
"BAZARR this file doesn't seems to exist or isn't accessible.")
logging.debug('BAZARR ended subtitles indexing for this file: ' + file)
return actual_subtitles
def get_video(tv_episode, video_path, subtitles_dir=None, subtitles=True, embedded_subtitles=None, release_name=None):
"""Return the subliminal video for the given path.
The video_path is used as a key to cache the video to avoid
scanning and parsing the video metadata all the time
:param tv_episode:
:type tv_episode: medusa.tv.Episode
:param video_path: the video path
:type video_path: str
:param subtitles_dir: the subtitles directory
:type subtitles_dir: str or None
:param subtitles: True if existing external subtitles should be taken into account
:type subtitles: bool or None
:param embedded_subtitles: True if embedded subtitles should be taken into account
:type embedded_subtitles: bool or None
:param release_name: the release name
:type release_name: str or None
:return: video
:rtype: subliminal.video.Video
"""
key = video_key.format(video_path=video_path)
payload = {'subtitles_dir': subtitles_dir, 'subtitles': subtitles, 'embedded_subtitles': embedded_subtitles,
'release_name': release_name}
cached_payload = memory_cache.get(key, expiration_time=VIDEO_EXPIRATION_TIME)
if cached_payload != NO_VALUE and {k: v for k, v in iteritems(cached_payload) if k != 'video'} == payload:
logger.debug(u'Found cached video information under key %s', key)
return cached_payload['video']
video_is_mkv = video_path.endswith('.mkv')
subtitles_dir = subtitles_dir or get_subtitles_dir(video_path)
logger.debug(u'Scanning video %s...', video_path)
try:
video = scan_video(video_path)
except ValueError as error:
logger.warning(u'Unable to scan video: %s. Error: %r', video_path, error)
else:
# Add hash of our custom provider Itasa
video.size = os.path.getsize(video_path)
if video.size > 10485760:
video.hashes['itasa'] = hash_itasa(video_path)
# external subtitles
if subtitles:
video.subtitle_languages |= set(search_external_subtitles(video_path, directory=subtitles_dir).values())
if embedded_subtitles is None:
embedded_subtitles = bool(not app.IGNORE_EMBEDDED_SUBS and video_is_mkv)
refine(video, episode_refiners=episode_refiners, embedded_subtitles=embedded_subtitles,
release_name=release_name, tv_episode=tv_episode)
video.alternative_series = list(tv_episode.series.aliases)
payload['video'] = video
memory_cache.set(key, payload)
logger.debug(u'Video information cached under key %s', key)
return video
