def get_lang_list(self, provider=None):
# advanced settings
if provider and self.advanced.providers and provider in self.advanced.providers:
adv_languages = self.advanced.providers[provider].get(
"languages", None)
if adv_languages:
adv_out = set()
for adv_lang in adv_languages:
adv_lang = adv_lang.strip()
try:
real_lang = Language.fromietf(adv_lang)
except:
try:
real_lang = Language.fromname(adv_lang)
except:
continue
adv_out.update({real_lang})
# fallback to default languages if no valid language was found in advanced settings
if adv_out:
return adv_out
l = {Language.fromietf(Prefs["langPref1a"])}
lang_custom = Prefs["langPrefCustom"].strip()
if Prefs['subtitles.only_one']:
return l
if Prefs["langPref2a"] != "None":
try:
l.update({Language.fromietf(Prefs["langPref2a"])})
except:
pass
if Prefs["langPref3a"] != "None":
try:
l.update({Language.fromietf(Prefs["langPref3a"])})
except:
pass
if len(lang_custom) and lang_custom != "None":
for lang in lang_custom.split(u","):
lang = lang.strip()
try:
real_lang = Language.fromietf(lang)
except:
try:
real_lang = Language.fromname(lang)
except:
continue
l.update({real_lang})
return l
def ManageBlacklistMenu(**kwargs):
oc = SubFolderObjectContainer(title2=unicode(kwargs["title"]),
replace_parent=True)
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = kwargs["language"]
remove_sub_key = kwargs.pop("remove_sub_key", None)
current_data = unicode(kwargs["current_data"])
current_sub, stored_subs, storage = get_current_sub(
rating_key, part_id, language)
current_bl, subs = stored_subs.get_blacklist(part_id, language)
if remove_sub_key:
remove_sub_key = tuple(remove_sub_key.split("__"))
stored_subs.blacklist(part_id, language, remove_sub_key, add=False)
storage.save(stored_subs)
Log.Info("Removed %s from blacklist", remove_sub_key)
kwargs.pop("randomize")
oc.add(
DirectoryObject(key=Callback(ItemDetailsMenu,
rating_key=kwargs["rating_key"],
item_title=kwargs["item_title"],
title=kwargs["title"],
randomize=timestamp()),
title=_(u"< Back to %s", kwargs["title"]),
summary=current_data,
thumb=default_thumb))
def sorter(pair):
# thanks RestrictedModule parser for messing with lambda (x, y)
return pair[1]["date_added"]
for sub_key, data in sorted(current_bl.iteritems(),
key=sorter,
reverse=True):
provider_name, subtitle_id = sub_key
title = _(
u"%(provider_name)s, %(subtitle_id)s (added: %(date_added)s, %(mode)s), Language: %(language)s, "
u"Score: %(score)i, Storage: %(storage_type)s",
provider_name=provider_name,
subtitle_id=subtitle_id,
date_added=df(data["date_added"]),
mode=current_sub.get_mode_verbose(data["mode"]),
language=display_language(Language.fromietf(language)),
score=data["score"],
storage_type=data["storage_type"])
oc.add(
DirectoryObject(key=Callback(ManageBlacklistMenu,
remove_sub_key="__".join(sub_key),
randomize=timestamp(),
**kwargs),
title=title,
summary=_(u"Remove subtitle from blacklist")))
storage.destroy()
return oc
def BlacklistAllPartsSubtitleMenu(**kwargs):
rating_key = kwargs.get("rating_key")
language = kwargs.get("language")
if language:
language = Language.fromietf(language)
item = get_item(rating_key)
if not item:
return
item_title = get_item_title(item)
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load_or_new(item)
for part_id, languages in stored_subs.parts.iteritems():
sub_dict = languages
if language:
key = str(language)
if key not in sub_dict:
continue
sub_dict = {key: sub_dict[key]}
for language, subs in sub_dict.iteritems():
if "current" in subs:
stored_subs.blacklist(part_id, language, subs["current"])
Log.Info("Added %s to blacklist", subs["current"])
subtitle_storage.save(stored_subs)
subtitle_storage.destroy()
return RefreshItem(rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(), timeout=30000)
def query(self, languages, video):
query = "{} {} {}".format(video.series, video.season, video.episode)
logger.debug("Searching subtitles: {}".format(query))
results = self.search(video.series, str(video.season),
str(video.episode))
if results:
subtitles = []
for i in results:
matches = set()
# self.search only returns results for the specific episode
matches.add("title")
matches.add("series")
matches.add("season")
matches.add("episode")
matches.add("year")
subtitles.append(
TuSubtituloSubtitle(
Language.fromietf(i["language"]),
i["metadata"],
i["download_url"],
i["episode_url"],
matches,
))
return subtitles
else:
logger.debug("No subtitles found")
return []
def SelectStoredSubForItemMenu(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = Language.fromietf(kwargs["language"])
item_type = kwargs["item_type"]
sub_key = tuple(kwargs.pop("sub_key").split("__"))
plex_item = get_item(rating_key)
storage = get_subtitle_storage()
stored_subs = storage.load(plex_item.rating_key)
subtitles = stored_subs.get_all(part_id, language)
subtitle = subtitles[sub_key]
save_stored_sub(subtitle, rating_key, part_id, language, item_type, plex_item=plex_item, storage=storage,
stored_subs=stored_subs)
stored_subs.set_current(part_id, language, sub_key)
storage.save(stored_subs)
storage.destroy()
kwa = {
"header": _("Success"),
"message": _("Subtitle saved to disk"),
"title": kwargs["title"],
"item_title": kwargs["item_title"],
"base_title": kwargs.get("base_title")
}
# fixme: return to SubtitleOptionsMenu properly? (needs recomputation of current_data
return ItemDetailsMenu(rating_key, randomize=timestamp(), **kwa)
def SelectStoredSubForItemMenu(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = Language.fromietf(kwargs["language"])
item_type = kwargs["item_type"]
sub_key = tuple(kwargs.pop("sub_key").split("__"))
plex_item = get_item(rating_key)
storage = get_subtitle_storage()
stored_subs = storage.load(plex_item.rating_key)
subtitles = stored_subs.get_all(part_id, language)
subtitle = subtitles[sub_key]
subtitles["current"] = sub_key
save_stored_sub(subtitle, rating_key, part_id, language, item_type, plex_item=plex_item, storage=storage,
stored_subs=stored_subs)
storage.destroy()
kwargs.pop("randomize")
kwargs["header"] = 'Success'
kwargs["message"] = 'Subtitle saved to disk'
return SubtitleOptionsMenu(randomize=timestamp(), **kwargs)
def ListStoredSubsForItemMenu(**kwargs):
oc = SubFolderObjectContainer(title2=unicode(kwargs["title"]), replace_parent=True)
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = Language.fromietf(kwargs["language"])
current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
all_subs = stored_subs.get_all(part_id, language)
kwargs.pop("randomize")
for key, subtitle in sorted(filter(lambda x: x[0] != "current", all_subs.items()),
key=lambda x: x[1].date_added, reverse=True):
is_current = key == all_subs["current"]
summary = u"added: %s, %s, Language: %s, Score: %i, Storage: %s" % \
(df(subtitle.date_added),
subtitle.mode_verbose, display_language(language), subtitle.score,
subtitle.storage_type)
sub_name = subtitle.provider_name
if sub_name == "embedded":
sub_name += " (%s)" % subtitle.id
oc.add(DirectoryObject(
key=Callback(SelectStoredSubForItemMenu, randomize=timestamp(), sub_key="__".join(key), **kwargs),
title=u"%s%s, Score: %s" % ("Current: " if is_current else "Stored: ", sub_name,
subtitle.score),
summary=summary
))
return oc
def guess_external_subtitles(dest_folder, subtitles):
for subtitle, language in subtitles.iteritems():
if not language:
subtitle_path = os.path.join(dest_folder, subtitle)
if os.path.exists(subtitle_path) and os.path.splitext(
subtitle_path)[1] in core.SUBTITLE_EXTENSIONS:
logging.debug(
"BAZARR falling back to file content analysis to detect language."
)
detected_language = None
with open(subtitle_path, 'r') as f:
text = ' '.join(list(islice(f, 100)))
try:
encoding = UnicodeDammit(text)
text = text.decode(encoding.original_encoding)
detected_language = langdetect.detect(text)
except Exception as e:
logging.exception(
'BAZARR Error trying to detect language for this subtitles file: '
+ subtitle_path +
' You should try to delete this subtitles file manually and ask '
'Bazarr to download it again.')
else:
if detected_language:
logging.debug(
"BAZARR external subtitles detected and guessed this language: "
+ str(detected_language))
try:
subtitles[subtitle] = Language.rebuild(
Language.fromietf(detected_language))
except:
pass
return subtitles
def BlacklistAllPartsSubtitleMenu(**kwargs):
rating_key = kwargs.get("rating_key")
language = kwargs.get("language")
if language:
language = Language.fromietf(language)
item = get_item(rating_key)
if not item:
return
item_title = get_item_title(item)
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load_or_new(item)
for part_id, languages in stored_subs.parts.iteritems():
sub_dict = languages
if language:
key = str(language)
if key not in sub_dict:
continue
sub_dict = {key: sub_dict[key]}
for language, subs in sub_dict.iteritems():
if "current" in subs:
stored_subs.blacklist(part_id, language, subs["current"])
Log.Info("Added %s to blacklist", subs["current"])
subtitle_storage.save(stored_subs)
subtitle_storage.destroy()
return RefreshItem(rating_key=rating_key, item_title=item_title, force=True, randomize=timestamp(), timeout=30000)
def _search_external_subtitles(path, languages=None, only_one=False, scandir_generic=False):
dirpath, filename = os.path.split(path)
dirpath = dirpath or '.'
fileroot, fileext = os.path.splitext(filename)
subtitles = {}
_scandir = _scandir_generic if scandir_generic else scandir
for entry in _scandir(dirpath):
if not entry.name and not scandir_generic:
logger.debug('Could not determine the name of the file, retrying with scandir_generic')
return _search_external_subtitles(path, languages, only_one, True)
if not entry.is_file(follow_symlinks=False):
continue
p = entry.name
# keep only valid subtitle filenames
if not p.lower().startswith(fileroot.lower()) or not p.lower().endswith(SUBTITLE_EXTENSIONS):
continue
p_root, p_ext = os.path.splitext(p)
if not INCLUDE_EXOTIC_SUBS and p_ext not in (".srt", ".ass", ".ssa", ".vtt"):
continue
# extract potential forced/normal/default tag
# fixme: duplicate from subtitlehelpers
split_tag = p_root.rsplit('.', 1)
adv_tag = None
if len(split_tag) > 1:
adv_tag = split_tag[1].lower()
if adv_tag in ['forced', 'normal', 'default', 'embedded', 'embedded-forced', 'custom']:
p_root = split_tag[0]
forced = False
if adv_tag:
forced = "forced" in adv_tag
# extract the potential language code
language_code = p_root[len(fileroot):].replace('_', '-')[1:]
# default language is undefined
language = Language('und')
# attempt to parse
if language_code:
try:
language = Language.fromietf(language_code)
language.forced = forced
except ValueError:
logger.error('Cannot parse language code %r', language_code)
language = None
elif not language_code and only_one:
language = Language.rebuild(list(languages)[0], forced=forced)
subtitles[p] = language
logger.debug('Found subtitles %r', subtitles)
return subtitles
def _search_external_subtitles(path, languages=None, only_one=False, scandir_generic=False):
dirpath, filename = os.path.split(path)
dirpath = dirpath or '.'
fileroot, fileext = os.path.splitext(filename)
subtitles = {}
_scandir = _scandir_generic if scandir_generic else scandir
for entry in _scandir(dirpath):
if not entry.name and not scandir_generic:
logger.debug('Could not determine the name of the file, retrying with scandir_generic')
return _search_external_subtitles(path, languages, only_one, True)
if not entry.is_file(follow_symlinks=False):
continue
p = entry.name
# keep only valid subtitle filenames
if not p.lower().startswith(fileroot.lower()) or not p.lower().endswith(SUBTITLE_EXTENSIONS):
continue
p_root, p_ext = os.path.splitext(p)
if not INCLUDE_EXOTIC_SUBS and p_ext not in (".srt", ".ass", ".ssa", ".vtt"):
continue
# extract potential forced/normal/default tag
# fixme: duplicate from subtitlehelpers
split_tag = p_root.rsplit('.', 1)
adv_tag = None
if len(split_tag) > 1:
adv_tag = split_tag[1].lower()
if adv_tag in ['forced', 'normal', 'default', 'embedded', 'embedded-forced', 'custom']:
p_root = split_tag[0]
forced = False
if adv_tag:
forced = "forced" in adv_tag
# extract the potential language code
language_code = p_root[len(fileroot):].replace('_', '-')[1:]
# default language is undefined
language = Language('und')
# attempt to parse
if language_code:
try:
language = Language.fromietf(language_code)
language.forced = forced
except ValueError:
logger.error('Cannot parse language code %r', language_code)
language = None
elif not language_code and only_one:
language = Language.rebuild(list(languages)[0], forced=forced)
subtitles[p] = language
logger.debug('Found subtitles %r', subtitles)
return subtitles
def apply_default_mods(reapply_current=False, scandir_generic=False):
storage = get_subtitle_storage()
subs_applied = 0
try:
for fn in storage.get_all_files(scandir_generic=scandir_generic):
data = storage.load(None, filename=fn)
if data:
video_id = data.video_id
item_type = get_item_kind_from_rating_key(video_id)
if not item_type:
continue
for part_id, part in data.parts.iteritems():
for lang, subs in part.iteritems():
current_sub = subs.get("current")
if not current_sub:
continue
sub = subs[current_sub]
if not sub.content:
continue
current_mods = sub.mods or []
if not reapply_current:
add_mods = list(
set(config.default_mods).difference(
set(current_mods)))
if not add_mods:
continue
else:
if not current_mods:
continue
add_mods = []
try:
set_mods_for_part(video_id,
part_id,
Language.fromietf(lang),
item_type,
add_mods,
mode="add")
except:
Log.Error("Couldn't set mods for %s:%s: %s",
video_id, part_id,
traceback.format_exc())
continue
subs_applied += 1
except OSError:
return apply_default_mods(reapply_current=reapply_current,
scandir_generic=True)
storage.destroy()
Log.Debug("Applied mods to %i items" % subs_applied)
def query(self, languages, video):
# query the server
result = None
year = (" (%d)" % video.year) if video.year else ""
q = "%s%s %dx%02d" % (video.series, year, video.season, video.episode)
logger.debug('Searching subtitles "%s"', q)
res = self.session.get(
server_url + 'search/query', params={'q':q}, timeout=10)
res.raise_for_status()
result = res.json()
subtitles = []
for s in [s for s in result if len(s['episodes'])]:
for e in s['episodes']:
res = self.session.get(
server_url + 'episodes/%d' % e['id'], timeout=10)
res.raise_for_status()
html = res.text
for lang_m in re.finditer(r"<div class=\"subtitle_language\">(.*?)<\/div>.*?(?=<div class=\"subtitle_language\">|<div id=\"subtitle-actions\">)", html, re.S):
lang = lang_m.group(1)
language = "es"
if "English" in lang:
language = "en"
logger.debug('Found subtitles in "%s" language.', language)
for subt_m in re.finditer(r"<div class=\"version_name\">(.*?)</div>.*?<a href=\"/(subtitles/\d+/download)\" rel=\"nofollow\">(?:.*?<div class=\"version_comments ?\">.*?</i>(.*?)</p>)?", lang_m.group(0), re.S):
matches = set()
if video.alternative_series is None:
if video.series.lower() == s['name'].lower():
matches.add('series')
elif s['name'].lower() in [video.series.lower()]+map(lambda name: name.lower(), video.alternative_series):
matches.add('series')
if video.season == e['season']:
matches.add('season')
if video.episode == e['number']:
matches.add('episode')
if video.title == e['name']:
matches.add('title')
#if video.year is None or ("(%d)" % video.year) in s['name']:
matches.add('year')
subtitles.append(
SubtitulamosTVSubtitle(
Language.fromietf(language),
server_url + 'episodes/%d' % e['id'],
server_url + subt_m.group(2),
subt_m.group(1)+(subt_m.group(3) if not subt_m.group(3) is None else ""),
e['name'],
matches,
'%s %dx%d,%s,%s' % (s['name'], e['season'], e['number'], subt_m.group(1), lang_m.group(1)),
)
)
return subtitles
def get_language_from_stream(lang_code):
if lang_code:
lang = Locale.Language.Match(lang_code)
if lang and lang != "xx":
# Log.Debug("Found language: %r", lang)
return Language.fromietf(lang)
elif lang:
try:
return language_from_stream(lang)
except LanguageError:
pass
def SubtitleReapplyMods(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
lang_a2 = kwargs["language"]
item_type = kwargs["item_type"]
language = Language.fromietf(lang_a2)
set_mods_for_part(rating_key, part_id, language, item_type, [], mode="add")
kwargs.pop("randomize")
return SubtitleModificationsMenu(randomize=timestamp(), **kwargs)
def SubtitleReapplyMods(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
lang_a2 = kwargs["language"]
item_type = kwargs["item_type"]
language = Language.fromietf(lang_a2)
set_mods_for_part(rating_key, part_id, language, item_type, [], mode="add")
kwargs.pop("randomize")
return SubtitleModificationsMenu(randomize=timestamp(), **kwargs)
def guess_external_subtitles(dest_folder, subtitles):
for subtitle, language in six.iteritems(subtitles):
if not language:
subtitle_path = os.path.join(dest_folder, subtitle)
if os.path.exists(subtitle_path) and os.path.splitext(
subtitle_path)[1] in core.SUBTITLE_EXTENSIONS:
logging.debug(
"BAZARR falling back to file content analysis to detect language."
)
detected_language = None
# to improve performance, skip detection of files larger that 5M
if os.path.getsize(subtitle_path) > 5 * 1024 * 1024:
logging.debug(
"BAZARR subtitles file is too large to be text based. Skipping this file: "
+ subtitle_path)
continue
with open(subtitle_path, 'rb') as f:
text = f.read()
try:
# to improve performance, use only the first 32K to detect encoding
guess = chardet.detect(text[:32768])
logging.debug('BAZARR detected encoding %r', guess)
if guess["confidence"] < 0.6:
raise UnicodeError
if guess["confidence"] < 0.8 or guess[
"encoding"] == "ascii":
guess["encoding"] = "utf-8"
text = text.decode(guess["encoding"])
detected_language = guess_language(text)
except UnicodeError:
logging.exception(
"BAZARR subtitles file doesn't seems to be text based. Skipping this file: "
+ subtitle_path)
except:
logging.exception(
'BAZARR Error trying to detect language for this subtitles file: '
+ subtitle_path +
' You should try to delete this subtitles file manually and ask '
'Bazarr to download it again.')
else:
if detected_language:
logging.debug(
"BAZARR external subtitles detected and guessed this language: "
+ str(detected_language))
try:
subtitles[subtitle] = Language.rebuild(
Language.fromietf(detected_language))
except:
pass
return subtitles
def ManageBlacklistMenu(**kwargs):
oc = SubFolderObjectContainer(title2=unicode(kwargs["title"]), replace_parent=True)
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = kwargs["language"]
remove_sub_key = kwargs.pop("remove_sub_key", None)
current_data = unicode(kwargs["current_data"])
current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
current_bl, subs = stored_subs.get_blacklist(part_id, language)
if remove_sub_key:
remove_sub_key = tuple(remove_sub_key.split("__"))
stored_subs.blacklist(part_id, language, remove_sub_key, add=False)
storage.save(stored_subs)
Log.Info("Removed %s from blacklist", remove_sub_key)
kwargs.pop("randomize")
oc.add(DirectoryObject(
key=Callback(ItemDetailsMenu, rating_key=kwargs["rating_key"], item_title=kwargs["item_title"],
title=kwargs["title"], randomize=timestamp()),
title=_(u"< Back to %s", kwargs["title"]),
summary=current_data,
thumb=default_thumb
))
def sorter(pair):
# thanks RestrictedModule parser for messing with lambda (x, y)
return pair[1]["date_added"]
for sub_key, data in sorted(current_bl.iteritems(), key=sorter, reverse=True):
provider_name, subtitle_id = sub_key
title = _(u"%(provider_name)s, %(subtitle_id)s (added: %(date_added)s, %(mode)s), Language: %(language)s, "
u"Score: %(score)i, Storage: %(storage_type)s",
provider_name=_(provider_name),
subtitle_id=subtitle_id,
date_added=df(data["date_added"]),
mode=_(current_sub.get_mode_verbose(data["mode"])),
language=display_language(Language.fromietf(language)),
score=data["score"],
storage_type=data["storage_type"])
oc.add(DirectoryObject(
key=Callback(ManageBlacklistMenu, remove_sub_key="__".join(sub_key), randomize=timestamp(), **kwargs),
title=title,
summary=_(u"Remove subtitle from blacklist")
))
storage.destroy()
return oc
def query(self, movie_id, title, year):
# get the season list of the show
logger.info('Getting the subtitle list of show id %s', movie_id)
if movie_id:
page_link = self.server_url + '/' + movie_id
else:
page_link = self.server_url + self.search_url.format(' '.join(
[title, str(year)]))
r = self.session.get(page_link, timeout=10)
r.raise_for_status()
if not r.content:
logger.debug('No data returned from provider')
return []
soup = ParserBeautifulSoup(r.content, ['html.parser'])
year = None
year_element = soup.select_one('td#dates_header > table div')
matches = False
if year_element:
matches = year_re.match(str(year_element.contents[2]).strip())
if matches:
year = int(matches.group(1))
title_tag = soup.select_one('td#dates_header > table u')
show_title = str(title_tag.contents[0]).strip() if title_tag else None
subtitles = []
# loop over episode rows
for subs_tag in soup.select('.movie-details'):
# read common info
version = subs_tag.find('span').text
download_link = self.server_url + subs_tag.find('a')['href']
uploader = subs_tag.select_one('.movie-info').find('p').find(
'a').text
language_code = subs_tag.select_one('.sprite')['class'][1].split(
'gif')[0]
language = Language.fromietf(language_code)
subtitle = self.subtitle_class(language, page_link, show_title,
year, version, download_link,
uploader)
logger.debug('Found subtitle {!r}'.format(subtitle))
subtitles.append(subtitle)
return subtitles
def SubtitleSetMods(mods=None, mode=None, **kwargs):
if not isinstance(mods, types.ListType) and mods:
mods = [mods]
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
lang_a2 = kwargs["language"]
item_type = kwargs["item_type"]
language = Language.fromietf(lang_a2)
set_mods_for_part(rating_key, part_id, language, item_type, mods, mode=mode)
kwargs.pop("randomize")
return SubtitleModificationsMenu(randomize=timestamp(), **kwargs)
def SubtitleSetMods(mods=None, mode=None, **kwargs):
if not isinstance(mods, types.ListType) and mods:
mods = [mods]
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
lang_a2 = kwargs["language"]
item_type = kwargs["item_type"]
language = Language.fromietf(lang_a2)
set_mods_for_part(rating_key, part_id, language, item_type, mods, mode=mode)
kwargs.pop("randomize")
return SubtitleModificationsMenu(randomize=timestamp(), **kwargs)
def get_missing_languages(video, part):
languages = set([Language.fromietf(str(l)) for l in config.lang_list])
# should we treat IETF as alpha3? (ditch the country part)
alpha3_map = {}
if config.ietf_as_alpha3:
for language in languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
if not Prefs['subtitles.save.filesystem']:
# scan for existing metadata subtitles
meta_subs = get_subtitles_from_metadata(part)
for language, subList in meta_subs.iteritems():
if subList:
video.subtitle_languages.add(language)
Log.Debug("Found metadata subtitle %s for %s", language, video)
have_languages = video.subtitle_languages.copy()
if config.ietf_as_alpha3:
for language in have_languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
missing_languages = (set(str(l) for l in languages) -
set(str(l) for l in have_languages))
# all languages are found if we either really have subs for all languages or we only want to have exactly one language
# and we've only found one (the case for a selected language, Prefs['subtitles.only_one'] (one found sub matches any language))
found_one_which_is_enough = len(
video.subtitle_languages) >= 1 and Prefs['subtitles.only_one']
if not missing_languages or found_one_which_is_enough:
if found_one_which_is_enough:
Log.Debug(
'Only one language was requested, and we\'ve got a subtitle for %s',
video)
else:
Log.Debug('All languages %r exist for %s', languages, video)
return False
# re-add country codes to the missing languages, in case we've removed them above
if config.ietf_as_alpha3:
for language in languages:
language.country = alpha3_map.get(language.alpha3, None)
return missing_languages
def download_best_subtitles(video_part_map,
min_score=0,
throttle_time=None,
providers=None):
hearing_impaired = Prefs['subtitles.search.hearingImpaired']
languages = set([Language.fromietf(str(l)) for l in config.lang_list])
if not languages:
return
use_videos = []
for video, part in video_part_map.iteritems():
if not video.ignore_all:
missing_languages = get_missing_languages(video, part)
else:
missing_languages = languages
if missing_languages:
Log.Info(u"%s has missing languages: %s",
os.path.basename(video.name), missing_languages)
refine_video(video, refiner_settings=config.refiner_settings)
use_videos.append(video)
# prepare blacklist
blacklist = get_blacklist_from_part_map(video_part_map, languages)
if use_videos:
Log.Debug(
"Download best subtitles using settings: min_score: %s, hearing_impaired: %s, languages: %s"
% (min_score, hearing_impaired, languages))
return subliminal.download_best_subtitles(
set(use_videos),
languages,
min_score,
hearing_impaired,
providers=providers or config.providers,
provider_configs=config.provider_settings,
pool_class=config.provider_pool,
compute_score=compute_score,
throttle_time=throttle_time,
blacklist=blacklist,
throttle_callback=config.provider_throttle,
pre_download_hook=pre_download_hook,
post_download_hook=post_download_hook,
language_hook=language_hook)
Log.Debug("All languages for all requested videos exist. Doing nothing.")
def guess_external_subtitles(dest_folder, subtitles):
for subtitle, language in six.iteritems(subtitles):
if not language:
subtitle_path = os.path.join(dest_folder, subtitle)
if os.path.exists(subtitle_path) and os.path.splitext(
subtitle_path)[1] in core.SUBTITLE_EXTENSIONS:
logging.debug(
"BAZARR falling back to file content analysis to detect language."
)
if is_binary(subtitle_path):
logging.debug(
"BAZARR subtitles file doesn't seems to be text based. Skipping this file: "
+ subtitle_path)
continue
detected_language = None
if six.PY3:
with open(subtitle_path, 'r', errors='ignore') as f:
text = f.read()
else:
with open(subtitle_path, 'r') as f:
text = f.read()
try:
encoding = UnicodeDammit(text)
if six.PY2:
text = text.decode(encoding.original_encoding)
detected_language = guess_language(text)
except Exception as e:
logging.exception(
'BAZARR Error trying to detect language for this subtitles file: '
+ subtitle_path +
' You should try to delete this subtitles file manually and ask '
'Bazarr to download it again.')
else:
if detected_language:
logging.debug(
"BAZARR external subtitles detected and guessed this language: "
+ str(detected_language))
try:
subtitles[subtitle] = Language.rebuild(
Language.fromietf(detected_language))
except:
pass
return subtitles
def apply_default_mods(reapply_current=False, scandir_generic=False):
storage = get_subtitle_storage()
subs_applied = 0
try:
for fn in storage.get_all_files(scandir_generic=scandir_generic):
data = storage.load(None, filename=fn)
if data:
video_id = data.video_id
item_type = get_item_kind_from_rating_key(video_id)
if not item_type:
continue
for part_id, part in data.parts.iteritems():
for lang, subs in part.iteritems():
current_sub = subs.get("current")
if not current_sub:
continue
sub = subs[current_sub]
if not sub.content:
continue
current_mods = sub.mods or []
if not reapply_current:
add_mods = list(set(config.default_mods).difference(set(current_mods)))
if not add_mods:
continue
else:
if not current_mods:
continue
add_mods = []
try:
set_mods_for_part(video_id, part_id, Language.fromietf(lang), item_type, add_mods, mode="add")
except:
Log.Error("Couldn't set mods for %s:%s: %s", video_id, part_id, traceback.format_exc())
continue
subs_applied += 1
except OSError:
return apply_default_mods(reapply_current=reapply_current, scandir_generic=True)
storage.destroy()
Log.Debug("Applied mods to %i items" % subs_applied)
def SelectStoredSubForItemMenu(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = Language.fromietf(kwargs["language"])
item_type = kwargs["item_type"]
sub_key = tuple(kwargs.pop("sub_key").split("__"))
plex_item = get_item(rating_key)
storage = get_subtitle_storage()
stored_subs = storage.load(plex_item.rating_key)
subtitles = stored_subs.get_all(part_id, language)
subtitle = subtitles[sub_key]
save_stored_sub(subtitle,
rating_key,
part_id,
language,
item_type,
plex_item=plex_item,
storage=storage,
stored_subs=stored_subs)
stored_subs.set_current(part_id, language, sub_key)
storage.save(stored_subs)
storage.destroy()
kwa = {
"header": _("Success"),
"message": _("Subtitle saved to disk"),
"title": kwargs["title"],
"item_title": kwargs["item_title"],
"base_title": kwargs.get("base_title")
}
# fixme: return to SubtitleOptionsMenu properly? (needs recomputation of current_data
return ItemDetailsMenu(rating_key, randomize=timestamp(), **kwa)
def query(self, languages, video):
query = f"{video.series} {video.season} {video.episode}"
logger.debug(f"Searching subtitles: {query}")
results = self.search(video.series, str(video.season),
str(video.episode))
if results:
subtitles = []
for sub in results:
matches = set()
# self.search only returns results for the specific episode
matches_ = ("title", "series", "season", "episode", "year")
[matches.add(match) for match in matches_]
subtitles.append(
TuSubtituloSubtitle(
Language.fromietf(sub["language"]),
sub,
matches,
))
return subtitles
logger.debug("No subtitles found")
return []
def ListStoredSubsForItemMenu(**kwargs):
oc = SubFolderObjectContainer(title2=unicode(kwargs["title"]), replace_parent=True)
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = Language.fromietf(kwargs["language"])
current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
all_subs = stored_subs.get_all(part_id, language)
kwargs.pop("randomize")
for key, subtitle in sorted(filter(lambda x: x[0] not in ("current", "blacklist"), all_subs.items()),
key=lambda x: x[1].date_added, reverse=True):
is_current = key == all_subs["current"]
summary = _(u"added: %(date_added)s, %(mode)s, Language: %(language)s, Score: %(score)i, Storage: "
u"%(storage_type)s",
date_added=df(subtitle.date_added),
mode=_(subtitle.mode_verbose),
language=display_language(language),
score=subtitle.score,
storage_type=subtitle.storage_type)
sub_name = subtitle.provider_name
if sub_name == "embedded":
sub_name += " (%s)" % subtitle.id
oc.add(DirectoryObject(
key=Callback(SelectStoredSubForItemMenu, randomize=timestamp(), sub_key="__".join(key), **kwargs),
title=_(u"%(current_state)s%(subtitle_name)s, Score: %(score)s",
current_state=_("Current: ") if is_current else _("Stored: "),
subtitle_name=sub_name,
score=subtitle.score),
summary=summary
))
return oc
def extract_embedded_sub(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs.pop("part_id")
stream_index = kwargs.pop("stream_index")
with_mods = kwargs.pop("with_mods", False)
language = Language.fromietf(kwargs.pop("language"))
refresh = kwargs.pop("refresh", True)
set_current = kwargs.pop("set_current", True)
plex_item = kwargs.pop("plex_item", get_item(rating_key))
item_type = get_item_kind_from_item(plex_item)
part = kwargs.pop("part", get_part(plex_item, part_id))
scanned_videos = kwargs.pop("scanned_videos", None)
extract_mode = kwargs.pop("extract_mode", "a")
any_successful = False
if part:
if not scanned_videos:
metadata = get_plex_metadata(rating_key,
part_id,
item_type,
plex_item=plex_item)
scanned_videos = scan_videos([metadata],
ignore_all=True,
skip_hashing=True)
for stream in part.streams:
# subtitle stream
if str(stream.index) == stream_index:
is_forced = is_stream_forced(stream)
bn = os.path.basename(part.file)
set_refresh_menu_state(
_(u"Extracting subtitle %(stream_index)s of %(filename)s",
stream_index=stream_index,
filename=bn))
Log.Info(u"Extracting stream %s (%s) of %s", stream_index,
str(language), bn)
out_codec = stream.codec if stream.codec != "mov_text" else "srt"
args = [
config.plex_transcoder, "-i", part.file, "-map",
"0:%s" % stream_index, "-f", out_codec, "-"
]
output = None
try:
output = subprocess.check_output(quote_args(args),
stderr=subprocess.PIPE,
shell=True)
except:
Log.Error("Extraction failed: %s", traceback.format_exc())
if output:
subtitle = ModifiedSubtitle(
language,
mods=config.default_mods if with_mods else None)
subtitle.content = output
subtitle.provider_name = "embedded"
subtitle.id = "stream_%s" % stream_index
subtitle.score = 0
subtitle.set_encoding("utf-8")
# fixme: speedup video; only video.name is needed
video = scanned_videos.keys()[0]
save_successful = save_subtitles(scanned_videos,
{video: [subtitle]},
mode="m",
set_current=set_current)
set_refresh_menu_state(None)
if save_successful and refresh:
refresh_item(rating_key)
# add item to history
item_title = get_title_for_video_metadata(
video.plexapi_metadata,
add_section_title=False,
add_episode_title=True)
history = get_history()
history.add(
item_title,
video.id,
section_title=video.plexapi_metadata["section"],
thumb=video.plexapi_metadata["super_thumb"],
subtitle=subtitle,
mode=extract_mode)
history.destroy()
any_successful = True
return any_successful
def list_subtitles(self,
rating_key,
item_type,
part_id,
language,
skip_wrong_fps=True,
metadata=None,
scanned_parts=None,
air_date_cutoff=None):
if not metadata:
metadata = get_plex_metadata(rating_key, part_id, item_type)
if not metadata:
return
providers = config.get_providers(
media_type="series" if item_type == "episode" else "movies")
if not scanned_parts:
scanned_parts = scan_videos([metadata],
ignore_all=True,
providers=providers)
if not scanned_parts:
Log.Error(u"%s: Couldn't list available subtitles for %s",
self.name, rating_key)
return
video, plex_part = scanned_parts.items()[0]
refine_video(video, refiner_settings=config.refiner_settings)
if air_date_cutoff is not None and metadata["item"].year and \
metadata["item"].year + air_date_cutoff < datetime.date.today().year:
Log.Debug(
"Skipping searching for subtitles: %s, it aired over %s year(s) ago.",
rating_key, air_date_cutoff)
return
config.init_subliminal_patches()
provider_settings = config.provider_settings
if not skip_wrong_fps:
provider_settings["opensubtitles"]["skip_wrong_fps"] = False
if item_type == "episode":
min_score = 240
if video.is_special:
min_score = 180
else:
min_score = 60
languages = {Language.fromietf(language)}
available_subs = list_all_subtitles(
[video],
languages,
providers=providers,
provider_configs=provider_settings,
pool_class=config.provider_pool,
throttle_callback=config.provider_throttle,
language_hook=language_hook)
use_hearing_impaired = Prefs['subtitles.search.hearingImpaired'] in (
"prefer", "force HI")
# sort subtitles by score
unsorted_subtitles = []
for s in available_subs[video]:
Log.Debug(u"%s: Starting score computation for %s", self.name, s)
try:
matches = s.get_matches(video)
except AttributeError:
Log.Error(u"%s: Match computation failed for %s: %s",
self.name, s, traceback.format_exc())
continue
# skip wrong season/episodes
if item_type == "episode":
can_verify_series = True
if not s.hash_verifiable and "hash" in matches:
can_verify_series = False
if can_verify_series and not {"series", "season", "episode"
}.issubset(matches):
if "series" not in matches:
s.wrong_series = True
else:
s.wrong_season_ep = True
unsorted_subtitles.append(
(s,
compute_score(matches,
s,
video,
hearing_impaired=use_hearing_impaired),
matches))
scored_subtitles = sorted(unsorted_subtitles,
key=operator.itemgetter(1),
reverse=True)
subtitles = []
for subtitle, score, matches in scored_subtitles:
# check score
if score < min_score and not subtitle.wrong_series:
Log.Info(u'%s: Score %d is below min_score (%d)', self.name,
score, min_score)
continue
subtitle.score = score
subtitle.matches = matches
subtitle.part_id = part_id
subtitle.item_type = item_type
subtitles.append(subtitle)
return subtitles
def get_language(lang_short):
return Language.fromietf(lang_short)
def query(self, languages, title, season=None, episode=None, year=None, video=None):
items_per_page = 10
current_page = 1
used_languages = languages
lang_strings = [str(lang) for lang in used_languages]
# handle possible duplicate use of Serbian Latin
if "sr" in lang_strings and "sr-Latn" in lang_strings:
logger.info('Duplicate entries <Language [sr]> and <Language [sr-Latn]> found, filtering languages')
used_languages = filter(lambda l: l != Language.fromietf('sr-Latn'), used_languages)
logger.info('Filtered language list %r', used_languages)
# convert list of languages into search string
langs = '|'.join(map(str, [l.titlovi for l in used_languages]))
# set query params
params = {'prijevod': title, 'jezik': langs}
is_episode = False
if season and episode:
is_episode = True
params['s'] = season
params['e'] = episode
if year:
params['g'] = year
# loop through paginated results
logger.info('Searching subtitles %r', params)
subtitles = []
while True:
# query the server
try:
r = self.session.get(self.search_url, params=params, timeout=10)
r.raise_for_status()
except RequestException as e:
logger.exception('RequestException %s', e)
break
else:
try:
soup = BeautifulSoup(r.content, 'lxml')
# number of results
result_count = int(soup.select_one('.results_count b').string)
except:
result_count = None
# exit if no results
if not result_count:
if not subtitles:
logger.debug('No subtitles found')
else:
logger.debug("No more subtitles found")
break
# number of pages with results
pages = int(math.ceil(result_count / float(items_per_page)))
# get current page
if 'pg' in params:
current_page = int(params['pg'])
try:
sublist = soup.select('section.titlovi > ul.titlovi > li.subtitleContainer.canEdit')
for sub in sublist:
# subtitle id
sid = sub.find(attrs={'data-id': True}).attrs['data-id']
# get download link
download_link = self.download_url + sid
# title and alternate title
match = title_re.search(sub.a.string)
if match:
_title = match.group('title')
alt_title = match.group('altitle')
else:
continue
# page link
page_link = self.server_url + sub.a.attrs['href']
# subtitle language
match = lang_re.search(sub.select_one('.lang').attrs['src'])
if match:
try:
# decode language
lang = Language.fromtitlovi(match.group('lang')+match.group('script'))
except ValueError:
continue
# relase year or series start year
match = year_re.search(sub.find(attrs={'data-id': True}).parent.i.string)
if match:
r_year = int(match.group('year'))
# fps
match = fps_re.search(sub.select_one('.fps').string)
if match:
fps = match.group('fps')
# releases
releases = str(sub.select_one('.fps').parent.contents[0].string)
# handle movies and series separately
if is_episode:
# season and episode info
sxe = sub.select_one('.s0xe0y').string
r_season = None
r_episode = None
if sxe:
match = season_re.search(sxe)
if match:
r_season = int(match.group('season'))
match = episode_re.search(sxe)
if match:
r_episode = int(match.group('episode'))
subtitle = self.subtitle_class(lang, page_link, download_link, sid, releases, _title,
alt_title=alt_title, season=r_season, episode=r_episode,
year=r_year, fps=fps,
asked_for_release_group=video.release_group,
asked_for_episode=episode)
else:
subtitle = self.subtitle_class(lang, page_link, download_link, sid, releases, _title,
alt_title=alt_title, year=r_year, fps=fps,
asked_for_release_group=video.release_group)
logger.debug('Found subtitle %r', subtitle)
# prime our matches so we can use the values later
subtitle.get_matches(video)
# add found subtitles
subtitles.append(subtitle)
finally:
soup.decompose()
# stop on last page
if current_page >= pages:
break
# increment current page
params['pg'] = current_page + 1
logger.debug('Getting page %d', params['pg'])
return subtitles
class Addic7edProvider(_Addic7edProvider):
languages = {Language('por', 'BR')} | {
Language(l)
for l in [
'ara', 'aze', 'ben', 'bos', 'bul', 'cat', 'ces', 'dan', 'deu',
'ell', 'eng', 'eus', 'fas', 'fin', 'fra', 'glg', 'heb', 'hrv',
'hun', 'hye', 'ind', 'ita', 'jpn', 'kor', 'mkd', 'msa', 'nld',
'nor', 'pol', 'por', 'ron', 'rus', 'slk', 'slv', 'spa', 'sqi',
'srp', 'swe', 'tha', 'tur', 'ukr', 'vie', 'zho'
]
} | {Language.fromietf(l)
for l in ["sr-Latn", "sr-Cyrl"]}
USE_ADDICTED_RANDOM_AGENTS = False
hearing_impaired_verifiable = True
subtitle_class = Addic7edSubtitle
server_url = 'https://www.addic7ed.com/'
sanitize_characters = {'-', ':', '(', ')', '.', '/'}
last_show_ids_fetch_key = "addic7ed_last_id_fetch"
def __init__(self, username=None, password=None, use_random_agents=False):
super(Addic7edProvider, self).__init__(username=username,
password=password)
self.USE_ADDICTED_RANDOM_AGENTS = use_random_agents
if not all((username, password)):
raise ConfigurationError('Username and password must be specified')
def initialize(self):
self.session = Session()
self.session.headers[
'User-Agent'] = 'Subliminal/%s' % subliminal.__short_version__
from .utils import FIRST_THOUSAND_OR_SO_USER_AGENTS as AGENT_LIST
logger.debug("Addic7ed: using random user agents")
self.session.headers['User-Agent'] = AGENT_LIST[randint(
0,
len(AGENT_LIST) - 1)]
self.session.headers['Referer'] = self.server_url
# login
if self.username and self.password:
def check_verification(cache_region):
try:
rr = self.session.get(self.server_url + 'panel.php',
allow_redirects=False,
timeout=10,
headers={"Referer": self.server_url})
if rr.status_code == 302:
logger.info('Addic7ed: Login expired')
cache_region.delete("addic7ed_data")
else:
logger.info('Addic7ed: Re-using old login')
self.logged_in = True
return True
except ConnectionError as e:
logger.debug(
"Addic7ed: There was a problem reaching the server: %s."
% e)
raise IPAddressBlocked(
"Addic7ed: Your IP is temporarily blocked.")
if load_verification("addic7ed",
self.session,
callback=check_verification):
return
logger.info('Addic7ed: Logging in')
data = {
'username': self.username,
'password': self.password,
'Submit': 'Log in',
'url': '',
'remember': 'true'
}
tries = 0
while tries <= 3:
tries += 1
r = self.session.get(self.server_url + 'login.php',
timeout=10,
headers={"Referer": self.server_url})
if "g-recaptcha" in r.text or "grecaptcha" in r.text:
logger.info(
'Addic7ed: Solving captcha. This might take a couple of minutes, but should only '
'happen once every so often')
for g, s in (("g-recaptcha-response",
r'g-recaptcha.+?data-sitekey=\"(.+?)\"'),
("recaptcha_response",
r'grecaptcha.execute\(\'(.+?)\',')):
site_key = re.search(s, r.text).group(1)
if site_key:
break
if not site_key:
logger.error("Addic7ed: Captcha site-key not found!")
return
pitcher = pitchers.get_pitcher()(
"Addic7ed",
self.server_url + 'login.php',
site_key,
user_agent=self.session.headers["User-Agent"],
cookies=self.session.cookies.get_dict(),
is_invisible=True)
result = pitcher.throw()
if not result:
if tries >= 3:
raise Exception(
"Addic7ed: Couldn't solve captcha!")
logger.info(
"Addic7ed: Couldn't solve captcha! Retrying")
continue
data[g] = result
r = self.session.post(
self.server_url + 'dologin.php',
data,
allow_redirects=False,
timeout=10,
headers={"Referer": self.server_url + "login.php"})
if "relax, slow down" in r.text:
raise TooManyRequests(self.username)
if "Wrong password" in r.text or "doesn't exist" in r.text:
raise AuthenticationError(self.username)
if r.status_code != 302:
if tries >= 3:
logger.error(
"Addic7ed: Something went wrong when logging in")
raise AuthenticationError(self.username)
logger.info(
"Addic7ed: Something went wrong when logging in; retrying"
)
continue
break
store_verification("addic7ed", self.session)
logger.debug('Addic7ed: Logged in')
self.logged_in = True
def terminate(self):
self.session.close()
def get_show_id(self,
series,
year=None,
country_code=None,
ignore_cache=False):
"""Get the best matching show id for `series`, `year` and `country_code`.
First search in the result of :meth:`_get_show_ids` and fallback on a search with :meth:`_search_show_id`.
:param str series: series of the episode.
:param year: year of the series, if any.
:type year: int
:param country_code: country code of the series, if any.
:type country_code: str
:return: the show id, if found.
:rtype: int
"""
show_id = None
ids_to_look_for = {
sanitize(series).lower(),
sanitize(series.replace(".", "")).lower(),
sanitize(series.replace("&", "and")).lower()
}
show_ids = self._get_show_ids()
if ignore_cache or not show_ids:
show_ids = self._get_show_ids.refresh(self)
logger.debug("Trying show ids: %s", ids_to_look_for)
for series_sanitized in ids_to_look_for:
# attempt with country
if not show_id and country_code:
logger.debug('Getting show id with country')
show_id = show_ids.get(
'%s %s' % (series_sanitized, country_code.lower()))
# attempt with year
if not show_id and year:
logger.debug('Getting show id with year')
show_id = show_ids.get('%s %d' % (series_sanitized, year))
# attempt clean
if not show_id:
logger.debug('Getting show id')
show_id = show_ids.get(series_sanitized)
if not show_id:
now = datetime.datetime.now()
last_fetch = region.get(self.last_show_ids_fetch_key)
# re-fetch show ids once per day if any show ID not found
if not ignore_cache and last_fetch != NO_VALUE and last_fetch + datetime.timedelta(
days=1) < now:
logger.info("Show id not found; re-fetching show ids")
return self.get_show_id(series,
year=year,
country_code=country_code,
ignore_cache=True)
logger.debug(
"Not refreshing show ids, as the last fetch has been too recent"
)
# search as last resort
# broken right now
# if not show_id:
# logger.warning('Series %s not found in show ids', series)
# show_id = self._search_show_id(series)
return show_id
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
def _get_show_ids(self):
"""Get the ``dict`` of show ids per series by querying the `shows.php` page.
:return: show id per series, lower case and without quotes.
:rtype: dict
# patch: add punctuation cleaning
"""
# get the show page
logger.info('Getting show ids')
region.set(self.last_show_ids_fetch_key, datetime.datetime.now())
r = self.session.get(self.server_url + 'shows.php', timeout=10)
r.raise_for_status()
# LXML parser seems to fail when parsing Addic7ed.com HTML markup.
# Last known version to work properly is 3.6.4 (next version, 3.7.0, fails)
# Assuming the site's markup is bad, and stripping it down to only contain what's needed.
show_cells = re.findall(show_cells_re, r.content)
if show_cells:
soup = ParserBeautifulSoup(
b''.join(show_cells).decode('utf-8', 'ignore'),
['lxml', 'html.parser'])
else:
# If RegEx fails, fall back to original r.text and use 'html.parser'
soup = ParserBeautifulSoup(r.text, ['html.parser'])
# populate the show ids
show_ids = {}
shows = soup.select('td > h3 > a[href^="/show/"]')
for show in shows:
show_clean = sanitize(show.text,
default_characters=self.sanitize_characters)
try:
show_id = int(show['href'][6:])
except ValueError:
continue
show_ids[show_clean] = show_id
match = series_year_re.match(show_clean)
if match and match.group(2) and match.group(1) not in show_ids:
# year found, also add it without year
show_ids[match.group(1)] = show_id
soup.decompose()
soup = None
logger.debug('Found %d show ids', len(show_ids))
if not show_ids:
raise Exception("Addic7ed: No show IDs found!")
return show_ids
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
def _search_show_id(self, series, year=None):
"""Search the show id from the `series` and `year`.
:param str series: series of the episode.
:param year: year of the series, if any.
:type year: int
:return: the show id, if found.
:rtype: int
"""
# addic7ed doesn't support search with quotes
series = series.replace('\'', ' ')
# build the params
series_year = '%s %d' % (series, year) if year is not None else series
params = {'search': series_year, 'Submit': 'Search'}
# make the search
logger.info('Searching show ids with %r', params)
# currently addic7ed searches via srch.php from the front page, then a re-search is needed which calls
# search.php
for endpoint in (
"srch.php",
"search.php",
):
headers = None
if endpoint == "search.php":
headers = {"referer": self.server_url + "srch.php"}
r = self.session.get(self.server_url + endpoint,
params=params,
timeout=10,
headers=headers)
r.raise_for_status()
if r.text and "Sorry, your search" not in r.text:
break
time.sleep(4)
if r.status_code == 304:
raise TooManyRequests()
soup = ParserBeautifulSoup(r.text, ['lxml', 'html.parser'])
suggestion = None
# get the suggestion
try:
suggestion = soup.select('span.titulo > a[href^="/show/"]')
if not suggestion:
logger.warning('Show id not found: no suggestion')
return None
if not sanitize(suggestion[0].i.text.replace('\'', ' '),
default_characters=self.sanitize_characters) == \
sanitize(series_year, default_characters=self.sanitize_characters):
logger.warning('Show id not found: suggestion does not match')
return None
show_id = int(suggestion[0]['href'][6:])
logger.debug('Found show id %d', show_id)
return show_id
finally:
soup.decompose()
soup = None
def query(self, show_id, series, season, year=None, country=None):
# patch: fix logging
# get the page of the season of the show
logger.info('Getting the page of show id %d, season %d', show_id,
season)
r = self.session.get(self.server_url + 'ajax_loadShow.php',
params={
'show': show_id,
'season': season
},
timeout=10,
headers={
"referer":
"%sshow/%s" % (self.server_url, show_id),
"X-Requested-With":
"XMLHttpRequest"
})
r.raise_for_status()
if r.status_code == 304:
raise TooManyRequests()
if not r.text:
# Provider wrongful return a status of 304 Not Modified with an empty content
# raise_for_status won't raise exception for that status code
logger.error('No data returned from provider')
return []
soup = ParserBeautifulSoup(r.text, ['lxml', 'html.parser'])
# loop over subtitle rows
subtitles = []
for row in soup.select('tr.epeven'):
cells = row('td')
# ignore incomplete subtitles
status = cells[5].text
if "%" in status:
logger.debug('Ignoring subtitle with status %s', status)
continue
# read the item
language = Language.fromaddic7ed(cells[3].text)
hearing_impaired = bool(cells[6].text)
page_link = self.server_url + cells[2].a['href'][1:]
season = int(cells[0].text)
episode = int(cells[1].text)
title = cells[2].text
version = cells[4].text
download_link = cells[9].a['href'][1:]
subtitle = self.subtitle_class(language, hearing_impaired,
page_link, series, season, episode,
title, year, version, download_link)
logger.debug('Found subtitle %r', subtitle)
subtitles.append(subtitle)
soup.decompose()
soup = None
return subtitles
def download_subtitle(self, subtitle):
# download the subtitle
r = self.session.get(self.server_url + subtitle.download_link,
headers={'Referer': subtitle.page_link},
timeout=10)
r.raise_for_status()
if r.status_code == 304:
raise TooManyRequests()
if not r.text:
# Provider wrongful return a status of 304 Not Modified with an empty content
# raise_for_status won't raise exception for that status code
logger.error(
'Unable to download subtitle. No data returned from provider')
return
# detect download limit exceeded
if r.headers['Content-Type'] == 'text/html':
raise DownloadLimitExceeded
subtitle.content = fix_line_ending(r.content)
from subzero.language import Language
from subliminal_patch import Subtitle
from subliminal_patch.subtitle import ftfy_defaults
logger = logging.getLogger(__name__)
from subzero.modification import SubMod
fn = sys.argv[1]
debug = "--debug" in sys.argv
if debug:
logging.basicConfig(level=logging.DEBUG)
#sub = Subtitle(Language.fromietf("eng:forced"), mods=["common", "remove_HI", "OCR_fixes", "fix_uppercase", "shift_offset(ms=-500)", "shift_offset(ms=500)", "shift_offset(s=2,ms=800)"])
sub = Subtitle(Language.fromietf("eng:forced"), mods=["common", "remove_HI", "OCR_fixes", "fix_uppercase", "shift_offset(ms=0,s=1)"])
sub.content = open(fn).read()
sub.normalize()
content = sub.get_modified_content(debug=True)
#submod = SubMod(debug=debug)
#submod.load(fn, language=Language.fromietf("pol"), encoding="utf-8")
#submod.modify("OCR_fixes", "common", "remove_tags", "OCR_fixes", "OCR_fixes")
#submod.modify("shift_offset(s=20)", "OCR_fixes")
#submod.modify("remove_HI", "OCR_fixes", "common", "OCR_fixes", "shift_offset(s=20)", "OCR_fixes", "color(name=white)", "shift_offset(s=-5, ms=-350)")
#srt = Subtitle.pysubs2_to_unicode(submod.f)
#content = fix_text(Subtitle.pysubs2_to_unicode(submod.f, format=format), **ftfy_defaults)\
# .encode(encoding="utf-8")
#print submod.f.to_string("srt", encoding="utf-8")
#print repr(content)
def get_language(lang_short):
return Language.fromietf(lang_short)
def prepare_video(pms_video_info, ignore_all=False, hints=None, rating_key=None, providers=None, skip_hashing=False):
"""
returnes a subliminal/guessit-refined parsed video
:param pms_video_info:
:param ignore_all:
:param hints:
:param rating_key:
:return:
"""
embedded_subtitles = not ignore_all and Prefs['subtitles.scan.embedded']
external_subtitles = not ignore_all and Prefs['subtitles.scan.external']
plex_part = pms_video_info["plex_part"]
if ignore_all:
Log.Debug("Force refresh intended.")
Log.Debug("Detecting streams: %s, external_subtitles=%s, embedded_subtitles=%s" % (
plex_part.file, external_subtitles, embedded_subtitles))
known_embedded = []
parts = []
for media in list(Plex["library"].metadata(rating_key))[0].media:
parts += media.parts
plexpy_part = None
for part in parts:
if int(part.id) == int(plex_part.id):
plexpy_part = part
# embedded subtitles
# fixme: skip the whole scanning process if known_embedded == wanted languages?
audio_languages = []
if plexpy_part:
for stream in plexpy_part.streams:
if stream.stream_type == 2:
lang = None
try:
lang = language_from_stream(stream.language_code)
except LanguageError:
Log.Debug("Couldn't detect embedded audio stream language: %s", stream.language_code)
# treat unknown language as lang1?
if not lang and config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
audio_languages.append(lang)
# subtitle stream
elif stream.stream_type == 3 and embedded_subtitles:
is_forced = helpers.is_stream_forced(stream)
if ((config.forced_only or config.forced_also) and is_forced) or not is_forced:
# embedded subtitle
# fixme: tap into external subtitles here instead of scanning for ourselves later?
if stream.codec and getattr(stream, "index", None):
if config.exotic_ext or stream.codec.lower() in config.text_based_formats:
lang = None
try:
lang = language_from_stream(stream.language_code)
except LanguageError:
Log.Debug("Couldn't detect embedded subtitle stream language: %s", stream.language_code)
# treat unknown language as lang1?
if not lang and config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
if lang:
if is_forced:
lang.forced = True
known_embedded.append(lang)
else:
Log.Warn("Part %s missing of %s, not able to scan internal streams", plex_part.id, rating_key)
# metadata subtitles
known_metadata_subs = set()
meta_subs = get_subtitles_from_metadata(plex_part)
for language, subList in meta_subs.iteritems():
try:
lang = Language.fromietf(Locale.Language.Match(language))
except LanguageError:
if config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
else:
continue
if subList:
for key in subList:
if key.startswith("subzero_md_forced"):
lang = Language.rebuild(lang, forced=True)
known_metadata_subs.add(lang)
Log.Debug("Found metadata subtitle %r:%s for %s", lang, key, plex_part.file)
Log.Debug("Known metadata subtitles: %r", known_metadata_subs)
Log.Debug("Known embedded subtitles: %r", known_embedded)
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load(rating_key)
subtitle_storage.destroy()
try:
# get basic video info scan (filename)
video = parse_video(plex_part.file, hints, skip_hashing=config.low_impact_mode or skip_hashing,
providers=providers)
# set stream languages
if audio_languages:
video.audio_languages = audio_languages
Log.Info("Found audio streams: %s" % ", ".join([str(l) for l in audio_languages]))
if not ignore_all:
set_existing_languages(video, pms_video_info, external_subtitles=external_subtitles,
embedded_subtitles=embedded_subtitles, known_embedded=known_embedded,
stored_subs=stored_subs, languages=config.lang_list,
only_one=config.only_one, known_metadata_subs=known_metadata_subs)
# add video fps info
video.fps = plex_part.fps
return video
except ValueError:
Log.Warn("File could not be guessed: %s: %s", plex_part.file, traceback.format_exc())
def query(self, languages, video):
self.video = video
if self.use_hash:
hash = self.video.hashes.get('opensubtitlescom')
logging.debug('Searching using this hash: {}'.format(hash))
else:
hash = None
if isinstance(self.video, Episode):
title = self.video.series
else:
title = self.video.title
title_id = self.search_titles(title)
if not title_id:
return []
lang_strings = [str(lang) for lang in languages]
langs = ','.join(lang_strings)
logging.debug('Searching for this languages: {}'.format(lang_strings))
# query the server
if isinstance(self.video, Episode):
res = self.session.get(self.server_url + 'subtitles',
params={'parent_feature_id': title_id,
'languages': langs,
'episode_number': self.video.episode,
'season_number': self.video.season,
'moviehash': hash},
timeout=10)
else:
res = self.session.get(self.server_url + 'subtitles',
params={'id': title_id,
'languages': langs,
'moviehash': hash},
timeout=10)
res.raise_for_status()
if res.status_code == 429:
raise TooManyRequests()
subtitles = []
try:
result = res.json()
except ValueError:
raise ProviderError('Invalid JSON returned by provider')
else:
logging.debug('Query returned {} subtitles'.format(len(result['data'])))
if len(result['data']):
for item in result['data']:
if 'season_number' in item['attributes']['feature_details']:
season_number = item['attributes']['feature_details']['season_number']
else:
season_number = None
if 'episode_number' in item['attributes']['feature_details']:
episode_number = item['attributes']['feature_details']['episode_number']
else:
episode_number = None
if 'moviehash_match' in item['attributes']:
moviehash_match = item['attributes']['moviehash_match']
else:
moviehash_match = False
if len(item['attributes']['files']):
subtitle = OpenSubtitlesComSubtitle(
language=Language.fromietf(item['attributes']['language']),
hearing_impaired=item['attributes']['hearing_impaired'],
page_link=item['attributes']['url'],
file_id=item['attributes']['files'][0]['file_id'],
releases=item['attributes']['release'],
uploader=item['attributes']['uploader']['name'],
title=item['attributes']['feature_details']['movie_name'],
year=item['attributes']['feature_details']['year'],
season=season_number,
episode=episode_number,
hash_matched=moviehash_match
)
subtitle.get_matches(self.video)
subtitles.append(subtitle)
return subtitles
def query(self, language, keyword, video, season=None, episode=None, year=None, only_foreign=False,
also_foreign=False):
search_language = str(language).lower()
# sr-Cyrl specialcase
if search_language == "sr-cyrl":
search_language = "sr"
# set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652
params = {'sXML': 1, 'sL': search_language, 'sK': keyword}
is_episode = False
if season and episode:
is_episode = True
params['sTS'] = season
params['sTE'] = episode
if year:
params['sY'] = year
# loop over paginated results
logger.info('Searching subtitles %r', params)
subtitles = []
pids = set()
while True:
# query the server
content = None
try:
content = self.session.get(self.server_url + 'search/old', params=params, timeout=10).content
xml = etree.fromstring(content)
except etree.ParseError:
logger.error("Wrong data returned: %r", content)
break
# exit if no results
if not int(xml.find('pagination/results').text):
logger.debug('No subtitles found')
break
# loop over subtitles
for subtitle_xml in xml.findall('subtitle'):
# read xml elements
pid = subtitle_xml.find('pid').text
# ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321
if pid in pids:
continue
_language = Language.fromietf(subtitle_xml.find('language').text)
hearing_impaired = 'n' in (subtitle_xml.find('flags').text or '')
foreign = 'f' in (subtitle_xml.find('flags').text or '')
if only_foreign and not foreign:
continue
elif not only_foreign and not also_foreign and foreign:
continue
elif also_foreign and foreign:
_language = Language.rebuild(_language, forced=True)
if language != _language:
continue
page_link = subtitle_xml.find('url').text
releases = []
if subtitle_xml.find('release').text:
for release in subtitle_xml.find('release').text.split():
releases.append(re.sub(r'\.+$', '', release)) # remove trailing dots
title = subtitle_xml.find('title').text
r_season = int(subtitle_xml.find('tvSeason').text)
r_episode = int(subtitle_xml.find('tvEpisode').text)
r_year = int(subtitle_xml.find('year').text)
if is_episode:
subtitle = self.subtitle_class(_language, hearing_impaired, page_link, pid, releases, title,
season=r_season, episode=r_episode, year=r_year,
asked_for_release_group=video.release_group,
asked_for_episode=episode)
else:
subtitle = self.subtitle_class(_language, hearing_impaired, page_link, pid, releases, title,
year=r_year, asked_for_release_group=video.release_group)
logger.debug('Found subtitle %r', subtitle)
subtitles.append(subtitle)
pids.add(pid)
# stop on last page
if int(xml.find('pagination/current').text) >= int(xml.find('pagination/count').text):
break
# increment current page
params['page'] = int(xml.find('pagination/current').text) + 1
logger.debug('Getting page %d', params['page'])
xml = None
return subtitles
def get_lang_list(self, provider=None, ordered=False):
# advanced settings
if provider and self.advanced.providers and provider in self.advanced.providers:
adv_languages = self.advanced.providers[provider].get("languages", None)
if adv_languages:
adv_out = set()
for adv_lang in adv_languages:
adv_lang = adv_lang.strip()
try:
real_lang = Language.fromietf(adv_lang)
except:
try:
real_lang = Language.fromname(adv_lang)
except:
continue
adv_out.update({real_lang})
# fallback to default languages if no valid language was found in advanced settings
if adv_out:
return adv_out
l = [Language.fromietf(Prefs["langPref1a"])]
lang_custom = Prefs["langPrefCustom"].strip()
if Prefs['subtitles.only_one']:
return set(l) if not ordered else l
if Prefs["langPref2a"] != "None":
try:
l.append(Language.fromietf(Prefs["langPref2a"]))
except:
pass
if Prefs["langPref3a"] != "None":
try:
l.append(Language.fromietf(Prefs["langPref3a"]))
except:
pass
if len(lang_custom) and lang_custom != "None":
for lang in lang_custom.split(u","):
lang = lang.strip()
try:
real_lang = Language.fromietf(lang)
except:
try:
real_lang = Language.fromname(lang)
except:
continue
l.append(real_lang)
if self.forced_also:
if Prefs["subtitles.when_forced"] == "Always":
for lang in list(l):
l.append(Language.rebuild(lang, forced=True))
else:
for (setting, index) in (("Only for Subtitle Language (1)", 0),
("Only for Subtitle Language (2)", 1),
("Only for Subtitle Language (3)", 2)):
if Prefs["subtitles.when_forced"] == setting:
try:
l.append(Language.rebuild(list(l)[index], forced=True))
break
except:
pass
elif self.forced_only:
for lang in l:
lang.forced = True
if not self.normal_subs:
for lang in l[:]:
if not lang.forced:
l.remove(lang)
return set(l) if not ordered else l
def query(self, language, keyword, video, season=None, episode=None, year=None, only_foreign=False,
also_foreign=False):
search_language = str(language).lower()
# sr-Cyrl specialcase
if search_language == "sr-cyrl":
search_language = "sr"
# set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652
params = {'sXML': 1, 'sL': search_language, 'sK': keyword}
is_episode = False
if season and episode:
is_episode = True
params['sTS'] = season
params['sTE'] = episode
if year:
params['sY'] = year
# loop over paginated results
logger.info('Searching subtitles %r', params)
subtitles = []
pids = set()
while True:
# query the server
content = None
try:
content = self.session.get(self.server_url + 'search/old', params=params, timeout=10).content
xml = etree.fromstring(content)
except etree.ParseError:
logger.error("Wrong data returned: %r", content)
break
# exit if no results
if not int(xml.find('pagination/results').text):
logger.debug('No subtitles found')
break
# loop over subtitles
for subtitle_xml in xml.findall('subtitle'):
# read xml elements
pid = subtitle_xml.find('pid').text
# ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321
if pid in pids:
continue
_language = Language.fromietf(subtitle_xml.find('language').text)
hearing_impaired = 'n' in (subtitle_xml.find('flags').text or '')
foreign = 'f' in (subtitle_xml.find('flags').text or '')
if only_foreign and not foreign:
continue
elif not only_foreign and not also_foreign and foreign:
continue
elif also_foreign and foreign:
_language = Language.rebuild(_language, forced=True)
# set subtitle language to hi if it's hearing_impaired
if hearing_impaired:
_language = Language.rebuild(_language, hi=True)
if language != _language:
continue
page_link = subtitle_xml.find('url').text
releases = []
if subtitle_xml.find('release').text:
for release in subtitle_xml.find('release').text.split():
releases.append(re.sub(r'\.+$', '', release)) # remove trailing dots
title = subtitle_xml.find('title').text
r_season = int(subtitle_xml.find('tvSeason').text)
r_episode = int(subtitle_xml.find('tvEpisode').text)
r_year = int(subtitle_xml.find('year').text)
if is_episode:
subtitle = self.subtitle_class(_language, hearing_impaired, page_link, pid, releases, title,
season=r_season, episode=r_episode, year=r_year,
asked_for_release_group=video.release_group,
asked_for_episode=episode)
else:
subtitle = self.subtitle_class(_language, hearing_impaired, page_link, pid, releases, title,
year=r_year, asked_for_release_group=video.release_group)
logger.debug('Found subtitle %r', subtitle)
subtitles.append(subtitle)
pids.add(pid)
# stop on last page
if int(xml.find('pagination/current').text) >= int(xml.find('pagination/count').text):
break
# increment current page
params['page'] = int(xml.find('pagination/current').text) + 1
logger.debug('Getting page %d', params['page'])
xml = None
return subtitles
def query(self, languages, title, season=None, episode=None, year=None, video=None):
items_per_page = 10
current_page = 1
# convert list of languages into search string
langs = '|'.join(
map(str, [l.titlovi if l != Language.fromietf('sr') else 'cirilica' for l in languages]))
# set query params
params = {'prijevod': title, 'jezik': langs}
is_episode = False
if season and episode:
is_episode = True
params['s'] = season
params['e'] = episode
if year:
params['g'] = year
# loop through paginated results
logger.info('Searching subtitles %r', params)
subtitles = []
while True:
# query the server
try:
r = self.session.get(self.search_url, params=params, timeout=10)
r.raise_for_status()
soup = BeautifulSoup(r.content, 'lxml')
# number of results
result_count = int(soup.select_one('.results_count b').string)
except:
result_count = None
# exit if no results
if not result_count:
if not subtitles:
logger.debug('No subtitles found')
else:
logger.debug("No more subtitles found")
break
# number of pages with results
pages = int(math.ceil(result_count / float(items_per_page)))
# get current page
if 'pg' in params:
current_page = int(params['pg'])
try:
sublist = soup.select('section.titlovi > ul.titlovi > li')
for sub in sublist:
# subtitle id
sid = sub.find(attrs={'data-id': True}).attrs['data-id']
# get download link
download_link = self.download_url + sid
# title and alternate title
match = title_re.search(sub.a.string)
if match:
_title = match.group('title')
alt_title = match.group('altitle')
else:
continue
# page link
page_link = self.server_url + sub.a.attrs['href']
# subtitle language
match = lang_re.search(sub.select_one('.lang').attrs['src'])
if match:
try:
lang = Language.fromtitlovi(match.group('lang'))
script = match.group('script')
if script:
lang.script = Script(script)
except ValueError:
continue
# relase year or series start year
match = year_re.search(sub.find(attrs={'data-id': True}).parent.i.string)
if match:
r_year = int(match.group('year'))
# fps
match = fps_re.search(sub.select_one('.fps').string)
if match:
fps = match.group('fps')
# releases
releases = str(sub.select_one('.fps').parent.contents[0].string)
# handle movies and series separately
if is_episode:
# season and episode info
sxe = sub.select_one('.s0xe0y').string
r_season = None
r_episode = None
if sxe:
match = season_re.search(sxe)
if match:
r_season = int(match.group('season'))
match = episode_re.search(sxe)
if match:
r_episode = int(match.group('episode'))
subtitle = self.subtitle_class(lang, page_link, download_link, sid, releases, _title,
alt_title=alt_title, season=r_season, episode=r_episode,
year=r_year, fps=fps,
asked_for_release_group=video.release_group,
asked_for_episode=episode)
else:
subtitle = self.subtitle_class(lang, page_link, download_link, sid, releases, _title,
alt_title=alt_title, year=r_year, fps=fps,
asked_for_release_group=video.release_group)
logger.debug('Found subtitle %r', subtitle)
# prime our matches so we can use the values later
subtitle.get_matches(video)
# add found subtitles
subtitles.append(subtitle)
finally:
soup.decompose()
# stop on last page
if current_page >= pages:
break
# increment current page
params['pg'] = current_page + 1
logger.debug('Getting page %d', params['pg'])
return subtitles
def get_language_from_stream(lang_code):
if lang_code:
lang = Locale.Language.Match(lang_code)
if lang and lang != "xx":
# Log.Debug("Found language: %r", lang)
return Language.fromietf(lang)
def SubtitleModificationsMenu(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs["part_id"]
language = kwargs["language"]
lang_instance = Language.fromietf(language)
current_sub, stored_subs, storage = get_current_sub(rating_key, part_id, language)
kwargs.pop("randomize")
current_mods = current_sub.mods or []
oc = SubFolderObjectContainer(title2=kwargs["title"], replace_parent=True)
from interface.item_details import SubtitleOptionsMenu
oc.add(DirectoryObject(
key=Callback(SubtitleOptionsMenu, randomize=timestamp(), **kwargs),
title=_(u"< Back to subtitle options for: %s", kwargs["title"]),
summary=unicode(kwargs["current_data"]),
thumb=default_thumb
))
for identifier, mod in mod_registry.mods.iteritems():
if mod.advanced:
continue
if mod.exclusive and identifier in current_mods:
continue
if mod.languages and lang_instance not in mod.languages:
continue
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=identifier, mode="add", randomize=timestamp(), **kwargs),
title=pad_title(_(mod.description)), summary=_(mod.long_description) or ""
))
fps_mod = SubtitleModifications.get_mod_class("change_FPS")
oc.add(DirectoryObject(
key=Callback(SubtitleFPSModMenu, randomize=timestamp(), **kwargs),
title=pad_title(_(fps_mod.description)), summary=_(fps_mod.long_description) or ""
))
shift_mod = SubtitleModifications.get_mod_class("shift_offset")
oc.add(DirectoryObject(
key=Callback(SubtitleShiftModUnitMenu, randomize=timestamp(), **kwargs),
title=pad_title(_(shift_mod.description)), summary=_(shift_mod.long_description) or ""
))
color_mod = SubtitleModifications.get_mod_class("color")
oc.add(DirectoryObject(
key=Callback(SubtitleColorModMenu, randomize=timestamp(), **kwargs),
title=pad_title(_(color_mod.description)), summary=_(color_mod.long_description) or ""
))
if current_mods:
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=None, mode="remove_last", randomize=timestamp(), **kwargs),
title=pad_title(_("Remove last applied mod (%s)", current_mods[-1])),
summary=_(u"Currently applied mods: %(mod_list)s", mod_list=", ".join(current_mods) if current_mods else _("none"))
))
oc.add(DirectoryObject(
key=Callback(SubtitleListMods, randomize=timestamp(), **kwargs),
title=pad_title(_("Manage applied mods")),
summary=_(u"Currently applied mods: %(mod_list)s", mod_list=", ".join(current_mods))
))
oc.add(DirectoryObject(
key=Callback(SubtitleReapplyMods, randomize=timestamp(), **kwargs),
title=pad_title(_("Reapply applied mods")),
summary=_(u"Currently applied mods: %(mod_list)s", mod_list=", ".join(current_mods) if current_mods else _("none"))
))
oc.add(DirectoryObject(
key=Callback(SubtitleSetMods, mods=None, mode="clear", randomize=timestamp(), **kwargs),
title=pad_title(_("Restore original version")),
summary=_(u"Currently applied mods: %(mod_list)s", mod_list=", ".join(current_mods) if current_mods else _("none"))
))
storage.destroy()
return oc
class TitloviProvider(Provider, ProviderSubtitleArchiveMixin):
subtitle_class = TitloviSubtitle
languages = {Language.fromtitlovi(l) for l in language_converters['titlovi'].codes} | {Language.fromietf('sr')}
server_url = 'http://titlovi.com'
search_url = server_url + '/titlovi/?'
download_url = server_url + '/download/?type=1&mediaid='
def initialize(self):
self.session = Session()
def terminate(self):
self.session.close()
def query(self, languages, title, season=None, episode=None, year=None, video=None):
items_per_page = 10
current_page = 1
# convert list of languages into search string
langs = '|'.join(
map(str, [l.titlovi if l != Language.fromietf('sr') else 'cirilica' for l in languages]))
# set query params
params = {'prijevod': title, 'jezik': langs}
is_episode = False
if season and episode:
is_episode = True
params['s'] = season
params['e'] = episode
if year:
params['g'] = year
# loop through paginated results
logger.info('Searching subtitles %r', params)
subtitles = []
while True:
# query the server
try:
r = self.session.get(self.search_url, params=params, timeout=10)
r.raise_for_status()
soup = BeautifulSoup(r.content, 'lxml')
# number of results
result_count = int(soup.select_one('.results_count b').string)
except:
result_count = None
# exit if no results
if not result_count:
if not subtitles:
logger.debug('No subtitles found')
else:
logger.debug("No more subtitles found")
break
# number of pages with results
pages = int(math.ceil(result_count / float(items_per_page)))
# get current page
if 'pg' in params:
current_page = int(params['pg'])
try:
sublist = soup.select('section.titlovi > ul.titlovi > li')
for sub in sublist:
# subtitle id
sid = sub.find(attrs={'data-id': True}).attrs['data-id']
# get download link
download_link = self.download_url + sid
# title and alternate title
match = title_re.search(sub.a.string)
if match:
_title = match.group('title')
alt_title = match.group('altitle')
else:
continue
# page link
page_link = self.server_url + sub.a.attrs['href']
# subtitle language
match = lang_re.search(sub.select_one('.lang').attrs['src'])
if match:
try:
lang = Language.fromtitlovi(match.group('lang'))
script = match.group('script')
if script:
lang.script = Script(script)
except ValueError:
continue
# relase year or series start year
match = year_re.search(sub.find(attrs={'data-id': True}).parent.i.string)
if match:
r_year = int(match.group('year'))
# fps
match = fps_re.search(sub.select_one('.fps').string)
if match:
fps = match.group('fps')
# releases
releases = str(sub.select_one('.fps').parent.contents[0].string)
# handle movies and series separately
if is_episode:
# season and episode info
sxe = sub.select_one('.s0xe0y').string
r_season = None
r_episode = None
if sxe:
match = season_re.search(sxe)
if match:
r_season = int(match.group('season'))
match = episode_re.search(sxe)
if match:
r_episode = int(match.group('episode'))
subtitle = self.subtitle_class(lang, page_link, download_link, sid, releases, _title,
alt_title=alt_title, season=r_season, episode=r_episode,
year=r_year, fps=fps,
asked_for_release_group=video.release_group,
asked_for_episode=episode)
else:
subtitle = self.subtitle_class(lang, page_link, download_link, sid, releases, _title,
alt_title=alt_title, year=r_year, fps=fps,
asked_for_release_group=video.release_group)
logger.debug('Found subtitle %r', subtitle)
# prime our matches so we can use the values later
subtitle.get_matches(video)
# add found subtitles
subtitles.append(subtitle)
finally:
soup.decompose()
# stop on last page
if current_page >= pages:
break
# increment current page
params['pg'] = current_page + 1
logger.debug('Getting page %d', params['pg'])
return subtitles
def list_subtitles(self, video, languages):
season = episode = None
if isinstance(video, Episode):
title = video.series
season = video.season
episode = video.episode
else:
title = video.title
return [s for s in
self.query(languages, fix_inconsistent_naming(title), season=season, episode=episode, year=video.year,
video=video)]
def download_subtitle(self, subtitle):
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
# open the archive
archive_stream = io.BytesIO(r.content)
if is_rarfile(archive_stream):
logger.debug('Archive identified as rar')
archive = RarFile(archive_stream)
elif is_zipfile(archive_stream):
logger.debug('Archive identified as zip')
archive = ZipFile(archive_stream)
else:
raise ProviderError('Unidentified archive type')
subtitle.content = self.get_subtitle_from_archive(subtitle, archive)
def extract_embedded_sub(**kwargs):
rating_key = kwargs["rating_key"]
part_id = kwargs.pop("part_id")
stream_index = kwargs.pop("stream_index")
with_mods = kwargs.pop("with_mods", False)
language = Language.fromietf(kwargs.pop("language"))
refresh = kwargs.pop("refresh", True)
set_current = kwargs.pop("set_current", True)
plex_item = kwargs.pop("plex_item", get_item(rating_key))
item_type = get_item_kind_from_item(plex_item)
part = kwargs.pop("part", get_part(plex_item, part_id))
scanned_videos = kwargs.pop("scanned_videos", None)
extract_mode = kwargs.pop("extract_mode", "a")
any_successful = False
if part:
if not scanned_videos:
metadata = get_plex_metadata(rating_key, part_id, item_type, plex_item=plex_item)
scanned_videos = scan_videos([metadata], ignore_all=True, skip_hashing=True)
for stream in part.streams:
# subtitle stream
if str(stream.index) == stream_index:
is_forced = is_stream_forced(stream)
bn = os.path.basename(part.file)
set_refresh_menu_state(_(u"Extracting subtitle %(stream_index)s of %(filename)s",
stream_index=stream_index,
filename=bn))
Log.Info(u"Extracting stream %s (%s) of %s", stream_index, str(language), bn)
out_codec = stream.codec if stream.codec != "mov_text" else "srt"
args = [
config.plex_transcoder, "-i", part.file, "-map", "0:%s" % stream_index, "-f", out_codec, "-"
]
cmdline = quote_args(args)
Log.Debug(u"Calling: %s", cmdline)
if mswindows:
Log.Debug("MSWindows: Fixing encoding")
cmdline = cmdline.encode("mbcs")
output = None
try:
output = subprocess.check_output(cmdline, stderr=subprocess.PIPE, shell=True)
except:
Log.Error("Extraction failed: %s", traceback.format_exc())
if output:
subtitle = ModifiedSubtitle(language, mods=config.default_mods if with_mods else None)
subtitle.content = output
subtitle.provider_name = "embedded"
subtitle.id = "stream_%s" % stream_index
subtitle.score = 0
subtitle.set_encoding("utf-8")
# fixme: speedup video; only video.name is needed
video = scanned_videos.keys()[0]
save_successful = save_subtitles(scanned_videos, {video: [subtitle]}, mode="m",
set_current=set_current)
set_refresh_menu_state(None)
if save_successful and refresh:
refresh_item(rating_key)
# add item to history
item_title = get_title_for_video_metadata(video.plexapi_metadata,
add_section_title=False, add_episode_title=True)
history = get_history()
history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
thumb=video.plexapi_metadata["super_thumb"],
subtitle=subtitle, mode=extract_mode)
history.destroy()
any_successful = True
return any_successful
def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_title=None, internal=False, external=True, languages=()):
item_id = int(rating_key)
item = get_item(rating_key)
if kind == "show":
item_title = get_plex_item_display_title(item, kind, parent=item.season, section_title=section_title, parent_title=item.show.title)
else:
item_title = get_plex_item_display_title(item, kind, section_title=section_title)
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load(rating_key)
subtitle_storage.destroy()
subtitle_target_dir, tdir_is_absolute = config.subtitle_sub_dir
missing = set()
languages_set = set([Language.rebuild(l) for l in languages])
for media in item.media:
existing_subs = {"internal": [], "external": [], "own_external": [], "count": 0}
for part in media.parts:
# did we already download an external subtitle before?
if subtitle_target_dir and stored_subs:
for language in languages_set:
if has_external_subtitle(part.id, stored_subs, language):
# check the existence of the actual subtitle file
# get media filename without extension
part_basename = os.path.splitext(os.path.basename(part.file))[0]
# compute target directory for subtitle
# fixme: move to central location
if tdir_is_absolute:
possible_subtitle_path_base = subtitle_target_dir
else:
possible_subtitle_path_base = os.path.join(os.path.dirname(part.file), subtitle_target_dir)
possible_subtitle_path_base = os.path.realpath(possible_subtitle_path_base)
# folder actually exists?
if not os.path.isdir(possible_subtitle_path_base):
continue
found_any = False
for ext in config.subtitle_formats:
if cast_bool(Prefs['subtitles.only_one']):
possible_subtitle_path = os.path.join(possible_subtitle_path_base,
u"%s.%s" % (part_basename, ext))
else:
possible_subtitle_path = os.path.join(possible_subtitle_path_base,
u"%s.%s.%s" % (part_basename, language, ext))
# check for subtitle existence
if os.path.isfile(possible_subtitle_path):
found_any = True
Log.Debug(u"Found: %s", possible_subtitle_path)
break
if found_any:
existing_subs["own_external"].append(language)
existing_subs["count"] = existing_subs["count"] + 1
for stream in part.streams:
if stream.stream_type == 3:
is_forced = is_stream_forced(stream)
if stream.index:
key = "internal"
else:
key = "external"
if not config.exotic_ext and stream.codec.lower() not in TEXT_SUBTITLE_EXTS:
continue
# treat unknown language as lang1?
if not stream.language_code and config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
# we can't parse empty language codes
elif not stream.language_code or not stream.codec:
continue
else:
# parse with internal language parser first
try:
lang = get_language_from_stream(stream.language_code)
if not lang:
if config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
else:
continue
except (ValueError, LanguageReverseError):
continue
if lang:
# Log.Debug("Found babelfish language: %r", lang)
lang.forced = is_forced
existing_subs[key].append(lang)
existing_subs["count"] = existing_subs["count"] + 1
missing_from_part = set([Language.rebuild(l) for l in languages])
if existing_subs["count"]:
# fixme: this is actually somewhat broken with IETF, as Plex doesn't store the country portion
# (pt instead of pt-BR) inside the database. So it might actually download pt-BR if there's a local pt-BR
# subtitle but not our own.
existing_flat = set((existing_subs["internal"] if internal else [])
+ (existing_subs["external"] if external else [])
+ existing_subs["own_external"])
check_languages = set([Language.rebuild(l) for l in languages])
alpha3_map = {}
if config.ietf_as_alpha3:
for language in existing_flat:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
for language in check_languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
# compare sets of strings, not sets of different Language instances
check_languages_str = set(str(l) for l in check_languages)
existing_flat_str = set(str(l) for l in existing_flat)
if check_languages_str.issubset(existing_flat_str) or \
(len(existing_flat) >= 1 and Prefs['subtitles.only_one']):
# all subs found
#Log.Info(u"All subtitles exist for '%s'", item_title)
continue
missing_from_part = set(Language.fromietf(l) for l in check_languages_str - existing_flat_str)
if config.ietf_as_alpha3:
for language in missing_from_part:
language.country = alpha3_map.get(language.alpha3, None)
if missing_from_part:
Log.Info(u"Subs still missing for '%s' (%s: %s): %s", item_title, rating_key, media.id,
missing_from_part)
missing.update(missing_from_part)
if missing:
# deduplicate
missing = set(Language.fromietf(la) for la in set(str(l) for l in missing))
return added_at, item_id, item_title, item, missing
def query(self,
languages,
title,
season=None,
episode=None,
year=None,
imdb_id=None,
video=None):
search_params = dict()
used_languages = languages
lang_strings = [str(lang) for lang in used_languages]
# handle possible duplicate use of Serbian Latin
if "sr" in lang_strings and "sr-Latn" in lang_strings:
logger.info(
'Duplicate entries <Language [sr]> and <Language [sr-Latn]> found, filtering languages'
)
used_languages = [
l for l in used_languages if l != Language.fromietf('sr-Latn')
]
logger.info('Filtered language list %r', used_languages)
# convert list of languages into search string
langs = '|'.join(map(str, [l.titlovi for l in used_languages]))
# set query params
search_params['query'] = title
search_params['lang'] = langs
is_episode = False
if season and episode:
is_episode = True
#search_params['season'] = season
#search_params['episode'] = episode
#if year:
# search_params['year'] = year
if imdb_id:
search_params['imdbID'] = imdb_id
# loop through paginated results
logger.info('Searching subtitles %r', search_params)
subtitles = []
query_results = []
try:
search_params['token'] = self.login_token
search_params['userid'] = self.user_id
search_params['json'] = True
#response = self.get_result(search_url=self.api_search_url, search_params=search_params)
response = self.get_result(self.api_search_url, search_params)
resp_json = response.json()
if resp_json['SubtitleResults']:
query_results.extend(resp_json['SubtitleResults'])
except Exception as e:
logger.error(e)
for sub in query_results:
# title and alternate title
match = title_re.search(sub.get('Title'))
if match:
_title = match.group('title')
alt_title = match.group('altitle')
else:
continue
# handle movies and series separately
if is_episode:
# skip if season and episode number does not match
if season and season != sub.get('Season'):
continue
elif episode and episode != sub.get('Episode'):
continue
subtitle = self.subtitle_class(
Language.fromtitlovi(sub.get('Lang')),
sub.get('Link'),
sub.get('Id'),
sub.get('Release'),
_title,
alt_title=alt_title,
season=sub.get('Season'),
episode=sub.get('Episode'),
year=sub.get('Year'),
rating=sub.get('Rating'),
download_count=sub.get('DownloadCount'),
asked_for_release_group=video.release_group,
asked_for_episode=episode)
else:
subtitle = self.subtitle_class(
Language.fromtitlovi(sub.get('Lang')),
sub.get('Link'),
sub.get('Id'),
sub.get('Release'),
_title,
alt_title=alt_title,
year=sub.get('Year'),
rating=sub.get('Rating'),
download_count=sub.get('DownloadCount'),
asked_for_release_group=video.release_group)
logger.debug('Found subtitle %r', subtitle)
# prime our matches so we can use the values later
subtitle.get_matches(video)
# add found subtitles
subtitles.append(subtitle)
return subtitles
