def agent_extract_embedded(video_part_map):
try:
subtitle_storage = get_subtitle_storage()
to_extract = []
item_count = 0
for scanned_video, part_info in video_part_map.iteritems():
plexapi_item = scanned_video.plexapi_metadata["item"]
stored_subs = subtitle_storage.load_or_new(plexapi_item)
for plexapi_part in get_all_parts(plexapi_item):
item_count = item_count + 1
for requested_language in config.lang_list:
embedded_subs = stored_subs.get_by_provider(
plexapi_part.id, requested_language, "embedded")
current = stored_subs.get_any(plexapi_part.id,
requested_language)
if not embedded_subs:
stream_data = get_embedded_subtitle_streams(
plexapi_part,
requested_language=requested_language,
get_forced=config.forced_only)
if stream_data:
stream = stream_data[0]["stream"]
to_extract.append(
({
scanned_video: part_info
}, plexapi_part, str(stream.index),
str(requested_language), not current))
if not cast_bool(
Prefs["subtitles.search_after_autoextract"]
):
scanned_video.subtitle_languages.update(
{requested_language})
else:
Log.Debug(
"Skipping embedded subtitle extraction for %s, already got %r from %s",
plexapi_item.rating_key, requested_language,
embedded_subs[0].id)
if to_extract:
Log.Info(
"Triggering extraction of %d embedded subtitles of %d items",
len(to_extract), item_count)
Thread.Create(
multi_extract_embedded,
stream_list=to_extract,
refresh=True,
with_mods=True,
single_thread=not config.advanced.auto_extract_multithread)
except:
Log.Error(
"Something went wrong when auto-extracting subtitles, continuing: %s",
traceback.format_exc())
def Start():
HTTP.CacheTime = 0
HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT
config.init_cache()
# clear expired intents
intent = get_intent()
intent.cleanup()
#Locale.DefaultLocale = "de"
# clear expired menu history items
now = datetime.datetime.now()
if "menu_history" in Dict:
for key, timeout in Dict["menu_history"].copy().items():
if now > timeout:
try:
del Dict["menu_history"][key]
except:
pass
# run migrations
if "subs" in Dict or "history" in Dict:
Thread.Create(dispatch_migrate)
# clear old task data
scheduler.clear_task_data()
# init defaults; perhaps not the best idea to use ValidatePrefs here, but we'll see
ValidatePrefs()
Log.Debug(config.full_version)
if not config.permissions_ok:
Log.Error("Insufficient permissions on library folders:")
for title, path in config.missing_permissions:
Log.Error("Insufficient permissions on library %s, folder: %s" %
(title, path))
# run task scheduler
scheduler.run()
# bind activities
if config.enable_channel:
Thread.Create(activity.start)
if "anon_id" not in Dict:
Dict["anon_id"] = get_identifier()
# track usage
if cast_bool(Prefs["track_usage"]):
if "first_use" not in Dict:
Dict["first_use"] = datetime.datetime.utcnow()
Dict.Save()
track_usage("General", "plugin", "first_start", config.version)
track_usage("General", "plugin", "start", config.version)
def Start():
HTTP.CacheTime = 0
HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT
config.init_cache()
# clear expired intents
intent = get_intent()
intent.cleanup()
#Locale.DefaultLocale = "de"
# clear expired menu history items
now = datetime.datetime.now()
if "menu_history" in Dict:
for key, timeout in Dict["menu_history"].copy().items():
if now > timeout:
try:
del Dict["menu_history"][key]
except:
pass
# run migrations
if "subs" in Dict or "history" in Dict:
Thread.Create(dispatch_migrate)
# clear old task data
scheduler.clear_task_data()
# init defaults; perhaps not the best idea to use ValidatePrefs here, but we'll see
ValidatePrefs()
Log.Debug(config.full_version)
if not config.permissions_ok:
Log.Error("Insufficient permissions on library folders:")
for title, path in config.missing_permissions:
Log.Error("Insufficient permissions on library %s, folder: %s" % (title, path))
# run task scheduler
scheduler.run()
# bind activities
if config.enable_channel:
Thread.Create(activity.start)
if "anon_id" not in Dict:
Dict["anon_id"] = get_identifier()
# track usage
if cast_bool(Prefs["track_usage"]):
if "first_use" not in Dict:
Dict["first_use"] = datetime.datetime.utcnow()
Dict.Save()
track_usage("General", "plugin", "first_start", config.version)
track_usage("General", "plugin", "start", config.version)
def items_get_all_missing_subs(items):
missing = []
for added_at, kind, section_title, key in items:
try:
state = item_discover_missing_subs(
key,
kind=kind,
added_at=added_at,
section_title=section_title,
languages=config.lang_list,
internal=cast_bool(Prefs["subtitles.scan.embedded"]),
external=cast_bool(Prefs["subtitles.scan.external"]))
if state:
# (added_at, item_id, title, item, missing_languages)
missing.append(state)
except:
Log.Error(
"Something went wrong when getting the state of item %s: %s",
key, traceback.format_exc())
return missing
def items_get_all_missing_subs(items, sleep_after_request=False):
missing = []
for added_at, kind, section_title, key in items:
try:
state = item_discover_missing_subs(
key,
kind=kind,
added_at=added_at,
section_title=section_title,
languages=config.lang_list.copy(),
internal=cast_bool(Prefs["subtitles.scan.embedded"]),
external=cast_bool(Prefs["subtitles.scan.external"])
)
if state:
# (added_at, item_id, title, item, missing_languages)
missing.append(state)
except:
Log.Error("Something went wrong when getting the state of item %s: %s", key, traceback.format_exc())
if sleep_after_request:
time.sleep(sleep_after_request)
return missing
def Start():
HTTP.CacheTime = 0
HTTP.Headers['User-agent'] = OS_PLEX_USERAGENT
# configured cache to be in memory as per https://github.com/Diaoul/subliminal/issues/303
subliminal.region.configure('dogpile.cache.memory')
# clear expired intents
intent = get_intent()
intent.cleanup()
# clear expired menu history items
now = datetime.datetime.now()
if "menu_history" in Dict:
for key, timeout in Dict["menu_history"].items():
if now > timeout:
del Dict["menu_history"][key]
# run migrations
if "subs" in Dict or "history" in Dict:
Thread.Create(dispatch_migrate)
# clear old task data
scheduler.clear_task_data()
# init defaults; perhaps not the best idea to use ValidatePrefs here, but we'll see
ValidatePrefs()
Log.Debug(config.full_version)
if not config.permissions_ok:
Log.Error("Insufficient permissions on library folders:")
for title, path in config.missing_permissions:
Log.Error("Insufficient permissions on library %s, folder: %s" %
(title, path))
# run task scheduler
scheduler.run()
# bind activities
Thread.Create(activity.start)
if "anon_id" not in Dict:
Dict["anon_id"] = get_identifier()
# track usage
if cast_bool(Prefs["track_usage"]):
if "first_use" not in Dict:
Dict["first_use"] = datetime.datetime.utcnow()
Dict.Save()
track_usage("General", "plugin", "first_start", config.version)
track_usage("General", "plugin", "start", config.version)
def agent_extract_embedded(video_part_map):
try:
subtitle_storage = get_subtitle_storage()
to_extract = []
item_count = 0
for scanned_video, part_info in video_part_map.iteritems():
plexapi_item = scanned_video.plexapi_metadata["item"]
stored_subs = subtitle_storage.load_or_new(plexapi_item)
valid_langs_in_media = audio_streams_match_languages(scanned_video, config.get_lang_list(ordered=True))
if not config.lang_list.difference(valid_langs_in_media):
Log.Debug("Skipping embedded subtitle extraction for %s, audio streams are in correct language(s)",
plexapi_item.rating_key)
continue
for plexapi_part in get_all_parts(plexapi_item):
item_count = item_count + 1
used_one_unknown_stream = False
for requested_language in config.lang_list:
embedded_subs = stored_subs.get_by_provider(plexapi_part.id, requested_language, "embedded")
current = stored_subs.get_any(plexapi_part.id, requested_language) or \
requested_language in scanned_video.external_subtitle_languages
if not embedded_subs:
stream_data = get_embedded_subtitle_streams(plexapi_part, requested_language=requested_language,
skip_unknown=used_one_unknown_stream)
if stream_data:
stream = stream_data[0]["stream"]
if stream_data[0]["is_unknown"]:
used_one_unknown_stream = True
to_extract.append(({scanned_video: part_info}, plexapi_part, str(stream.index),
str(requested_language), not current))
if not cast_bool(Prefs["subtitles.search_after_autoextract"]):
scanned_video.subtitle_languages.update({requested_language})
else:
Log.Debug("Skipping embedded subtitle extraction for %s, already got %r from %s",
plexapi_item.rating_key, requested_language, embedded_subs[0].id)
if to_extract:
Log.Info("Triggering extraction of %d embedded subtitles of %d items", len(to_extract), item_count)
Thread.Create(multi_extract_embedded, stream_list=to_extract, refresh=True, with_mods=True,
single_thread=not config.advanced.auto_extract_multithread)
except:
Log.Error("Something went wrong when auto-extracting subtitles, continuing: %s", traceback.format_exc())
def download_best_subtitles(video_part_map, min_score=0, throttle_time=None):
hearing_impaired = Prefs['subtitles.search.hearingImpaired']
ietf_as_alpha3 = cast_bool(Prefs["subtitles.language.ietf_normalize"])
languages = set([Language.fromietf(str(l)) for l in config.lang_list])
if not languages:
return
# should we treat IETF as alpha3? (ditch the country part)
alpha3_map = {}
if ietf_as_alpha3:
for language in languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
missing_languages = False
for video, part in video_part_map.iteritems():
if not Prefs['subtitles.save.filesystem']:
# scan for existing metadata subtitles
meta_subs = get_subtitles_from_metadata(part)
for language, subList in meta_subs.iteritems():
if subList:
video.subtitle_languages.add(language)
Log.Debug("Found metadata subtitle %s for %s", language, video)
have_languages = video.subtitle_languages.copy()
if ietf_as_alpha3:
for language in have_languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
missing_subs = (set(str(l) for l in languages) - set(str(l) for l in have_languages))
# all languages are found if we either really have subs for all languages or we only want to have exactly one language
# and we've only found one (the case for a selected language, Prefs['subtitles.only_one'] (one found sub matches any language))
found_one_which_is_enough = len(video.subtitle_languages) >= 1 and Prefs['subtitles.only_one']
if not missing_subs or found_one_which_is_enough:
if found_one_which_is_enough:
Log.Debug('Only one language was requested, and we\'ve got a subtitle for %s', video)
else:
Log.Debug('All languages %r exist for %s', languages, video)
continue
missing_languages = True
break
if missing_languages:
# re-add country codes to the missing languages, in case we've removed them above
if ietf_as_alpha3:
for language in languages:
language.country = alpha3_map.get(language.alpha3, None)
Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s, languages: %s" %
(min_score, hearing_impaired, languages))
# prepare blacklist
blacklist = get_blacklist_from_part_map(video_part_map, languages)
return subliminal.download_best_subtitles(video_part_map.keys(), languages, min_score, hearing_impaired, providers=config.providers,
provider_configs=config.provider_settings, pool_class=config.provider_pool,
compute_score=compute_score, throttle_time=throttle_time, blacklist=blacklist)
Log.Debug("All languages for all requested videos exist. Doing nothing.")
def run(self):
super(FindBetterSubtitles, self).run()
self.running = True
better_found = 0
try:
max_search_days = int(Prefs[
"scheduler.tasks.FindBetterSubtitles.max_days_after_added"].
strip())
except ValueError:
Log.Error(
u"Please only put numbers into the FindBetterSubtitles.max_days_after_added setting. Exiting"
)
return
else:
if max_search_days > 30:
Log.Error(
u"%s: FindBetterSubtitles.max_days_after_added is too big. Max is 30 days.",
self.name)
return
now = datetime.datetime.now()
min_score_series = int(
Prefs["subtitles.search.minimumTVScore2"].strip())
min_score_movies = int(
Prefs["subtitles.search.minimumMovieScore2"].strip())
min_score_extracted_series = config.advanced.find_better_as_extracted_tv_score or 352
min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 82
overwrite_manually_modified = cast_bool(Prefs[
"scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified"])
overwrite_manually_selected = cast_bool(Prefs[
"scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"])
air_date_cutoff_pref = Prefs[
"scheduler.tasks.FindBetterSubtitles.air_date_cutoff"]
if air_date_cutoff_pref == "don't limit":
air_date_cutoff = None
else:
air_date_cutoff = int(air_date_cutoff_pref.split()[0])
subtitle_storage = get_subtitle_storage()
viable_item_count = 0
try:
for fn in subtitle_storage.get_recent_files(
age_days=max_search_days):
stored_subs = subtitle_storage.load(filename=fn)
if not stored_subs:
continue
video_id = stored_subs.video_id
if stored_subs.item_type == "episode":
cutoff = self.series_cutoff
min_score = min_score_series
min_score_extracted = min_score_extracted_series
else:
cutoff = self.movies_cutoff
min_score = min_score_movies
min_score_extracted = min_score_extracted_movies
# don't search for better subtitles until at least 30 minutes have passed
if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
Log.Debug(u"%s: Item %s too new, skipping", self.name,
video_id)
continue
# added_date <= max_search_days?
if stored_subs.added_at + datetime.timedelta(
days=max_search_days) <= now:
continue
viable_item_count += 1
ditch_parts = []
# look through all stored subtitle data
for part_id, languages in stored_subs.parts.iteritems():
part_id = str(part_id)
# all languages
for language, current_subs in languages.iteritems():
current_key = current_subs.get("current")
current = current_subs.get(current_key)
# currently got subtitle?
# fixme: check for existence
if not current:
continue
current_score = current.score
current_mode = current.mode
# late cutoff met? skip
if current_score >= cutoff:
Log.Debug(
u"%s: Skipping finding better subs, "
u"cutoff met (current: %s, cutoff: %s): %s (%s)",
self.name, current_score, cutoff,
stored_subs.title, video_id)
continue
# got manual subtitle but don't want to touch those?
if current_mode == "m" and not overwrite_manually_selected:
Log.Debug(
u"%s: Skipping finding better subs, "
u"had manual: %s (%s)", self.name,
stored_subs.title, video_id)
continue
# subtitle modifications different from default
if not overwrite_manually_modified and current.mods \
and set(current.mods).difference(set(config.default_mods)):
Log.Debug(
u"%s: Skipping finding better subs, it has manual modifications: %s (%s)",
self.name, stored_subs.title, video_id)
continue
try:
subs = self.list_subtitles(
video_id,
stored_subs.item_type,
part_id,
language,
air_date_cutoff=air_date_cutoff)
except PartUnknownException:
Log.Info(
u"%s: Part %s unknown/gone; ditching subtitle info",
self.name, part_id)
ditch_parts.append(part_id)
continue
hit_providers = subs is not None
if subs:
# subs are already sorted by score
better_downloaded = False
better_tried_download = 0
better_visited = 0
for sub in subs:
if sub.score > current_score and sub.score > min_score:
if current.provider_name == "embedded" and sub.score < min_score_extracted:
Log.Debug(
u"%s: Not downloading subtitle for %s, we've got an active extracted "
u"embedded sub and the min score %s isn't met (%s).",
self.name, video_id,
min_score_extracted, sub.score)
better_visited += 1
break
Log.Debug(
u"%s: Better subtitle found for %s, downloading",
self.name, video_id)
better_tried_download += 1
ret = self.download_subtitle(sub,
video_id,
mode="b")
if ret:
better_found += 1
better_downloaded = True
break
else:
Log.Debug(
u"%s: Couldn't download/save subtitle. "
u"Continuing to the next one",
self.name)
Log.Debug(
u"%s: Waiting %s seconds before continuing",
self.name, self.DL_PROVIDER_SLACK)
Thread.Sleep(self.DL_PROVIDER_SLACK)
better_visited += 1
if better_tried_download and not better_downloaded:
Log.Debug(
u"%s: Tried downloading better subtitle for %s, "
u"but every try failed.", self.name,
video_id)
elif better_downloaded:
Log.Debug(
u"%s: Better subtitle downloaded for %s",
self.name, video_id)
if better_tried_download or better_downloaded:
Log.Debug(
u"%s: Waiting %s seconds before continuing",
self.name, self.DL_PROVIDER_SLACK)
Thread.Sleep(self.DL_PROVIDER_SLACK)
elif better_visited:
Log.Debug(
u"%s: Waiting %s seconds before continuing",
self.name, self.PROVIDER_SLACK)
Thread.Sleep(self.PROVIDER_SLACK)
subs = None
elif hit_providers:
# hit the providers but didn't try downloading? wait.
Log.Debug(
u"%s: Waiting %s seconds before continuing",
self.name, self.PROVIDER_SLACK)
Thread.Sleep(self.PROVIDER_SLACK)
if ditch_parts:
for part_id in ditch_parts:
try:
del stored_subs.parts[part_id]
except KeyError:
pass
subtitle_storage.save(stored_subs)
ditch_parts = None
stored_subs = None
Thread.Sleep(1)
finally:
subtitle_storage.destroy()
if better_found:
Log.Debug(u"%s: done. Better subtitles found for %s/%s items",
self.name, better_found, viable_item_count)
else:
Log.Debug(u"%s: done. No better subtitles found for %s items",
self.name, viable_item_count)
def update(self, metadata, media, lang):
if not config.enable_agent:
Log.Debug("Skipping Sub-Zero agent(s)")
return
Log.Debug("Sub-Zero %s, %s update called" %
(config.version, self.agent_type))
intent = get_intent()
if not media:
Log.Error(
"Called with empty media, something is really wrong with your setup!"
)
return
item_ids = []
try:
config.init_subliminal_patches()
videos = media_to_videos(media, kind=self.agent_type)
# find local media
update_local_media(metadata, media, media_type=self.agent_type)
# media ignored?
use_any_parts = False
for video in videos:
if is_ignored(video["id"]):
Log.Debug(u"Ignoring %s" % video)
continue
use_any_parts = True
if not use_any_parts:
Log.Debug(u"Nothing to do.")
return
try:
use_score = int(Prefs[self.score_prefs_key].strip())
except ValueError:
Log.Error(
"Please only put numbers into the scores setting. Exiting")
return
set_refresh_menu_state(media, media_type=self.agent_type)
# scanned_video_part_map = {subliminal.Video: plex_part, ...}
providers = config.get_providers(media_type=self.agent_type)
try:
scanned_video_part_map = scan_videos(videos,
providers=providers)
except IOError, e:
Log.Exception(
"Permission error, please check your folder/file permissions. Exiting."
)
if cast_bool(Prefs["check_permissions"]):
config.permissions_ok = False
config.missing_permissions = e.message
return
# auto extract embedded
if config.embedded_auto_extract:
if config.plex_transcoder:
agent_extract_embedded(scanned_video_part_map)
else:
Log.Warning(
"Plex Transcoder not found, can't auto extract")
# clear missing subtitles menu data
if not scheduler.is_task_running("MissingSubtitles"):
scheduler.clear_task_data("MissingSubtitles")
downloaded_subtitles = None
# debounce for self.debounce seconds
now = datetime.datetime.now()
if "last_call" in Dict:
last_call = Dict["last_call"]
if last_call + datetime.timedelta(seconds=self.debounce) > now:
wait = self.debounce - (now - last_call).seconds
if wait >= 1:
Log.Debug("Waiting %s seconds until continuing", wait)
Thread.Sleep(wait)
# downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
try:
downloaded_subtitles = download_best_subtitles(
scanned_video_part_map,
min_score=use_score,
throttle_time=self.debounce,
providers=providers)
except:
Log.Exception(
"Something went wrong when downloading subtitles")
if downloaded_subtitles is not None:
Dict["last_call"] = datetime.datetime.now()
item_ids = get_media_item_ids(media, kind=self.agent_type)
downloaded_any = False
if downloaded_subtitles:
downloaded_any = any(downloaded_subtitles.values())
if downloaded_any:
save_successful = False
try:
save_successful = save_subtitles(scanned_video_part_map,
downloaded_subtitles,
mods=config.default_mods)
except:
Log.Exception("Something went wrong when saving subtitles")
track_usage("Subtitle", "refreshed", "download", 1)
# store SZ meta info even if download wasn't successful
if not save_successful:
self.store_blank_subtitle_metadata(scanned_video_part_map)
else:
for video, video_subtitles in downloaded_subtitles.items():
# store item(s) in history
for subtitle in video_subtitles:
item_title = get_title_for_video_metadata(
video.plexapi_metadata,
add_section_title=False)
history = get_history()
history.add(item_title,
video.id,
section_title=video.
plexapi_metadata["section"],
subtitle=subtitle)
history.destroy()
else:
# store SZ meta info even if we've downloaded none
self.store_blank_subtitle_metadata(scanned_video_part_map)
update_local_media(metadata, media, media_type=self.agent_type)
def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_title=None, internal=False, external=True, languages=()):
item_id = int(rating_key)
item = get_item(rating_key)
if kind == "show":
item_title = get_plex_item_display_title(item, kind, parent=item.season, section_title=section_title, parent_title=item.show.title)
else:
item_title = get_plex_item_display_title(item, kind, section_title=section_title)
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load(rating_key)
subtitle_storage.destroy()
subtitle_target_dir, tdir_is_absolute = config.subtitle_sub_dir
missing = set()
languages_set = set([Language.rebuild(l) for l in languages])
for media in item.media:
existing_subs = {"internal": [], "external": [], "own_external": [], "count": 0}
for part in media.parts:
update_stream_info(part)
# did we already download an external subtitle before?
if subtitle_target_dir and stored_subs:
for language in languages_set:
if has_external_subtitle(part.id, stored_subs, language):
# check the existence of the actual subtitle file
# get media filename without extension
part_basename = os.path.splitext(os.path.basename(part.file))[0]
# compute target directory for subtitle
# fixme: move to central location
if tdir_is_absolute:
possible_subtitle_path_base = subtitle_target_dir
else:
possible_subtitle_path_base = os.path.join(os.path.dirname(part.file), subtitle_target_dir)
possible_subtitle_path_base = os.path.realpath(possible_subtitle_path_base)
# folder actually exists?
if not os.path.isdir(possible_subtitle_path_base):
continue
found_any = False
for ext in config.subtitle_formats:
if cast_bool(Prefs['subtitles.only_one']):
possible_subtitle_path = os.path.join(possible_subtitle_path_base,
u"%s.%s" % (part_basename, ext))
else:
possible_subtitle_path = os.path.join(possible_subtitle_path_base,
u"%s.%s.%s" % (part_basename, language, ext))
# check for subtitle existence
if os.path.isfile(possible_subtitle_path):
found_any = True
Log.Debug(u"Found: %s", possible_subtitle_path)
break
if found_any:
existing_subs["own_external"].append(language)
existing_subs["count"] = existing_subs["count"] + 1
for stream in part.streams:
if stream.stream_type == 3:
is_forced = is_stream_forced(stream)
if stream.index:
key = "internal"
else:
key = "external"
if not config.exotic_ext and stream.codec.lower() not in TEXT_SUBTITLE_EXTS:
continue
# treat unknown language as lang1?
if not stream.language_code and config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
# we can't parse empty language codes
elif not stream.language_code or not stream.codec:
continue
else:
# parse with internal language parser first
try:
lang = get_language_from_stream(stream.language_code)
if not lang:
if config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
else:
continue
except (ValueError, LanguageReverseError):
continue
if lang:
# Log.Debug("Found babelfish language: %r", lang)
lang.forced = is_forced
existing_subs[key].append(lang)
existing_subs["count"] = existing_subs["count"] + 1
missing_from_part = set([Language.rebuild(l) for l in languages])
if existing_subs["count"]:
# fixme: this is actually somewhat broken with IETF, as Plex doesn't store the country portion
# (pt instead of pt-BR) inside the database. So it might actually download pt-BR if there's a local pt-BR
# subtitle but not our own.
existing_flat = set((existing_subs["internal"] if internal else [])
+ (existing_subs["external"] if external else [])
+ existing_subs["own_external"])
check_languages = set([Language.rebuild(l) for l in languages])
alpha3_map = {}
if config.ietf_as_alpha3:
for language in existing_flat:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
for language in check_languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
# compare sets of strings, not sets of different Language instances
check_languages_str = set(str(l) for l in check_languages)
existing_flat_str = set(str(l) for l in existing_flat)
if check_languages_str.issubset(existing_flat_str) or \
(len(existing_flat) >= 1 and Prefs['subtitles.only_one']):
# all subs found
#Log.Info(u"All subtitles exist for '%s'", item_title)
continue
missing_from_part = set(Language.fromietf(l) for l in check_languages_str - existing_flat_str)
if config.ietf_as_alpha3:
for language in missing_from_part:
language.country = alpha3_map.get(language.alpha3, None)
if missing_from_part:
Log.Info(u"Subs still missing for '%s' (%s: %s): %s", item_title, rating_key, media.id,
missing_from_part)
missing.update(missing_from_part)
if missing:
# deduplicate
missing = set(Language.fromietf(la) for la in set(str(l) for l in missing))
return added_at, item_id, item_title, item, missing
def ItemDetailsMenu(rating_key,
title=None,
base_title=None,
item_title=None,
randomize=None):
"""
displays the item details menu of an item that doesn't contain any deeper tree, such as a movie or an episode
:param rating_key:
:param title:
:param base_title:
:param item_title:
:param randomize:
:return:
"""
title = unicode(base_title) + " > " + unicode(
title) if base_title else unicode(title)
item = get_item(rating_key)
current_kind = get_item_kind_from_rating_key(rating_key)
timeout = 30
oc = SubFolderObjectContainer(title2=title, replace_parent=True)
oc.add(
DirectoryObject(
key=Callback(RefreshItem,
rating_key=rating_key,
item_title=item_title,
randomize=timestamp(),
timeout=timeout * 1000),
title=u"Refresh: %s" % item_title,
summary=
"Refreshes the %s, possibly searching for missing and picking up new subtitles on disk"
% current_kind,
thumb=item.thumb or default_thumb))
oc.add(
DirectoryObject(
key=Callback(RefreshItem,
rating_key=rating_key,
item_title=item_title,
force=True,
randomize=timestamp(),
timeout=timeout * 1000),
title=u"Auto-search: %s" % item_title,
summary=
"Issues a forced refresh, ignoring known subtitles and searching for new ones",
thumb=item.thumb or default_thumb))
# get stored subtitle info for item id
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load_or_new(item)
# get the plex item
plex_item = list(Plex["library"].metadata(rating_key))[0]
# get current media info for that item
media = plex_item.media
# look for subtitles for all available media parts and all of their languages
for part in media.parts:
filename = os.path.basename(part.file)
part_id = str(part.id)
# iterate through all configured languages
for lang in config.lang_list:
lang_a2 = lang.alpha2
# ietf lang?
if cast_bool(Prefs["subtitles.language.ietf"]) and "-" in lang_a2:
lang_a2 = lang_a2.split("-")[0]
# get corresponding stored subtitle data for that media part (physical media item), for language
current_sub = stored_subs.get_any(part_id, lang_a2)
current_sub_id = None
current_sub_provider_name = None
summary = u"No current subtitle in storage"
current_score = None
if current_sub:
current_sub_id = current_sub.id
current_sub_provider_name = current_sub.provider_name
current_score = current_sub.score
summary = u"Current subtitle: %s (added: %s, %s), Language: %s, Score: %i, Storage: %s" % \
(current_sub.provider_name, df(current_sub.date_added), current_sub.mode_verbose, lang,
current_sub.score, current_sub.storage_type)
oc.add(
DirectoryObject(key=Callback(
ListAvailableSubsForItemMenu,
rating_key=rating_key,
part_id=part_id,
title=title,
item_title=item_title,
language=lang,
current_id=current_sub_id,
item_type=plex_item.type,
filename=filename,
current_data=summary,
randomize=timestamp(),
current_provider=current_sub_provider_name,
current_score=current_score),
title=u"List %s subtitles" % lang.name,
summary=summary))
add_ignore_options(oc,
"videos",
title=item_title,
rating_key=rating_key,
callback_menu=IgnoreMenu)
return oc
def agent_extract_embedded(video_part_map):
try:
subtitle_storage = get_subtitle_storage()
to_extract = []
item_count = 0
for scanned_video, part_info in video_part_map.iteritems():
plexapi_item = scanned_video.plexapi_metadata["item"]
stored_subs = subtitle_storage.load_or_new(plexapi_item)
valid_langs_in_media = audio_streams_match_languages(
scanned_video, config.get_lang_list(ordered=True))
if not config.lang_list.difference(valid_langs_in_media):
Log.Debug(
"Skipping embedded subtitle extraction for %s, audio streams are in correct language(s)",
plexapi_item.rating_key)
continue
for plexapi_part in get_all_parts(plexapi_item):
item_count = item_count + 1
used_one_unknown_stream = False
used_one_known_stream = False
for requested_language in config.lang_list:
skip_unknown = used_one_unknown_stream or used_one_known_stream
embedded_subs = stored_subs.get_by_provider(
plexapi_part.id, requested_language, "embedded")
current = stored_subs.get_any(plexapi_part.id, requested_language) or \
requested_language in scanned_video.external_subtitle_languages
if not embedded_subs:
stream_data = get_embedded_subtitle_streams(
plexapi_part,
requested_language=requested_language,
skip_unknown=skip_unknown)
if stream_data and stream_data[0]["language"]:
stream = stream_data[0]["stream"]
if stream_data[0]["is_unknown"]:
used_one_unknown_stream = True
else:
used_one_known_stream = True
to_extract.append(
({
scanned_video: part_info
}, plexapi_part, str(stream.index),
str(requested_language), not current))
if not cast_bool(
Prefs["subtitles.search_after_autoextract"]
):
scanned_video.subtitle_languages.update(
{requested_language})
else:
Log.Debug(
"Skipping embedded subtitle extraction for %s, already got %r from %s",
plexapi_item.rating_key, requested_language,
embedded_subs[0].id)
if to_extract:
Log.Info(
"Triggering extraction of %d embedded subtitles of %d items",
len(to_extract), item_count)
Thread.Create(
multi_extract_embedded,
stream_list=to_extract,
refresh=True,
with_mods=True,
single_thread=not config.advanced.auto_extract_multithread)
except:
Log.Error(
"Something went wrong when auto-extracting subtitles, continuing: %s",
traceback.format_exc())
def run(self):
super(FindBetterSubtitles, self).run()
self.running = True
better_found = 0
try:
max_search_days = int(Prefs[
"scheduler.tasks.FindBetterSubtitles.max_days_after_added"].
strip())
except ValueError:
Log.Error(
"Please only put numbers into the FindBetterSubtitles.max_days_after_added setting. Exiting"
)
return
else:
if max_search_days > 30:
Log.Error(
"FindBetterSubtitles.max_days_after_added is too big. Max is 30 days."
)
return
now = datetime.datetime.now()
subtitle_storage = get_subtitle_storage()
recent_subs = subtitle_storage.load_recent_files(
age_days=max_search_days)
for fn, stored_subs in recent_subs.iteritems():
video_id = stored_subs.video_id
cutoff = self.series_cutoff if stored_subs.item_type == "episode" else self.movies_cutoff
# don't search for better subtitles until at least 30 minutes have passed
if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
Log.Debug("Item %s too new, skipping", video_id)
continue
# added_date <= max_search_days?
if stored_subs.added_at + datetime.timedelta(
days=max_search_days) <= now:
continue
ditch_parts = []
# look through all stored subtitle data
for part_id, languages in stored_subs.parts.iteritems():
part_id = str(part_id)
# all languages
for language, current_subs in languages.iteritems():
current_key = current_subs.get("current")
current = current_subs.get(current_key)
# currently got subtitle?
if not current:
continue
current_score = current.score
current_mode = current.mode
# late cutoff met? skip
if current_score >= cutoff:
Log.Debug(
u"Skipping finding better subs, cutoff met (current: %s, cutoff: %s): %s",
current_score, cutoff, stored_subs.title)
continue
# got manual subtitle but don't want to touch those?
if current_mode == "m" and \
not cast_bool(Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"]):
Log.Debug(
u"Skipping finding better subs, had manual: %s",
stored_subs.title)
continue
try:
subs = self.list_subtitles(video_id,
stored_subs.item_type,
part_id, language)
except PartUnknownException:
Log.Info(
"Part %s unknown/gone; ditching subtitle info",
part_id)
ditch_parts.append(part_id)
continue
if subs:
# subs are already sorted by score
sub = subs[0]
if sub.score > current_score:
Log.Debug(
"Better subtitle found for %s, downloading",
video_id)
self.download_subtitle(sub, video_id, mode="b")
better_found += 1
if ditch_parts:
for part_id in ditch_parts:
try:
del stored_subs.parts[part_id]
except KeyError:
pass
subtitle_storage.save(stored_subs)
if better_found:
Log.Debug("Task: %s, done. Better subtitles found for %s items",
self.name, better_found)
self.running = False
def item_discover_missing_subs(rating_key, kind="show", added_at=None, section_title=None, internal=False, external=True, languages=()):
item_id = int(rating_key)
item = get_item(rating_key)
if kind == "show":
item_title = get_plex_item_display_title(item, kind, parent=item.season, section_title=section_title, parent_title=item.show.title)
else:
item_title = get_plex_item_display_title(item, kind, section_title=section_title)
subtitle_storage = get_subtitle_storage()
stored_subs = subtitle_storage.load(rating_key)
subtitle_storage.destroy()
subtitle_target_dir, tdir_is_absolute = config.subtitle_sub_dir
missing = set()
languages_set = set([Language.rebuild(l) for l in languages])
for media in item.media:
existing_subs = {"internal": [], "external": [], "own_external": [], "count": 0}
for part in media.parts:
# did we already download an external subtitle before?
if subtitle_target_dir and stored_subs:
for language in languages_set:
if has_external_subtitle(part.id, stored_subs, language):
# check the existence of the actual subtitle file
# get media filename without extension
part_basename = os.path.splitext(os.path.basename(part.file))[0]
# compute target directory for subtitle
# fixme: move to central location
if tdir_is_absolute:
possible_subtitle_path_base = subtitle_target_dir
else:
possible_subtitle_path_base = os.path.join(os.path.dirname(part.file), subtitle_target_dir)
possible_subtitle_path_base = os.path.realpath(possible_subtitle_path_base)
# folder actually exists?
if not os.path.isdir(possible_subtitle_path_base):
continue
found_any = False
for ext in config.subtitle_formats:
if cast_bool(Prefs['subtitles.only_one']):
possible_subtitle_path = os.path.join(possible_subtitle_path_base,
u"%s.%s" % (part_basename, ext))
else:
possible_subtitle_path = os.path.join(possible_subtitle_path_base,
u"%s.%s.%s" % (part_basename, language, ext))
# check for subtitle existence
if os.path.isfile(possible_subtitle_path):
found_any = True
Log.Debug(u"Found: %s", possible_subtitle_path)
break
if found_any:
existing_subs["own_external"].append(language)
existing_subs["count"] = existing_subs["count"] + 1
for stream in part.streams:
if stream.stream_type == 3:
is_forced = is_stream_forced(stream)
if stream.index:
key = "internal"
else:
key = "external"
if not config.exotic_ext and stream.codec.lower() not in TEXT_SUBTITLE_EXTS:
continue
# treat unknown language as lang1?
if not stream.language_code and config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
# we can't parse empty language codes
elif not stream.language_code or not stream.codec:
continue
else:
# parse with internal language parser first
try:
lang = get_language_from_stream(stream.language_code)
if not lang:
if config.treat_und_as_first:
lang = Language.rebuild(list(config.lang_list)[0])
else:
continue
except (ValueError, LanguageReverseError):
continue
if lang:
# Log.Debug("Found babelfish language: %r", lang)
lang.forced = is_forced
existing_subs[key].append(lang)
existing_subs["count"] = existing_subs["count"] + 1
missing_from_part = set([Language.rebuild(l) for l in languages])
if existing_subs["count"]:
# fixme: this is actually somewhat broken with IETF, as Plex doesn't store the country portion
# (pt instead of pt-BR) inside the database. So it might actually download pt-BR if there's a local pt-BR
# subtitle but not our own.
existing_flat = set((existing_subs["internal"] if internal else [])
+ (existing_subs["external"] if external else [])
+ existing_subs["own_external"])
check_languages = set([Language.rebuild(l) for l in languages])
alpha3_map = {}
if config.ietf_as_alpha3:
for language in existing_flat:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
for language in check_languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
# compare sets of strings, not sets of different Language instances
check_languages_str = set(str(l) for l in check_languages)
existing_flat_str = set(str(l) for l in existing_flat)
if check_languages_str.issubset(existing_flat_str) or \
(len(existing_flat) >= 1 and Prefs['subtitles.only_one']):
# all subs found
#Log.Info(u"All subtitles exist for '%s'", item_title)
continue
missing_from_part = set(Language.fromietf(l) for l in check_languages_str - existing_flat_str)
if config.ietf_as_alpha3:
for language in missing_from_part:
language.country = alpha3_map.get(language.alpha3, None)
if missing_from_part:
Log.Info(u"Subs still missing for '%s' (%s: %s): %s", item_title, rating_key, media.id,
missing_from_part)
missing.update(missing_from_part)
if missing:
# deduplicate
missing = set(Language.fromietf(la) for la in set(str(l) for l in missing))
return added_at, item_id, item_title, item, missing
def update(self, metadata, media, lang):
if not config.enable_agent:
Log.Debug("Skipping Sub-Zero agent(s)")
return
Log.Debug("Sub-Zero %s, %s update called" % (config.version, self.agent_type))
if not media:
Log.Error("Called with empty media, something is really wrong with your setup!")
return
intent = get_intent()
item_ids = []
try:
config.init_subliminal_patches()
all_videos = media_to_videos(media, kind=self.agent_type)
# media ignored?
ignore_parts_cleanup = []
videos = []
for video in all_videos:
if not is_wanted(video["id"], item=video["item"]):
Log.Debug(u'Skipping "%s"' % video["filename"])
ignore_parts_cleanup.append(video["path"])
continue
videos.append(video)
# find local media
update_local_media(all_videos, ignore_parts_cleanup=ignore_parts_cleanup)
if not videos:
Log.Debug(u"Nothing to do.")
return
try:
use_score = int(Prefs[self.score_prefs_key].strip())
except ValueError:
Log.Error("Please only put numbers into the scores setting. Exiting")
return
set_refresh_menu_state(media, media_type=self.agent_type)
# scanned_video_part_map = {subliminal.Video: plex_part, ...}
providers = config.get_providers(media_type=self.agent_type)
try:
scanned_video_part_map = scan_videos(videos, providers=providers)
except IOError, e:
Log.Exception("Permission error, please check your folder/file permissions. Exiting.")
if cast_bool(Prefs["check_permissions"]):
config.permissions_ok = False
config.missing_permissions = e.message
return
# auto extract embedded
if config.embedded_auto_extract:
if config.plex_transcoder:
agent_extract_embedded(scanned_video_part_map)
else:
Log.Warn("Plex Transcoder not found, can't auto extract")
# clear missing subtitles menu data
if not scheduler.is_task_running("MissingSubtitles"):
scheduler.clear_task_data("MissingSubtitles")
downloaded_subtitles = None
# debounce for self.debounce seconds
now = datetime.datetime.now()
if "last_call" in Dict:
last_call = Dict["last_call"]
if last_call + datetime.timedelta(seconds=self.debounce) > now:
wait = self.debounce - (now - last_call).seconds
if wait >= 1:
Log.Debug("Waiting %s seconds until continuing", wait)
Thread.Sleep(wait)
# downloaded_subtitles = {subliminal.Video: [subtitle, subtitle, ...]}
try:
downloaded_subtitles = download_best_subtitles(scanned_video_part_map, min_score=use_score,
throttle_time=self.debounce, providers=providers)
except:
Log.Exception("Something went wrong when downloading subtitles")
if downloaded_subtitles is not None:
Dict["last_call"] = datetime.datetime.now()
item_ids = get_media_item_ids(media, kind=self.agent_type)
downloaded_any = False
if downloaded_subtitles:
downloaded_any = any(downloaded_subtitles.values())
if downloaded_any:
save_successful = False
try:
save_successful = save_subtitles(scanned_video_part_map, downloaded_subtitles,
mods=config.default_mods)
except:
Log.Exception("Something went wrong when saving subtitles")
track_usage("Subtitle", "refreshed", "download", 1)
# store SZ meta info even if download wasn't successful
if not save_successful:
self.store_blank_subtitle_metadata(scanned_video_part_map)
else:
for video, video_subtitles in downloaded_subtitles.items():
# store item(s) in history
for subtitle in video_subtitles:
history = get_history()
item_title = get_title_for_video_metadata(video.plexapi_metadata, add_section_title=False)
history.add(item_title, video.id, section_title=video.plexapi_metadata["section"],
thumb=video.plexapi_metadata["super_thumb"],
subtitle=subtitle)
history.destroy()
else:
# store SZ meta info even if we've downloaded none
self.store_blank_subtitle_metadata(scanned_video_part_map)
update_local_media(videos)
