def run(self):
super(MigrateSubtitleStorage, self).run()
self.running = True
Log.Info(u"%s: Running subtitle storage migration", self.name)
storage = get_subtitle_storage()
for fn in storage.get_all_files():
if fn.endswith(".json.gz"):
continue
Log.Debug(u"%s: Migrating %s", self.name, fn)
storage.load(None, fn)
storage.destroy()
def run(self):
super(SubtitleStorageMaintenance, self).run()
self.running = True
Log.Info(u"%s: Running subtitle storage maintenance", self.name)
storage = get_subtitle_storage()
deleted_items = storage.delete_missing(wanted_languages=set(str(l) for l in config.lang_list))
if deleted_items:
Log.Info(u"%s: Subtitle information for %d non-existant videos have been cleaned up",
self.name, len(deleted_items))
Log.Debug(u"%s: Videos: %s", self.name, deleted_items)
else:
Log.Info(u"%s: Nothing to do", self.name)
storage.destroy()
def run(self):
super(MigrateSubtitleStorage, self).run()
self.running = True
Log.Info(u"%s: Running subtitle storage migration", self.name)
storage = get_subtitle_storage()
def migrate(scandir_generic=False):
for fn in storage.get_all_files(scandir_generic=scandir_generic):
if fn.endswith(".json.gz"):
continue
Log.Debug(u"%s: Migrating %s", self.name, fn)
storage.load(None, fn)
try:
migrate()
except OSError:
migrate(scandir_generic=True)
storage.destroy()
def run(self):
super(FindBetterSubtitles, self).run()
self.running = True
better_found = 0
try:
max_search_days = int(Prefs[
"scheduler.tasks.FindBetterSubtitles.max_days_after_added"].
strip())
except ValueError:
Log.Error(
u"Please only put numbers into the FindBetterSubtitles.max_days_after_added setting. Exiting"
)
return
else:
if max_search_days > 30:
Log.Error(
u"%s: FindBetterSubtitles.max_days_after_added is too big. Max is 30 days.",
self.name)
return
now = datetime.datetime.now()
min_score_series = int(
Prefs["subtitles.search.minimumTVScore2"].strip())
min_score_movies = int(
Prefs["subtitles.search.minimumMovieScore2"].strip())
min_score_extracted_series = config.advanced.find_better_as_extracted_tv_score or 352
min_score_extracted_movies = config.advanced.find_better_as_extracted_movie_score or 82
overwrite_manually_modified = cast_bool(Prefs[
"scheduler.tasks.FindBetterSubtitles.overwrite_manually_modified"])
overwrite_manually_selected = cast_bool(Prefs[
"scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"])
air_date_cutoff_pref = Prefs[
"scheduler.tasks.FindBetterSubtitles.air_date_cutoff"]
if air_date_cutoff_pref == "don't limit":
air_date_cutoff = None
else:
air_date_cutoff = int(air_date_cutoff_pref.split()[0])
subtitle_storage = get_subtitle_storage()
viable_item_count = 0
try:
for fn in subtitle_storage.get_recent_files(
age_days=max_search_days):
stored_subs = subtitle_storage.load(filename=fn)
if not stored_subs:
continue
video_id = stored_subs.video_id
if stored_subs.item_type == "episode":
cutoff = self.series_cutoff
min_score = min_score_series
min_score_extracted = min_score_extracted_series
else:
cutoff = self.movies_cutoff
min_score = min_score_movies
min_score_extracted = min_score_extracted_movies
# don't search for better subtitles until at least 30 minutes have passed
if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
Log.Debug(u"%s: Item %s too new, skipping", self.name,
video_id)
continue
# added_date <= max_search_days?
if stored_subs.added_at + datetime.timedelta(
days=max_search_days) <= now:
continue
viable_item_count += 1
ditch_parts = []
# look through all stored subtitle data
for part_id, languages in stored_subs.parts.iteritems():
part_id = str(part_id)
# all languages
for language, current_subs in languages.iteritems():
current_key = current_subs.get("current")
current = current_subs.get(current_key)
# currently got subtitle?
# fixme: check for existence
if not current:
continue
current_score = current.score
current_mode = current.mode
# late cutoff met? skip
if current_score >= cutoff:
Log.Debug(
u"%s: Skipping finding better subs, "
u"cutoff met (current: %s, cutoff: %s): %s (%s)",
self.name, current_score, cutoff,
stored_subs.title, video_id)
continue
# got manual subtitle but don't want to touch those?
if current_mode == "m" and not overwrite_manually_selected:
Log.Debug(
u"%s: Skipping finding better subs, "
u"had manual: %s (%s)", self.name,
stored_subs.title, video_id)
continue
# subtitle modifications different from default
if not overwrite_manually_modified and current.mods \
and set(current.mods).difference(set(config.default_mods)):
Log.Debug(
u"%s: Skipping finding better subs, it has manual modifications: %s (%s)",
self.name, stored_subs.title, video_id)
continue
try:
subs = self.list_subtitles(
video_id,
stored_subs.item_type,
part_id,
language,
air_date_cutoff=air_date_cutoff)
except PartUnknownException:
Log.Info(
u"%s: Part %s unknown/gone; ditching subtitle info",
self.name, part_id)
ditch_parts.append(part_id)
continue
hit_providers = subs is not None
if subs:
# subs are already sorted by score
better_downloaded = False
better_tried_download = 0
better_visited = 0
for sub in subs:
if sub.score > current_score and sub.score > min_score:
if current.provider_name == "embedded" and sub.score < min_score_extracted:
Log.Debug(
u"%s: Not downloading subtitle for %s, we've got an active extracted "
u"embedded sub and the min score %s isn't met (%s).",
self.name, video_id,
min_score_extracted, sub.score)
better_visited += 1
break
Log.Debug(
u"%s: Better subtitle found for %s, downloading",
self.name, video_id)
better_tried_download += 1
ret = self.download_subtitle(sub,
video_id,
mode="b")
if ret:
better_found += 1
better_downloaded = True
break
else:
Log.Debug(
u"%s: Couldn't download/save subtitle. "
u"Continuing to the next one",
self.name)
Log.Debug(
u"%s: Waiting %s seconds before continuing",
self.name, self.DL_PROVIDER_SLACK)
Thread.Sleep(self.DL_PROVIDER_SLACK)
better_visited += 1
if better_tried_download and not better_downloaded:
Log.Debug(
u"%s: Tried downloading better subtitle for %s, "
u"but every try failed.", self.name,
video_id)
elif better_downloaded:
Log.Debug(
u"%s: Better subtitle downloaded for %s",
self.name, video_id)
if better_tried_download or better_downloaded:
Log.Debug(
u"%s: Waiting %s seconds before continuing",
self.name, self.DL_PROVIDER_SLACK)
Thread.Sleep(self.DL_PROVIDER_SLACK)
elif better_visited:
Log.Debug(
u"%s: Waiting %s seconds before continuing",
self.name, self.PROVIDER_SLACK)
Thread.Sleep(self.PROVIDER_SLACK)
subs = None
elif hit_providers:
# hit the providers but didn't try downloading? wait.
Log.Debug(
u"%s: Waiting %s seconds before continuing",
self.name, self.PROVIDER_SLACK)
Thread.Sleep(self.PROVIDER_SLACK)
if ditch_parts:
for part_id in ditch_parts:
try:
del stored_subs.parts[part_id]
except KeyError:
pass
subtitle_storage.save(stored_subs)
ditch_parts = None
stored_subs = None
Thread.Sleep(1)
finally:
subtitle_storage.destroy()
if better_found:
Log.Debug(u"%s: done. Better subtitles found for %s/%s items",
self.name, better_found, viable_item_count)
else:
Log.Debug(u"%s: done. No better subtitles found for %s items",
self.name, viable_item_count)
def run(self):
super(SearchAllRecentlyAddedMissing, self).run()
self.running = True
self.prepare()
from support.history import get_history
history = get_history()
now = datetime.datetime.now()
min_score_series = int(
Prefs["subtitles.search.minimumTVScore2"].strip())
min_score_movies = int(
Prefs["subtitles.search.minimumMovieScore2"].strip())
series_providers = config.get_providers(media_type="series")
movie_providers = config.get_providers(media_type="movies")
is_recent_str = Prefs["scheduler.item_is_recent_age"]
num, ident = is_recent_str.split()
max_search_days = 0
if ident == "days":
max_search_days = int(num)
elif ident == "weeks":
max_search_days = int(num) * 7
subtitle_storage = get_subtitle_storage()
recent_files = subtitle_storage.get_recent_files(
age_days=max_search_days)
self.items_searching = len(recent_files)
download_count = 0
videos_with_downloads = 0
config.init_subliminal_patches()
Log.Info(u"%s: Searching for subtitles for %s items", self.name,
self.items_searching)
def skip_item():
self.items_searching = self.items_searching - 1
self.percentage = int(
self.items_done * 100 /
self.items_searching) if self.items_searching > 0 else 100
# search for subtitles in viable items
try:
for fn in recent_files:
stored_subs = subtitle_storage.load(filename=fn)
if not stored_subs:
Log.Debug("Skipping item %s because storage is empty", fn)
skip_item()
continue
video_id = stored_subs.video_id
# added_date <= max_search_days?
if stored_subs.added_at + datetime.timedelta(
days=max_search_days) <= now:
Log.Debug("Skipping item %s because it's too old",
video_id)
skip_item()
continue
if stored_subs.item_type == "episode":
min_score = min_score_series
providers = series_providers
else:
min_score = min_score_movies
providers = movie_providers
parts = []
plex_item = get_item(video_id)
if not plex_item:
Log.Info(u"%s: Item %s unknown, skipping", self.name,
video_id)
skip_item()
continue
if not is_wanted(video_id, item=plex_item):
skip_item()
continue
for media in plex_item.media:
parts += media.parts
downloads_per_video = 0
hit_providers = False
for part in parts:
part_id = part.id
try:
metadata = get_plex_metadata(video_id, part_id,
stored_subs.item_type)
except PartUnknownException:
Log.Info(u"%s: Part %s:%s unknown, skipping",
self.name, video_id, part_id)
continue
if not metadata:
Log.Info(u"%s: Part %s:%s unknown, skipping",
self.name, video_id, part_id)
continue
Log.Debug(u"%s: Looking for missing subtitles: %s",
self.name, get_item_title(plex_item))
scanned_parts = scan_videos([metadata],
providers=providers)
downloaded_subtitles = download_best_subtitles(
scanned_parts,
min_score=min_score,
providers=providers)
hit_providers = downloaded_subtitles is not None
download_successful = False
if downloaded_subtitles:
downloaded_any = any(downloaded_subtitles.values())
if not downloaded_any:
continue
try:
save_subtitles(scanned_parts,
downloaded_subtitles,
mode="a",
mods=config.default_mods)
Log.Debug(
u"%s: Downloaded subtitle for item with missing subs: %s",
self.name, video_id)
download_successful = True
refresh_item(video_id)
track_usage("Subtitle", "manual", "download", 1)
except:
Log.Error(
u"%s: Something went wrong when downloading specific subtitle: %s",
self.name, traceback.format_exc())
finally:
scanned_parts = None
try:
item_title = get_title_for_video_metadata(
metadata, add_section_title=False)
if download_successful:
# store item in history
for video, video_subtitles in downloaded_subtitles.items(
):
if not video_subtitles:
continue
for subtitle in video_subtitles:
downloads_per_video += 1
history.add(
item_title,
video.id,
section_title=metadata[
"section"],
thumb=video.plexapi_metadata[
"super_thumb"],
subtitle=subtitle,
mode="a")
downloaded_subtitles = None
except:
Log.Error(u"%s: DEBUG HIT: %s", self.name,
traceback.format_exc())
Log.Debug(u"%s: Waiting %s seconds before continuing",
self.name, self.PROVIDER_SLACK)
Thread.Sleep(self.PROVIDER_SLACK)
download_count += downloads_per_video
if downloads_per_video:
videos_with_downloads += 1
self.items_done = self.items_done + 1
self.percentage = int(
self.items_done * 100 /
self.items_searching) if self.items_searching > 0 else 100
stored_subs = None
if downloads_per_video:
Log.Debug(
u"%s: Subtitles have been downloaded, "
u"waiting %s seconds before continuing", self.name,
self.DL_PROVIDER_SLACK)
Thread.Sleep(self.DL_PROVIDER_SLACK)
else:
if hit_providers:
Log.Debug(u"%s: Waiting %s seconds before continuing",
self.name, self.PROVIDER_SLACK)
Thread.Sleep(self.PROVIDER_SLACK)
finally:
subtitle_storage.destroy()
history.destroy()
if download_count:
Log.Debug(
u"%s: done. Missing subtitles found for %s/%s items (%s subs downloaded)",
self.name, videos_with_downloads, self.items_searching,
download_count)
else:
Log.Debug(u"%s: done. No subtitles found for %s items", self.name,
self.items_searching)
def run(self):
super(FindBetterSubtitles, self).run()
self.running = True
better_found = 0
try:
max_search_days = int(Prefs[
"scheduler.tasks.FindBetterSubtitles.max_days_after_added"].
strip())
except ValueError:
Log.Error(
"Please only put numbers into the FindBetterSubtitles.max_days_after_added setting. Exiting"
)
return
else:
if max_search_days > 30:
Log.Error(
"FindBetterSubtitles.max_days_after_added is too big. Max is 30 days."
)
return
now = datetime.datetime.now()
subtitle_storage = get_subtitle_storage()
recent_subs = subtitle_storage.load_recent_files(
age_days=max_search_days)
for fn, stored_subs in recent_subs.iteritems():
video_id = stored_subs.video_id
cutoff = self.series_cutoff if stored_subs.item_type == "episode" else self.movies_cutoff
# don't search for better subtitles until at least 30 minutes have passed
if stored_subs.added_at + datetime.timedelta(minutes=30) > now:
Log.Debug("Item %s too new, skipping", video_id)
continue
# added_date <= max_search_days?
if stored_subs.added_at + datetime.timedelta(
days=max_search_days) <= now:
continue
ditch_parts = []
# look through all stored subtitle data
for part_id, languages in stored_subs.parts.iteritems():
part_id = str(part_id)
# all languages
for language, current_subs in languages.iteritems():
current_key = current_subs.get("current")
current = current_subs.get(current_key)
# currently got subtitle?
if not current:
continue
current_score = current.score
current_mode = current.mode
# late cutoff met? skip
if current_score >= cutoff:
Log.Debug(
u"Skipping finding better subs, cutoff met (current: %s, cutoff: %s): %s",
current_score, cutoff, stored_subs.title)
continue
# got manual subtitle but don't want to touch those?
if current_mode == "m" and \
not cast_bool(Prefs["scheduler.tasks.FindBetterSubtitles.overwrite_manually_selected"]):
Log.Debug(
u"Skipping finding better subs, had manual: %s",
stored_subs.title)
continue
try:
subs = self.list_subtitles(video_id,
stored_subs.item_type,
part_id, language)
except PartUnknownException:
Log.Info(
"Part %s unknown/gone; ditching subtitle info",
part_id)
ditch_parts.append(part_id)
continue
if subs:
# subs are already sorted by score
sub = subs[0]
if sub.score > current_score:
Log.Debug(
"Better subtitle found for %s, downloading",
video_id)
self.download_subtitle(sub, video_id, mode="b")
better_found += 1
if ditch_parts:
for part_id in ditch_parts:
try:
del stored_subs.parts[part_id]
except KeyError:
pass
subtitle_storage.save(stored_subs)
if better_found:
Log.Debug("Task: %s, done. Better subtitles found for %s items",
self.name, better_found)
self.running = False
