def download_best_subtitles(video_part_map, min_score=0):
hearing_impaired = Prefs['subtitles.search.hearingImpaired']
languages = config.lang_list
if not languages:
return
missing_languages = False
for video, part in video_part_map.iteritems():
if not Prefs['subtitles.save.filesystem']:
# scan for existing metadata subtitles
meta_subs = get_subtitles_from_metadata(part)
for language, subList in meta_subs.iteritems():
if subList:
video.subtitle_languages.add(language)
Log.Debug("Found metadata subtitle %s for %s", language,
video)
missing_subs = (languages - video.subtitle_languages)
# all languages are found if we either really have subs for all languages or we only want to have exactly one language
# and we've only found one (the case for a selected language, Prefs['subtitles.only_one'] (one found sub matches any language))
found_one_which_is_enough = len(
video.subtitle_languages) >= 1 and Prefs['subtitles.only_one']
if not missing_subs or found_one_which_is_enough:
if found_one_which_is_enough:
Log.Debug(
'Only one language was requested, and we\'ve got a subtitle for %s',
video)
else:
Log.Debug('All languages %r exist for %s', languages, video)
continue
missing_languages = True
break
if missing_languages:
Log.Debug(
"Download best subtitles using settings: min_score: %s, hearing_impaired: %s"
% (min_score, hearing_impaired))
return subliminal.download_best_subtitles(
video_part_map.keys(),
languages,
min_score,
hearing_impaired,
providers=config.providers,
provider_configs=config.provider_settings,
pool_class=config.provider_pool,
compute_score=compute_score)
Log.Debug("All languages for all requested videos exist. Doing nothing.")
def get_missing_languages(video, part):
languages = set([Language.fromietf(str(l)) for l in config.lang_list])
# should we treat IETF as alpha3? (ditch the country part)
alpha3_map = {}
if config.ietf_as_alpha3:
for language in languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
if not Prefs['subtitles.save.filesystem']:
# scan for existing metadata subtitles
meta_subs = get_subtitles_from_metadata(part)
for language, subList in meta_subs.iteritems():
if subList:
video.subtitle_languages.add(language)
Log.Debug("Found metadata subtitle %s for %s", language, video)
have_languages = video.subtitle_languages.copy()
if config.ietf_as_alpha3:
for language in have_languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
missing_languages = (set(str(l) for l in languages) -
set(str(l) for l in have_languages))
# all languages are found if we either really have subs for all languages or we only want to have exactly one language
# and we've only found one (the case for a selected language, Prefs['subtitles.only_one'] (one found sub matches any language))
found_one_which_is_enough = len(
video.subtitle_languages) >= 1 and Prefs['subtitles.only_one']
if not missing_languages or found_one_which_is_enough:
if found_one_which_is_enough:
Log.Debug(
'Only one language was requested, and we\'ve got a subtitle for %s',
video)
else:
Log.Debug('All languages %r exist for %s', languages, video)
return False
# re-add country codes to the missing languages, in case we've removed them above
if config.ietf_as_alpha3:
for language in languages:
language.country = alpha3_map.get(language.alpha3, None)
return missing_languages
def find_subtitles(part):
lang_sub_map = {}
part_filename = helpers.unicodize(part.file)
part_basename = os.path.splitext(os.path.basename(part_filename))[0]
use_filesystem = helpers.cast_bool(Prefs["subtitles.save.filesystem"])
paths = [os.path.dirname(part_filename)] if use_filesystem else []
global_subtitle_folder = None
global_folders = []
if use_filesystem:
# Check for local subtitles subdirectory
sub_dir_base = paths[0]
sub_dir_list = []
if Prefs["subtitles.save.subFolder"] != "current folder":
# got selected subfolder
sub_dir_list.append(
os.path.join(sub_dir_base, Prefs["subtitles.save.subFolder"]))
sub_dir_custom = Prefs["subtitles.save.subFolder.Custom"].strip() \
if Prefs["subtitles.save.subFolder.Custom"] else None
if sub_dir_custom:
# got custom subfolder
sub_dir_custom = os.path.normpath(sub_dir_custom)
if os.path.isdir(sub_dir_custom) and os.path.isabs(sub_dir_custom):
# absolute folder
sub_dir_list.append(sub_dir_custom)
global_folders.append(sub_dir_custom)
else:
# relative folder
fld = os.path.join(sub_dir_base, sub_dir_custom)
sub_dir_list.append(fld)
for sub_dir in sub_dir_list:
if os.path.isdir(sub_dir):
paths.append(sub_dir)
# Check for a global subtitle location
global_subtitle_folder = os.path.join(Core.app_support_path,
'Subtitles')
if os.path.exists(global_subtitle_folder):
paths.append(global_subtitle_folder)
global_folders.append(global_subtitle_folder)
# normalize all paths
paths = [
os.path.normpath(os.path.realpath(helpers.unicodize(path)))
for path in paths
]
# We start by building a dictionary of files to their absolute paths. We also need to know
# the number of media files that are actually present, in case the found local media asset
# is limited to a single instance per media file.
#
file_paths = {}
total_media_files = 0
media_files = []
for path in paths:
for file_path_listing in os.listdir(path.encode(
sz_config.fs_encoding)):
# When using os.listdir with a unicode path, it will always return a string using the
# NFD form. However, we internally are using the form NFC and therefore need to convert
# it to allow correct regex / comparisons to be performed.
#
file_path_listing = helpers.unicodize(file_path_listing)
if os.path.isfile(
os.path.join(path, file_path_listing).encode(
sz_config.fs_encoding)):
file_paths[file_path_listing.lower()] = os.path.join(
path, file_path_listing)
# If we've found an actual media file, we should record it.
(root, ext) = os.path.splitext(file_path_listing)
if ext.lower()[1:] in config.VIDEO_EXTS:
total_media_files += 1
# collect found media files
media_files.append(root)
# cleanup any leftover subtitle if no associated media file was found
if helpers.cast_bool(Prefs["subtitles.autoclean"]):
for path in paths:
# we can't housekeep the global subtitle folders as we don't know about *all* media files
# in a library; skip them
skip_path = False
for fld in global_folders:
if path.startswith(fld):
Log.Info("Skipping housekeeping of folder: %s", path)
skip_path = True
break
if skip_path:
continue
for file_path_listing in os.listdir(
path.encode(sz_config.fs_encoding)):
file_path_listing = helpers.unicodize(file_path_listing)
enc_fn = os.path.join(path, file_path_listing).encode(
sz_config.fs_encoding)
if os.path.isfile(enc_fn):
(root, ext) = os.path.splitext(file_path_listing)
# it's a subtitle file
if ext.lower()[1:] in config.SUBTITLE_EXTS:
# get fn without forced/default/normal tag
split_tag = root.rsplit(".", 1)
if len(split_tag) > 1 and split_tag[1].lower() in [
'forced', 'normal', 'default'
]:
root = split_tag[0]
# get associated media file name without language
sub_fn = subtitlehelpers.ENDSWITH_LANGUAGECODE_RE.sub(
"", root)
# subtitle basename and basename without possible language tag not found in collected
# media files? kill.
if root not in media_files and sub_fn not in media_files:
Log.Info("Removing leftover subtitle: %s",
os.path.join(path, file_path_listing))
try:
os.remove(enc_fn)
except (OSError, IOError):
Log.Error("Removing failed")
Log('Looking for subtitle media in %d paths with %d media files.',
len(paths), total_media_files)
Log('Paths: %s', ", ".join([helpers.unicodize(p) for p in paths]))
for file_path in file_paths.values():
local_filename = os.path.basename(file_path)
bn, ext = os.path.splitext(local_filename)
local_basename = helpers.unicodize(bn)
# get fn without forced/default/normal tag
split_tag = local_basename.rsplit(".", 1)
if len(split_tag) > 1 and split_tag[1].lower() in [
'forced', 'normal', 'default'
]:
local_basename = split_tag[0]
# split off possible language tag
local_basename2 = local_basename.rsplit('.', 1)[0]
filename_matches_part = local_basename == part_basename or local_basename2 == part_basename
filename_contains_part = part_basename in local_basename
if not ext.lower()[1:] in config.SUBTITLE_EXTS:
continue
# if the file is located within the global subtitle folders and its name doesn't match exactly, ignore it
if global_folders and not filename_matches_part:
skip_path = False
for fld in global_folders:
if file_path.startswith(fld):
skip_path = True
break
if skip_path:
continue
# determine whether to pick up the subtitle based on our match strictness
elif not filename_matches_part:
if sz_config.ext_match_strictness == "strict" or (
sz_config.ext_match_strictness == "loose"
and not filename_contains_part):
Log.Debug("%s doesn't match %s, skipping" %
(helpers.unicodize(local_filename),
helpers.unicodize(part_basename)))
continue
subtitle_helper = subtitlehelpers.subtitle_helpers(file_path)
if subtitle_helper is not None:
local_lang_map = subtitle_helper.process_subtitles(part)
for new_language, subtitles in local_lang_map.items():
# Add the possible new language along with the located subtitles so that we can validate them
# at the end...
#
if not lang_sub_map.has_key(new_language):
lang_sub_map[new_language] = []
lang_sub_map[
new_language] = lang_sub_map[new_language] + subtitles
# add known metadata subs to our sub list
if not use_filesystem:
for language, sub_list in subtitlehelpers.get_subtitles_from_metadata(
part).iteritems():
if sub_list:
if language not in lang_sub_map:
lang_sub_map[language] = []
lang_sub_map[language] = lang_sub_map[language] + sub_list
# Now whack subtitles that don't exist anymore.
for language in lang_sub_map.keys():
part.subtitles[language].validate_keys(lang_sub_map[language])
# Now whack the languages that don't exist anymore.
for language in list(
set(part.subtitles.keys()) - set(lang_sub_map.keys())):
part.subtitles[language].validate_keys({})
def download_best_subtitles(video_part_map, min_score=0, throttle_time=None):
hearing_impaired = Prefs['subtitles.search.hearingImpaired']
ietf_as_alpha3 = cast_bool(Prefs["subtitles.language.ietf_normalize"])
languages = set([Language.fromietf(str(l)) for l in config.lang_list])
if not languages:
return
# should we treat IETF as alpha3? (ditch the country part)
alpha3_map = {}
if ietf_as_alpha3:
for language in languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
missing_languages = False
for video, part in video_part_map.iteritems():
if not Prefs['subtitles.save.filesystem']:
# scan for existing metadata subtitles
meta_subs = get_subtitles_from_metadata(part)
for language, subList in meta_subs.iteritems():
if subList:
video.subtitle_languages.add(language)
Log.Debug("Found metadata subtitle %s for %s", language, video)
have_languages = video.subtitle_languages.copy()
if ietf_as_alpha3:
for language in have_languages:
if language.country:
alpha3_map[language.alpha3] = language.country
language.country = None
missing_subs = (set(str(l) for l in languages) - set(str(l) for l in have_languages))
# all languages are found if we either really have subs for all languages or we only want to have exactly one language
# and we've only found one (the case for a selected language, Prefs['subtitles.only_one'] (one found sub matches any language))
found_one_which_is_enough = len(video.subtitle_languages) >= 1 and Prefs['subtitles.only_one']
if not missing_subs or found_one_which_is_enough:
if found_one_which_is_enough:
Log.Debug('Only one language was requested, and we\'ve got a subtitle for %s', video)
else:
Log.Debug('All languages %r exist for %s', languages, video)
continue
missing_languages = True
break
if missing_languages:
# re-add country codes to the missing languages, in case we've removed them above
if ietf_as_alpha3:
for language in languages:
language.country = alpha3_map.get(language.alpha3, None)
Log.Debug("Download best subtitles using settings: min_score: %s, hearing_impaired: %s, languages: %s" %
(min_score, hearing_impaired, languages))
# prepare blacklist
blacklist = get_blacklist_from_part_map(video_part_map, languages)
return subliminal.download_best_subtitles(video_part_map.keys(), languages, min_score, hearing_impaired, providers=config.providers,
provider_configs=config.provider_settings, pool_class=config.provider_pool,
compute_score=compute_score, throttle_time=throttle_time, blacklist=blacklist)
Log.Debug("All languages for all requested videos exist. Doing nothing.")
def find_subtitles(part, ignore_parts_cleanup=None):
lang_sub_map = {}
ignore_parts_cleanup = ignore_parts_cleanup or []
part_filename = helpers.unicodize(part.file)
part_basename = os.path.splitext(os.path.basename(part_filename))[0]
use_filesystem = helpers.cast_bool(Prefs["subtitles.save.filesystem"])
sub_dir_custom = Prefs["subtitles.save.subFolder.Custom"].strip() \
if Prefs["subtitles.save.subFolder.Custom"] else None
use_sub_subfolder = Prefs["subtitles.save.subFolder"] != "current folder" and not sub_dir_custom
autoclean = helpers.cast_bool(Prefs["subtitles.autoclean"])
sub_subfolder = None
paths = [os.path.dirname(part_filename)] if use_filesystem else []
global_folders = []
if use_filesystem:
# Check for local subtitles subdirectory
sub_dir_base = paths[0]
sub_dir_list = []
if use_sub_subfolder:
# got selected subfolder
sub_subfolder = os.path.join(sub_dir_base, Prefs["subtitles.save.subFolder"])
sub_dir_list.append(sub_subfolder)
sub_subfolder = os.path.normpath(helpers.unicodize(sub_subfolder))
if sub_dir_custom:
# got custom subfolder
sub_dir_custom = os.path.normpath(sub_dir_custom)
if os.path.isdir(sub_dir_custom) and os.path.isabs(sub_dir_custom):
# absolute folder
sub_dir_list.append(sub_dir_custom)
global_folders.append(sub_dir_custom)
else:
# relative folder
fld = os.path.join(sub_dir_base, sub_dir_custom)
sub_dir_list.append(fld)
for sub_dir in sub_dir_list:
if os.path.isdir(sub_dir):
paths.append(sub_dir)
# Check for a global subtitle location
global_subtitle_folder = os.path.join(Core.app_support_path, 'Subtitles')
if os.path.exists(global_subtitle_folder):
paths.append(global_subtitle_folder)
global_folders.append(global_subtitle_folder)
# normalize all paths
paths = [os.path.normpath(helpers.unicodize(path)) for path in paths]
# We start by building a dictionary of files to their absolute paths. We also need to know
# the number of media files that are actually present, in case the found local media asset
# is limited to a single instance per media file.
#
file_paths = {}
total_media_files = 0
media_files = []
for path in paths:
for file_path_listing in os.listdir(path.encode(sz_config.fs_encoding)):
# When using os.listdir with a unicode path, it will always return a string using the
# NFD form. However, we internally are using the form NFC and therefore need to convert
# it to allow correct regex / comparisons to be performed.
#
file_path_listing = helpers.unicodize(file_path_listing)
if os.path.isfile(os.path.join(path, file_path_listing).encode(sz_config.fs_encoding)):
file_paths[file_path_listing.lower()] = os.path.join(path, file_path_listing)
# If we've found an actual media file, we should record it.
(root, ext) = os.path.splitext(file_path_listing)
if ext.lower()[1:] in config.VIDEO_EXTS:
total_media_files += 1
# collect found media files
media_files.append(root)
# cleanup any leftover subtitle if no associated media file was found
if autoclean and ignore_parts_cleanup:
Log.Info("Skipping housekeeping of: %s", paths)
if use_filesystem and autoclean and not ignore_parts_cleanup:
for path in paths:
# only housekeep in sub_subfolder if sub_subfolder is used
if use_sub_subfolder and path != sub_subfolder and not sz_config.advanced.thorough_cleaning:
continue
# we can't housekeep the global subtitle folders as we don't know about *all* media files
# in a library; skip them
skip_path = False
for fld in global_folders:
if path.startswith(fld):
Log.Info("Skipping housekeeping of folder: %s", path)
skip_path = True
break
if skip_path:
continue
for file_path_listing in os.listdir(path.encode(sz_config.fs_encoding)):
file_path_listing = helpers.unicodize(file_path_listing)
enc_fn = os.path.join(path, file_path_listing).encode(sz_config.fs_encoding)
if os.path.isfile(enc_fn):
(root, ext) = os.path.splitext(file_path_listing)
# it's a subtitle file
if ext.lower()[1:] in config.SUBTITLE_EXTS_BASE:
# get fn without forced/default/normal tag
split_tag = root.rsplit(".", 1)
if len(split_tag) > 1 and split_tag[1].lower() in SECONDARY_TAGS:
root = split_tag[0]
# get associated media file name without language
sub_fn = subtitlehelpers.ENDSWITH_LANGUAGECODE_RE.sub("", root)
# subtitle basename and basename without possible language tag not found in collected
# media files? kill.
if root not in media_files and sub_fn not in media_files:
Log.Info("Removing leftover subtitle: %s", os.path.join(path, file_path_listing))
try:
os.remove(enc_fn)
except (OSError, IOError):
Log.Error("Removing failed")
Log('Looking for subtitle media in %d paths with %d media files.', len(paths), total_media_files)
Log('Paths: %s', ", ".join([helpers.unicodize(p) for p in paths]))
for file_path in file_paths.values():
local_filename = os.path.basename(file_path)
bn, ext = os.path.splitext(local_filename)
local_basename = helpers.unicodize(bn)
# get fn without forced/default/normal tag
split_tag = local_basename.rsplit(".", 1)
has_additional_tag = False
if len(split_tag) > 1 and split_tag[1].lower() in SECONDARY_TAGS:
local_basename = split_tag[0]
has_additional_tag = True
# split off possible language tag
local_basename2 = local_basename.rsplit('.', 1)[0]
filename_matches_part = local_basename == part_basename or local_basename2 == part_basename
filename_contains_part = part_basename in local_basename
if not ext.lower()[1:] in config.SUBTITLE_EXTS:
continue
# if the file is located within the global subtitle folders and its name doesn't match exactly, ignore it
if global_folders and not filename_matches_part:
skip_path = False
for fld in global_folders:
if file_path.startswith(fld):
skip_path = True
break
if skip_path:
continue
# determine whether to pick up the subtitle based on our match strictness
if not filename_matches_part:
if sz_config.ext_match_strictness == "strict" or (
sz_config.ext_match_strictness == "loose" and not filename_contains_part):
# Log.Debug("%s doesn't match %s, skipping" % (helpers.unicodize(local_filename),
# helpers.unicodize(part_basename)))
continue
subtitle_helper = subtitlehelpers.subtitle_helpers(file_path)
if subtitle_helper is not None:
local_lang_map = subtitle_helper.process_subtitles(part)
for new_language, subtitles in local_lang_map.items():
# Add the possible new language along with the located subtitles so that we can validate them
# at the end...
#
if not lang_sub_map.has_key(new_language):
lang_sub_map[new_language] = []
lang_sub_map[new_language] = lang_sub_map[new_language] + subtitles
# add known metadata subs to our sub list
if not use_filesystem:
for language, sub_list in subtitlehelpers.get_subtitles_from_metadata(part).iteritems():
if sub_list:
if language not in lang_sub_map:
lang_sub_map[language] = []
lang_sub_map[language] = lang_sub_map[language] + sub_list
# Now whack subtitles that don't exist anymore.
for language in lang_sub_map.keys():
part.subtitles[language].validate_keys(lang_sub_map[language])
# Now whack the languages that don't exist anymore.
for language in list(set(part.subtitles.keys()) - set(lang_sub_map.keys())):
part.subtitles[language].validate_keys({})
def find_subtitles(part):
lang_sub_map = {}
part_filename = helpers.unicodize(part.file)
part_basename = os.path.splitext(os.path.basename(part_filename))[0]
use_filesystem = bool(Prefs["subtitles.save.filesystem"])
paths = [os.path.dirname(part_filename)] if use_filesystem else []
global_subtitle_folder = None
if use_filesystem:
# Check for local subtitles subdirectory
sub_dir_base = paths[0]
sub_dir_list = []
if Prefs["subtitles.save.subFolder"] != "current folder":
# got selected subfolder
sub_dir_list.append(os.path.join(sub_dir_base, Prefs["subtitles.save.subFolder"]))
sub_dir_custom = Prefs["subtitles.save.subFolder.Custom"].strip() if bool(Prefs["subtitles.save.subFolder.Custom"]) else None
if sub_dir_custom:
# got custom subfolder
if sub_dir_custom.startswith("/"):
# absolute folder
sub_dir_list.append(sub_dir_custom)
else:
# relative folder
sub_dir_list.append(os.path.join(sub_dir_base, sub_dir_custom))
for sub_dir in sub_dir_list:
if os.path.isdir(sub_dir):
paths.append(sub_dir)
# Check for a global subtitle location
global_subtitle_folder = os.path.join(Core.app_support_path, 'Subtitles')
if os.path.exists(global_subtitle_folder):
paths.append(global_subtitle_folder)
# We start by building a dictionary of files to their absolute paths. We also need to know
# the number of media files that are actually present, in case the found local media asset
# is limited to a single instance per media file.
#
file_paths = {}
total_media_files = 0
for path in paths:
path = helpers.unicodize(path)
for file_path_listing in os.listdir(path.encode(sz_config.fs_encoding)):
# When using os.listdir with a unicode path, it will always return a string using the
# NFD form. However, we internally are using the form NFC and therefore need to convert
# it to allow correct regex / comparisons to be performed.
#
file_path_listing = helpers.unicodize(file_path_listing)
if os.path.isfile(os.path.join(path, file_path_listing).encode(sz_config.fs_encoding)):
file_paths[file_path_listing.lower()] = os.path.join(path, file_path_listing)
# If we've found an actual media file, we should record it.
(root, ext) = os.path.splitext(file_path_listing)
if ext.lower()[1:] in config.VIDEO_EXTS:
total_media_files += 1
Log('Looking for subtitle media in %d paths with %d media files.', len(paths), total_media_files)
Log('Paths: %s', ", ".join([helpers.unicodize(p) for p in paths]))
for file_path in file_paths.values():
local_basename = helpers.unicodize(os.path.splitext(os.path.basename(file_path))[0])
local_basename2 = local_basename.rsplit('.', 1)[0]
filename_matches_part = local_basename == part_basename or local_basename2 == part_basename
# If the file is located within the global subtitle folder and it's name doesn't match exactly
# then we should simply ignore it.
#
if global_subtitle_folder and file_path.count(global_subtitle_folder) and not filename_matches_part:
continue
# If we have more than one media file within the folder and located filename doesn't match
# exactly then we should simply ignore it.
#
if total_media_files > 1 and not filename_matches_part:
continue
subtitle_helper = subtitlehelpers.subtitle_helpers(file_path)
if subtitle_helper != None:
local_lang_map = subtitle_helper.process_subtitles(part)
for new_language, subtitles in local_lang_map.items():
# Add the possible new language along with the located subtitles so that we can validate them
# at the end...
#
if not lang_sub_map.has_key(new_language):
lang_sub_map[new_language] = []
lang_sub_map[new_language] = lang_sub_map[new_language] + subtitles
# add known metadata subs to our sub list
if not use_filesystem:
for language, sub_list in subtitlehelpers.get_subtitles_from_metadata(part).iteritems():
if sub_list:
if language not in lang_sub_map:
lang_sub_map[language] = []
lang_sub_map[language] = lang_sub_map[language] + sub_list
# Now whack subtitles that don't exist anymore.
for language in lang_sub_map.keys():
part.subtitles[language].validate_keys(lang_sub_map[language])
# Now whack the languages that don't exist anymore.
for language in list(set(part.subtitles.keys()) - set(lang_sub_map.keys())):
part.subtitles[language].validate_keys({})