def postprocessEpisode(item):
postprocess(item)
if 'audio_language' in item and item['audio_language'] is not None:
item['audio_language'] = get_audio_profile_languages(episode_id=item['sonarrEpisodeId'])
if 'subtitles' in item:
if item['subtitles'] is None:
raw_subtitles = []
else:
raw_subtitles = ast.literal_eval(item['subtitles'])
subtitles = []
for subs in raw_subtitles:
subtitle = subs[0].split(':')
sub = {"name": language_from_alpha2(subtitle[0]),
"code2": subtitle[0],
"code3": alpha3_from_alpha2(subtitle[0]),
"path": path_mappings.path_replace(subs[1]),
"forced": False,
"hi": False}
if len(subtitle) > 1:
sub["forced"] = True if subtitle[1] == 'forced' else False
sub["hi"] = True if subtitle[1] == 'hi' else False
subtitles.append(sub)
item.update({"subtitles": subtitles})
# Parse missing subtitles
if 'missing_subtitles' in item:
if item['missing_subtitles'] is None:
item['missing_subtitles'] = []
else:
item['missing_subtitles'] = ast.literal_eval(item['missing_subtitles'])
for i, subs in enumerate(item['missing_subtitles']):
subtitle = subs.split(':')
item['missing_subtitles'][i] = {"name": language_from_alpha2(subtitle[0]),
"code2": subtitle[0],
"code3": alpha3_from_alpha2(subtitle[0]),
"forced": False,
"hi": False}
if len(subtitle) > 1:
item['missing_subtitles'][i].update({
"forced": True if subtitle[1] == 'forced' else False,
"hi": True if subtitle[1] == 'hi' else False
})
if 'scene_name' in item:
item["sceneName"] = item["scene_name"]
del item["scene_name"]
if 'path' in item and item['path']:
# Provide mapped path
item['path'] = path_mappings.path_replace(item['path'])
def _get_language_obj(languages):
language_set = set()
if not isinstance(languages, (set, list)):
languages = [languages]
for language in languages:
lang, hi_item, forced_item = language
if hi_item == "True":
hi = "force HI"
else:
hi = "force non-HI"
# Always use alpha2 in API Request
lang = alpha3_from_alpha2(lang)
lang_obj = _get_lang_obj(lang)
if forced_item == "True":
lang_obj = Language.rebuild(lang_obj, forced=True)
if hi == "force HI":
lang_obj = Language.rebuild(lang_obj, hi=True)
language_set.add(lang_obj)
return language_set
def historystats():
data_providers = database.execute(
"SELECT DISTINCT provider FROM table_history WHERE provider IS NOT null "
"UNION SELECT DISTINCT provider FROM table_history_movie WHERE provider "
"IS NOT null")
data_providers_list = []
for item in data_providers:
data_providers_list.append(item['provider'])
data_languages = database.execute(
"SELECT DISTINCT language FROM table_history WHERE language IS NOT null "
"AND language != '' UNION SELECT DISTINCT language FROM table_history_movie "
"WHERE language IS NOT null AND language != ''")
data_languages_list = []
for item in data_languages:
splitted_lang = item['language'].split(':')
item = {
"name": language_from_alpha2(splitted_lang[0]),
"code2": splitted_lang[0],
"code3": alpha3_from_alpha2(splitted_lang[0]),
"forced": True if len(splitted_lang) > 1 else False
}
data_languages_list.append(item)
return render_template('historystats.html',
data_providers=data_providers_list,
data_languages=sorted(data_languages_list,
key=lambda i: i['name']))
def postprocess(item):
# Remove ffprobe_cache
if 'ffprobe_cache' in item:
del (item['ffprobe_cache'])
# Parse tags
if 'tags' in item:
if item['tags'] is None:
item['tags'] = []
else:
item['tags'] = ast.literal_eval(item['tags'])
if 'monitored' in item:
if item['monitored'] is None:
item['monitored'] = False
else:
item['monitored'] = item['monitored'] == 'True'
if 'hearing_impaired' in item and item['hearing_impaired'] is not None:
if item['hearing_impaired'] is None:
item['hearing_impaired'] = False
else:
item['hearing_impaired'] = item['hearing_impaired'] == 'True'
if 'language' in item:
if item['language'] == 'None':
item['language'] = None
elif item['language'] is not None:
splitted_language = item['language'].split(':')
item['language'] = {"name": language_from_alpha2(splitted_language[0]),
"code2": splitted_language[0],
"code3": alpha3_from_alpha2(splitted_language[0]),
"forced": True if item['language'].endswith(':forced') else False,
"hi": True if item['language'].endswith(':hi') else False}
def series_download_subtitles(no):
if get_general_settings()[24] is True:
monitored_only_query_string = ' AND monitored = "True"'
else:
monitored_only_query_string = ""
conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
episodes_details = c_db.execute(
'SELECT path, missing_subtitles, sonarrEpisodeId, scene_name FROM table_episodes WHERE sonarrSeriesId = ? AND missing_subtitles != "[]"'
+ monitored_only_query_string, (no, )).fetchall()
series_details = c_db.execute(
"SELECT hearing_impaired FROM table_shows WHERE sonarrSeriesId = ?",
(no, )).fetchone()
c_db.close()
providers_list = get_providers()
providers_auth = get_providers_auth()
for episode in episodes_details:
for language in ast.literal_eval(episode[1]):
if language is not None:
message = download_subtitle(path_replace(episode[0]),
str(alpha3_from_alpha2(language)),
series_details[0],
providers_list, providers_auth,
str(episode[3]), 'series')
if message is not None:
store_subtitles(path_replace(episode[0]))
history_log(1, no, episode[2], message)
send_notifications(no, episode[2], message)
list_missing_subtitles(no)
def series_download_subtitles(no):
if settings.sonarr.getboolean('only_monitored'):
monitored_only_query_string = ' AND monitored = "True"'
else:
monitored_only_query_string = ""
conn_db = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
episodes_details = c_db.execute(
'SELECT path, missing_subtitles, sonarrEpisodeId, scene_name FROM table_episodes WHERE sonarrSeriesId = ? AND missing_subtitles != "[]"'
+ monitored_only_query_string, (no, )).fetchall()
series_details = c_db.execute(
"SELECT hearing_impaired, title FROM table_shows WHERE sonarrSeriesId = ?",
(no, )).fetchone()
c_db.close()
providers_list = get_providers()
providers_auth = get_providers_auth()
count_episodes_details = len(episodes_details)
for i, episode in enumerate(episodes_details, 1):
for language in ast.literal_eval(episode[1]):
if language is not None:
notifications.write(msg='Searching for series subtitles...',
queue='get_subtitle',
item=i,
length=count_episodes_details)
result = download_subtitle(path_replace(episode[0]),
str(alpha3_from_alpha2(language)),
series_details[0],
providers_list, providers_auth,
str(episode[3]), series_details[1],
'series')
if result is not None:
message = result[0]
path = result[1]
language_code = result[2]
provider = result[3]
score = result[4]
store_subtitles(path_replace(episode[0]))
history_log(1, no, episode[2], message, path,
language_code, provider, score)
send_notifications(no, episode[2], message)
list_missing_subtitles(no)
if count_episodes_details:
notifications.write(msg='Searching completed. Please reload the page.',
type='success',
duration='permanent',
button='refresh',
queue='get_subtitle')
def wanted_download_subtitles(path):
conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
episodes_details = c_db.execute(
"SELECT table_episodes.path, table_episodes.missing_subtitles, table_episodes.sonarrEpisodeId, table_episodes.sonarrSeriesId, table_shows.hearing_impaired, table_episodes.scene_name, table_episodes.failedAttempts FROM table_episodes INNER JOIN table_shows on table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE table_episodes.path = ? AND missing_subtitles != '[]'",
(path_replace_reverse(path), )).fetchall()
c_db.close()
providers_list = get_providers()
providers_auth = get_providers_auth()
for episode in episodes_details:
attempt = episode[6]
if type(attempt) == unicode:
attempt = ast.literal_eval(attempt)
for language in ast.literal_eval(episode[1]):
if attempt is None:
attempt = []
attempt.append([language, time.time()])
else:
att = zip(*attempt)[0]
if language not in att:
attempt.append([language, time.time()])
conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
c_db.execute(
'UPDATE table_episodes SET failedAttempts = ? WHERE sonarrEpisodeId = ?',
(unicode(attempt), episode[2]))
conn_db.commit()
c_db.close()
for i in range(len(attempt)):
if attempt[i][0] == language:
if search_active(attempt[i][1]) is True:
message = download_subtitle(
path_replace(episode[0]),
str(alpha3_from_alpha2(language)),
episode[4], providers_list, providers_auth,
str(episode[5]), 'series')
if message is not None:
store_subtitles(path_replace(episode[0]))
list_missing_subtitles(episode[3])
history_log(1, episode[3], episode[2], message)
send_notifications(episode[3], episode[2], message)
else:
logging.debug(
'BAZARR Search is not active for episode ' +
episode[0] + ' Language: ' + attempt[i][0])
def wanted_download_subtitles_movie(path):
conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
movies_details = c_db.execute(
"SELECT path, missing_subtitles, radarrId, radarrId, hearing_impaired, sceneName, failedAttempts FROM table_movies WHERE path = ? AND missing_subtitles != '[]'",
(path_replace_reverse_movie(path), )).fetchall()
c_db.close()
providers_list = get_providers()
providers_auth = get_providers_auth()
for movie in movies_details:
attempt = movie[6]
if type(attempt) == unicode:
attempt = ast.literal_eval(attempt)
for language in ast.literal_eval(movie[1]):
if attempt is None:
attempt = []
attempt.append([language, time.time()])
else:
att = zip(*attempt)[0]
if language not in att:
attempt.append([language, time.time()])
conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
c_db.execute(
'UPDATE table_movies SET failedAttempts = ? WHERE radarrId = ?',
(unicode(attempt), movie[2]))
conn_db.commit()
c_db.close()
for i in range(len(attempt)):
if attempt[i][0] == language:
if search_active(attempt[i][1]) is True:
message = download_subtitle(
path_replace_movie(movie[0]),
str(alpha3_from_alpha2(language)),
movie[4], providers_list, providers_auth,
str(movie[5]), 'movie')
if message is not None:
store_subtitles_movie(path_replace_movie(movie[0]))
list_missing_subtitles_movies(movie[3])
history_log_movie(1, movie[3], message)
send_notifications_movie(movie[3], message)
else:
logging.info('BAZARR Search is not active for movie ' +
movie[0] + ' Language: ' + attempt[i][0])
def translate_subtitles_file(video_path, source_srt_file, to_lang, forced, hi):
to_lang = alpha3_from_alpha2(to_lang)
lang_obj = Language(to_lang)
if forced:
lang_obj = Language.rebuild(lang_obj, forced=True)
if hi:
lang_obj = Language.rebuild(lang_obj, hi=True)
logging.debug('BAZARR is translating in {0} this subtitles {1}'.format(lang_obj, source_srt_file))
max_characters = 5000
dest_srt_file = get_subtitle_path(video_path, language=lang_obj, extension='.srt', forced_tag=forced, hi_tag=hi)
subs = pysubs2.load(source_srt_file, encoding='utf-8')
lines_list = [x.plaintext for x in subs]
joined_lines_str = '\n\n\n'.join(lines_list)
logging.debug('BAZARR splitting subtitles into {} characters blocks'.format(max_characters))
lines_block_list = []
translated_lines_list = []
while len(joined_lines_str):
partial_lines_str = joined_lines_str[:max_characters]
if len(joined_lines_str) > max_characters:
new_partial_lines_str = partial_lines_str.rsplit('\n\n\n', 1)[0]
else:
new_partial_lines_str = partial_lines_str
lines_block_list.append(new_partial_lines_str)
joined_lines_str = joined_lines_str.replace(new_partial_lines_str, '')
logging.debug('BAZARR is sending {} blocks to Google Translate'.format(len(lines_block_list)))
for block_str in lines_block_list:
try:
translated_partial_srt_text = GoogleTranslator(source='auto',
target=lang_obj.basename).translate(text=block_str)
except:
return False
else:
translated_partial_srt_list = translated_partial_srt_text.split('\n\n\n')
translated_lines_list += translated_partial_srt_list
logging.debug('BAZARR saving translated subtitles to {}'.format(dest_srt_file))
for i, line in enumerate(subs):
line.plaintext = translated_lines_list[i]
subs.save(dest_srt_file)
return dest_srt_file
def movies_download_subtitles(no):
conn_db = sqlite3.connect(os.path.join(config_dir, 'db/bazarr.db'), timeout=30)
c_db = conn_db.cursor()
movie = c_db.execute("SELECT path, missing_subtitles, radarrId, sceneName, hearing_impaired FROM table_movies WHERE radarrId = ?", (no,)).fetchone()
c_db.close()
providers_list = get_providers()
providers_auth = get_providers_auth()
for language in ast.literal_eval(movie[1]):
if language is not None:
message = download_subtitle(path_replace_movie(movie[0]), str(alpha3_from_alpha2(language)), movie[4], providers_list, providers_auth, str(movie[3]), 'movie')
if message is not None:
store_subtitles_movie(path_replace_movie(movie[0]))
history_log_movie(1, no, message)
send_notifications_movie(no, message)
list_missing_subtitles_movies(no)
def episode_download_subtitles(no):
if settings.sonarr.getboolean('only_monitored'):
monitored_only_query_string = ' AND monitored = "True"'
else:
monitored_only_query_string = ""
conn_db = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
episodes_details = c_db.execute(
'SELECT table_episodes.path, table_episodes.missing_subtitles, table_episodes.sonarrEpisodeId, table_episodes.scene_name, table_shows.hearing_impaired, table_shows.title, table_shows.sonarrSeriesId FROM table_episodes INNER JOIN table_shows on table_shows.sonarrSeriesId = table_episodes.sonarrSeriesId WHERE table_episodes.sonarrEpisodeId = ?'
+ monitored_only_query_string, (no, )).fetchall()
c_db.close()
providers_list = get_providers()
providers_auth = get_providers_auth()
for episode in episodes_details:
for language in ast.literal_eval(episode[1]):
if language is not None:
notifications.write(msg='Searching for ' +
str(language_from_alpha2(language)) +
' subtitles for this episode: ' +
path_replace(episode[0]),
queue='get_subtitle')
result = download_subtitle(path_replace(episode[0]),
str(alpha3_from_alpha2(language)),
episode[4],
providers_list, providers_auth,
str(episode[3]), episode[5],
'series')
if result is not None:
message = result[0]
path = result[1]
language_code = result[2]
provider = result[3]
score = result[4]
store_subtitles(path_replace(episode[0]))
history_log(1, episode[6], episode[2], message, path,
language_code, provider, score)
send_notifications(episode[6], episode[2], message)
list_missing_subtitles(episode[6])
def movies_download_subtitles(no):
conn_db = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'),
timeout=30)
c_db = conn_db.cursor()
movie = c_db.execute(
"SELECT path, missing_subtitles, radarrId, sceneName, hearing_impaired, title FROM table_movies WHERE radarrId = ?",
(no, )).fetchone()
c_db.close()
providers_list = get_providers()
providers_auth = get_providers_auth()
count_movie = len(ast.literal_eval(movie[1]))
for i, language in enumerate(ast.literal_eval(movie[1]), 1):
if language is not None:
notifications.write(msg='Searching for movies subtitles',
queue='get_subtitle',
item=i,
length=count_movie)
result = download_subtitle(path_replace_movie(movie[0]),
str(alpha3_from_alpha2(language)),
movie[4],
providers_list, providers_auth,
str(movie[3]), movie[5], 'movie')
if result is not None:
message = result[0]
path = result[1]
language_code = result[2]
provider = result[3]
score = result[4]
store_subtitles_movie(path_replace_movie(movie[0]))
history_log_movie(1, no, message, path, language_code,
provider, score)
send_notifications_movie(no, message)
list_missing_subtitles_movies(no)
if count_movie:
notifications.write(msg='Searching completed. Please reload the page.',
type='success',
duration='permanent',
button='refresh',
queue='get_subtitle')
def subtitles_apply_mods(language, subtitle_path, mods):
language = alpha3_from_alpha2(language)
custom = CustomLanguage.from_value(language, "alpha3")
if custom is None:
lang_obj = Language(language)
else:
lang_obj = custom.subzero_language()
sub = Subtitle(lang_obj, mods=mods)
with open(subtitle_path, 'rb') as f:
sub.content = f.read()
if not sub.is_valid():
logging.exception('BAZARR Invalid subtitle file: ' + subtitle_path)
return
content = sub.get_modified_content()
if content:
if os.path.exists(subtitle_path):
os.remove(subtitle_path)
with open(subtitle_path, 'wb') as f:
f.write(content)
def subtitles_apply_mods(language, subtitle_path, mods):
language = alpha3_from_alpha2(language)
if language == 'pob':
lang_obj = Language('por', 'BR')
elif language == 'zht':
lang_obj = Language('zho', 'TW')
else:
lang_obj = Language(language)
sub = Subtitle(lang_obj, mods=mods)
with open(subtitle_path, 'rb') as f:
sub.content = f.read()
if not sub.is_valid():
logging.exception('BAZARR Invalid subtitle file: ' + subtitle_path)
return
content = sub.get_modified_content()
if content:
if os.path.exists(subtitle_path):
os.remove(subtitle_path)
with open(subtitle_path, 'wb') as f:
f.write(content)
def _get_language_obj(profile_id):
initial_language_set = set()
language_set = set()
# where [3] is items list of dict(id, lang, forced, hi)
language_items = get_profiles_list(profile_id=int(profile_id))['items']
for language in language_items:
forced = language['forced']
hi = language['hi']
language = language['language']
lang = alpha3_from_alpha2(language)
lang_obj = _get_lang_obj(lang)
if forced == "True":
lang_obj = Language.rebuild(lang_obj, forced=True)
if hi == "True":
lang_obj = Language.rebuild(lang_obj, hi=True)
initial_language_set.add(lang_obj)
language_set = initial_language_set.copy()
for language in language_set.copy():
lang_obj_for_hi = language
if not language.forced and not language.hi:
lang_obj_hi = Language.rebuild(lang_obj_for_hi, hi=True)
elif not language.forced and language.hi:
lang_obj_hi = Language.rebuild(lang_obj_for_hi, hi=False)
else:
continue
language_set.add(lang_obj_hi)
return language_set, initial_language_set
def manual_upload_subtitle(path, language, forced, hi, title, scene_name, media_type, subtitle, audio_language):
logging.debug('BAZARR Manually uploading subtitles for this file: ' + path)
single = settings.general.getboolean('single_language')
use_postprocessing = settings.general.getboolean('use_postprocessing')
postprocessing_cmd = settings.general.postprocessing_cmd
chmod = int(settings.general.chmod, 8) if not sys.platform.startswith(
'win') and settings.general.getboolean('chmod_enabled') else None
language = alpha3_from_alpha2(language)
custom = CustomLanguage.from_value(language, "alpha3")
if custom is None:
lang_obj = Language(language)
else:
lang_obj = custom.subzero_language()
if forced:
lang_obj = Language.rebuild(lang_obj, forced=True)
sub = Subtitle(
lang_obj,
mods=get_array_from(settings.general.subzero_mods)
)
sub.content = subtitle.read()
if not sub.is_valid():
logging.exception('BAZARR Invalid subtitle file: ' + subtitle.filename)
sub.mods = None
if settings.general.getboolean('utf8_encode'):
sub.set_encoding("utf-8")
saved_subtitles = []
try:
saved_subtitles = save_subtitles(path,
[sub],
single=single,
tags=None, # fixme
directory=get_target_folder(path),
chmod=chmod,
# formats=("srt", "vtt")
path_decoder=force_unicode)
except Exception:
logging.exception('BAZARR Error saving Subtitles file to disk for this file:' + path)
return
if len(saved_subtitles) < 1:
logging.exception('BAZARR Error saving Subtitles file to disk for this file:' + path)
return
subtitle_path = saved_subtitles[0].storage_path
if hi:
modifier_string = " HI"
elif forced:
modifier_string = " forced"
else:
modifier_string = ""
message = language_from_alpha3(language) + modifier_string + " Subtitles manually uploaded."
if hi:
modifier_code = ":hi"
elif forced:
modifier_code = ":forced"
else:
modifier_code = ""
uploaded_language_code3 = language + modifier_code
uploaded_language = language_from_alpha3(language) + modifier_string
uploaded_language_code2 = alpha2_from_alpha3(language) + modifier_code
audio_language_code2 = alpha2_from_language(audio_language)
audio_language_code3 = alpha3_from_language(audio_language)
if media_type == 'series':
episode_metadata = TableEpisodes.select(TableEpisodes.sonarrSeriesId, TableEpisodes.sonarrEpisodeId) \
.where(TableEpisodes.path == path_mappings.path_replace_reverse(path)) \
.dicts() \
.get_or_none()
if not episode_metadata:
return
series_id = episode_metadata['sonarrSeriesId']
episode_id = episode_metadata['sonarrEpisodeId']
sync_subtitles(video_path=path, srt_path=subtitle_path, srt_lang=uploaded_language_code2, media_type=media_type,
percent_score=100, sonarr_series_id=episode_metadata['sonarrSeriesId'], forced=forced,
sonarr_episode_id=episode_metadata['sonarrEpisodeId'])
else:
movie_metadata = TableMovies.select(TableMovies.radarrId) \
.where(TableMovies.path == path_mappings.path_replace_reverse_movie(path)) \
.dicts() \
.get_or_none()
if not movie_metadata:
return
series_id = ""
episode_id = movie_metadata['radarrId']
sync_subtitles(video_path=path, srt_path=subtitle_path, srt_lang=uploaded_language_code2, media_type=media_type,
percent_score=100, radarr_id=movie_metadata['radarrId'], forced=forced)
if use_postprocessing:
command = pp_replace(postprocessing_cmd, path, subtitle_path, uploaded_language,
uploaded_language_code2, uploaded_language_code3, audio_language,
audio_language_code2, audio_language_code3, forced, 100, "1", "manual", series_id,
episode_id, hi=hi)
postprocessing(command, path)
if media_type == 'series':
reversed_path = path_mappings.path_replace_reverse(path)
reversed_subtitles_path = path_mappings.path_replace_reverse(subtitle_path)
notify_sonarr(episode_metadata['sonarrSeriesId'])
event_stream(type='series', action='update', payload=episode_metadata['sonarrSeriesId'])
event_stream(type='episode-wanted', action='delete', payload=episode_metadata['sonarrEpisodeId'])
else:
reversed_path = path_mappings.path_replace_reverse_movie(path)
reversed_subtitles_path = path_mappings.path_replace_reverse_movie(subtitle_path)
notify_radarr(movie_metadata['radarrId'])
event_stream(type='movie', action='update', payload=movie_metadata['radarrId'])
event_stream(type='movie-wanted', action='delete', payload=movie_metadata['radarrId'])
return message, reversed_path, reversed_subtitles_path
def manual_search(path, language, hi, providers, providers_auth, sceneName, media_type):
logging.debug('BAZARR Manually searching subtitles for this file: ' + path)
subtitles_dict = {}
if hi == "True":
hi = True
else:
hi = False
language_set = set()
for lang in ast.literal_eval(language):
lang = alpha3_from_alpha2(lang)
if lang == 'pob':
language_set.add(Language('por', 'BR'))
else:
language_set.add(Language(lang))
use_scenename = settings.general.getboolean('use_scenename')
use_postprocessing = settings.general.getboolean('use_postprocessing')
postprocessing_cmd = settings.general.postprocessing_cmd
try:
if sceneName == "None" or use_scenename is False:
used_sceneName = False
video = scan_video(path)
else:
used_sceneName = True
video = Video.fromname(sceneName)
except:
logging.exception("BAZARR Error trying to get video information for this file: " + path)
else:
if media_type == "movie":
max_score = 120.0
elif media_type == "series":
max_score = 360.0
try:
with AsyncProviderPool(max_workers=None, providers=providers, provider_configs=providers_auth) as p:
subtitles = p.list_subtitles(video, language_set)
except Exception as e:
logging.exception("BAZARR Error trying to get subtitle list from provider for this file: " + path)
else:
subtitles_list = []
for s in subtitles:
{s: compute_score(s, video, hearing_impaired=hi)}
if media_type == "movie":
matched = set(s.get_matches(video))
if hi == s.hearing_impaired:
matched.add('hearing_impaired')
not_matched = set(score.movie_scores.keys()) - matched
required = set(['title'])
if any(elem in required for elem in not_matched):
continue
if used_sceneName:
not_matched.remove('hash')
if type(s) is LegendasTVSubtitle:
# The pickle doesn't work very well with RAR (rarfile.RarFile) or ZIP (zipfile.ZipFile)
s.archive.content = None
elif media_type == "series":
matched = set(s.get_matches(video))
if hi == s.hearing_impaired:
matched.add('hearing_impaired')
not_matched = set(score.episode_scores.keys()) - matched
required = set(['series', 'season', 'episode'])
if any(elem in required for elem in not_matched):
continue
if used_sceneName:
not_matched.remove('hash')
if type(s) is LegendasTVSubtitle:
# The pickle doesn't work very well with RAR (rarfile.RarFile) or ZIP (zipfile.ZipFile)
s.archive.content = None
subtitles_list.append(dict(score=round((compute_score(s, video, hearing_impaired=hi) / max_score * 100), 2), language=alpha2_from_alpha3(s.language.alpha3), hearing_impaired=str(s.hearing_impaired), provider=s.provider_name, subtitle=codecs.encode(pickle.dumps(s), "base64").decode(), url=s.page_link, matches=list(matched), dont_matches=list(not_matched)))
subtitles_dict = sorted(subtitles_list, key=lambda x: x['score'], reverse=True)
logging.debug('BAZARR ' + str(len(subtitles_dict)) + " subtitles have been found for this file: " + path)
logging.debug('BAZARR Ended searching subtitles for this file: ' + path)
return(subtitles_dict)
def upgrade_subtitles():
days_to_upgrade_subs = settings.general.days_to_upgrade_subs
minimum_timestamp = (
(datetime.now() - timedelta(days=int(days_to_upgrade_subs))) -
datetime(1970, 1, 1)).total_seconds()
if settings.general.getboolean('upgrade_manual'):
query_actions = [1, 2, 3]
else:
query_actions = [1, 3]
db = sqlite3.connect(os.path.join(args.config_dir, 'db', 'bazarr.db'),
timeout=30)
c = db.cursor()
episodes_list = c.execute(
"""SELECT table_history.video_path, table_history.language, table_history.score,
table_shows.hearing_impaired, table_episodes.scene_name, table_episodes.title,
table_episodes.sonarrSeriesId, table_episodes.sonarrEpisodeId,
MAX(table_history.timestamp), table_shows.languages
FROM table_history
INNER JOIN table_shows on table_shows.sonarrSeriesId = table_history.sonarrSeriesId
INNER JOIN table_episodes on table_episodes.sonarrEpisodeId = table_history.sonarrEpisodeId
WHERE action IN (""" +
','.join(map(str, query_actions)) + """) AND timestamp > ? AND
score is not null
GROUP BY table_history.video_path, table_history.language""",
(minimum_timestamp, )).fetchall()
movies_list = c.execute(
"""SELECT table_history_movie.video_path, table_history_movie.language,
table_history_movie.score, table_movies.hearing_impaired, table_movies.sceneName,
table_movies.title, table_movies.radarrId, MAX(table_history_movie.timestamp),
table_movies.languages
FROM table_history_movie
INNER JOIN table_movies on table_movies.radarrId = table_history_movie.radarrId
WHERE action IN (""" +
','.join(map(str, query_actions)) + """) AND timestamp > ? AND
score is not null
GROUP BY table_history_movie.video_path, table_history_movie.language""",
(minimum_timestamp, )).fetchall()
db.close()
episodes_to_upgrade = []
for episode in episodes_list:
if os.path.exists(path_replace(episode[0])) and int(episode[2]) < 360:
episodes_to_upgrade.append(episode)
movies_to_upgrade = []
for movie in movies_list:
if os.path.exists(path_replace_movie(
movie[0])) and int(movie[2]) < 120:
movies_to_upgrade.append(movie)
providers_list = get_providers()
providers_auth = get_providers_auth()
count_episode_to_upgrade = len(episodes_to_upgrade)
count_movie_to_upgrade = len(movies_to_upgrade)
for i, episode in enumerate(episodes_to_upgrade, 1):
if episode[1] in ast.literal_eval(str(episode[9])):
notifications.write(msg='Upgrading series subtitles : ' + str(i) +
'/' + str(count_episode_to_upgrade),
queue='get_subtitle',
duration='long')
result = download_subtitle(path_replace(episode[0]),
str(alpha3_from_alpha2(episode[1])),
episode[3],
providers_list,
providers_auth,
str(episode[4]),
episode[5],
'series',
forced_minimum_score=int(episode[2]),
is_upgrade=True)
if result is not None:
message = result[0]
path = result[1]
language_code = result[2]
provider = result[3]
score = result[4]
store_subtitles(path_replace(episode[0]))
history_log(3, episode[6], episode[7], message, path,
language_code, provider, score)
send_notifications(episode[6], episode[7], message)
for i, movie in enumerate(movies_to_upgrade, 1):
if movie[1] in ast.literal_eval(str(movie[8])):
notifications.write(msg='Upgrading movie subtitles : ' + str(i) +
'/' + str(count_movie_to_upgrade),
queue='get_subtitle',
duration='long')
result = download_subtitle(path_replace_movie(movie[0]),
str(alpha3_from_alpha2(movie[1])),
movie[3],
providers_list,
providers_auth,
str(movie[4]),
movie[5],
'movie',
forced_minimum_score=int(movie[2]),
is_upgrade=True)
if result is not None:
message = result[0]
path = result[1]
language_code = result[2]
provider = result[3]
score = result[4]
store_subtitles_movie(path_replace_movie(movie[0]))
history_log_movie(3, movie[6], message, path, language_code,
provider, score)
send_notifications_movie(movie[6], message)
def manual_search(path, language, hi, providers, providers_auth, sceneName,
title, media_type):
logging.debug('BAZARR Manually searching subtitles for this file: ' + path)
final_subtitles = []
if hi == "True":
hi = True
else:
hi = False
language_set = set()
for lang in ast.literal_eval(language):
lang = alpha3_from_alpha2(lang)
if lang == 'pob':
language_set.add(Language('por', 'BR'))
else:
language_set.add(Language(lang))
use_scenename = settings.general.getboolean('use_scenename')
minimum_score = settings.general.minimum_score
minimum_score_movie = settings.general.minimum_score_movie
use_postprocessing = settings.general.getboolean('use_postprocessing')
postprocessing_cmd = settings.general.postprocessing_cmd
video = get_video(path,
title,
sceneName,
use_scenename,
providers=providers,
media_type=media_type)
if video:
min_score, max_score, scores = get_scores(
video,
media_type,
min_score_movie_perc=int(minimum_score_movie),
min_score_series_perc=int(minimum_score))
try:
if providers:
subtitles = list_subtitles([video],
language_set,
providers=providers,
provider_configs=providers_auth,
pool_class=provider_pool(),
throttle_callback=provider_throttle,
language_hook=None) # fixme
else:
subtitles = []
logging.info("BAZARR All providers are throttled")
except Exception as e:
logging.exception(
"BAZARR Error trying to get subtitle list from provider for this file: "
+ path)
else:
subtitles_list = []
for s in subtitles[video]:
try:
matches = s.get_matches(video)
except AttributeError:
continue
# skip wrong season/episodes
if media_type == "series":
can_verify_series = True
if not s.hash_verifiable and "hash" in matches:
can_verify_series = False
if can_verify_series and not {
"series", "season", "episode"
}.issubset(matches):
logging.debug(
u"BAZARR Skipping %s, because it doesn't match our series/episode",
s)
continue
score = compute_score(matches, s, video, hearing_impaired=hi)
not_matched = scores - matches
s.score = score
# if score < min_score:
# continue
subtitles_list.append(
dict(score=round((score / max_score * 100), 2),
language=alpha2_from_alpha3(s.language.alpha3),
hearing_impaired=str(s.hearing_impaired),
provider=s.provider_name,
subtitle=codecs.encode(
pickle.dumps(s.make_picklable()),
"base64").decode(),
url=s.page_link,
matches=list(matches),
dont_matches=list(not_matched)))
final_subtitles = sorted(subtitles_list,
key=lambda x: x['score'],
reverse=True)
logging.debug('BAZARR ' + str(len(final_subtitles)) +
" subtitles have been found for this file: " + path)
logging.debug('BAZARR Ended searching subtitles for this file: ' +
path)
return final_subtitles
def manual_download_subtitle(path, language, hi, subtitle, provider, providers_auth, sceneName, media_type):
logging.debug('BAZARR Manually downloading subtitles for this file: ' + path)
if hi == "True":
hi = True
else:
hi = False
subtitle = pickle.loads(codecs.decode(subtitle.encode(), "base64"))
if media_type == 'series':
type_of_score = 360
elif media_type == 'movie':
type_of_score = 120
use_scenename = settings.general.getboolean('use_scenename')
use_postprocessing = settings.general.getboolean('use_postprocessing')
postprocessing_cmd = settings.general.postprocessing_cmd
language = alpha3_from_alpha2(language)
if language == 'pob':
lang_obj = Language('por', 'BR')
else:
lang_obj = Language(language)
try:
if sceneName is None or use_scenename is False:
used_sceneName = False
video = scan_video(path)
else:
used_sceneName = True
video = Video.fromname(sceneName)
except Exception as e:
logging.exception("BAZARR Error trying to get video information for this file: " + path)
pass
else:
try:
download_subtitles([subtitle], providers=provider, provider_configs=providers_auth)
logging.debug('BAZARR Subtitles file downloaded for this file:' + path)
except Exception as e:
logging.exception('BAZARR Error downloading subtitles for this file ' + path)
return None
else:
single = settings.general.getboolean('single_language')
try:
score = round(float(compute_score(subtitle, video, hearing_impaired=hi)) / type_of_score * 100, 2)
if used_sceneName == True:
video = scan_video(path)
if single is True:
result = save_subtitles(video, [subtitle], single=True, encoding='utf-8')
else:
result = save_subtitles(video, [subtitle], encoding='utf-8')
except Exception as e:
logging.exception('BAZARR Error saving subtitles file to disk for this file:' + path)
return None
else:
if len(result) > 0:
downloaded_provider = result[0].provider_name
downloaded_language = language_from_alpha3(result[0].language.alpha3)
downloaded_language_code2 = alpha2_from_alpha3(result[0].language.alpha3)
downloaded_language_code3 = result[0].language.alpha3
downloaded_path = get_subtitle_path(path, downloaded_language_code2)
logging.debug('BAZARR Subtitles file saved to disk: ' + downloaded_path)
message = downloaded_language + " subtitles downloaded from " + downloaded_provider + " with a score of " + unicode(score) + "% using manual search."
if use_postprocessing is True:
command = pp_replace(postprocessing_cmd, path, downloaded_path, downloaded_language, downloaded_language_code2, downloaded_language_code3)
try:
if os.name == 'nt':
codepage = subprocess.Popen("chcp", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# wait for the process to terminate
out_codepage, err_codepage = codepage.communicate()
encoding = out_codepage.split(':')[-1].strip()
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# wait for the process to terminate
out, err = process.communicate()
if os.name == 'nt':
out = out.decode(encoding)
except:
if out == "":
logging.error('BAZARR Post-processing result for file ' + path + ' : Nothing returned from command execution')
else:
logging.error('BAZARR Post-processing result for file ' + path + ' : ' + out)
else:
if out == "":
logging.info('BAZARR Post-processing result for file ' + path + ' : Nothing returned from command execution')
else:
logging.info('BAZARR Post-processing result for file ' + path + ' : ' + out)
return message
else:
logging.error("BAZARR Tried to manually download a subtitles for file: " + path + " but we weren't able to do (probably throttled by " + str(subtitle.provider_name) + ". Please retry later or select a subtitles from another provider.")
return None
logging.debug('BAZARR Ended manually downloading subtitles for file: ' + path)
def translate_subtitles_file(video_path, source_srt_file, to_lang, forced, hi):
language_code_convert_dict = {
'he': 'iw',
'zt': 'zh-cn',
'zh': 'zh-tw',
}
to_lang = alpha3_from_alpha2(to_lang)
lang_obj = CustomLanguage.from_value(to_lang, "alpha3")
if not lang_obj:
lang_obj = Language(to_lang)
if forced:
lang_obj = Language.rebuild(lang_obj, forced=True)
if hi:
lang_obj = Language.rebuild(lang_obj, hi=True)
logging.debug('BAZARR is translating in {0} this subtitles {1}'.format(
lang_obj, source_srt_file))
max_characters = 5000
dest_srt_file = get_subtitle_path(
video_path,
language=lang_obj
if isinstance(lang_obj, Language) else lang_obj.subzero_language(),
extension='.srt',
forced_tag=forced,
hi_tag=hi)
subs = pysubs2.load(source_srt_file, encoding='utf-8')
lines_list = [x.plaintext for x in subs]
joined_lines_str = '\n\n\n'.join(lines_list)
logging.debug(
'BAZARR splitting subtitles into {} characters blocks'.format(
max_characters))
lines_block_list = []
translated_lines_list = []
while len(joined_lines_str):
partial_lines_str = joined_lines_str[:max_characters]
if len(joined_lines_str) > max_characters:
new_partial_lines_str = partial_lines_str.rsplit('\n\n\n', 1)[0]
else:
new_partial_lines_str = partial_lines_str
lines_block_list.append(new_partial_lines_str)
joined_lines_str = joined_lines_str.replace(new_partial_lines_str, '')
logging.debug('BAZARR is sending {} blocks to Google Translate'.format(
len(lines_block_list)))
for block_str in lines_block_list:
empty_first_line = False
if block_str.startswith('\n\n\n'):
# This happens when the first line of text in a subtitles file is an empty string
empty_first_line = True
try:
translated_partial_srt_text = GoogleTranslator(
source='auto',
target=language_code_convert_dict.get(
lang_obj.alpha2,
lang_obj.alpha2)).translate(text=block_str)
except Exception:
logging.exception(
f'BAZARR Unable to translate subtitles {source_srt_file}')
return False
else:
if empty_first_line:
# GoogleTranslate remove new lines at the beginning of the string, so we add it back.
translated_partial_srt_text = '\n\n\n' + translated_partial_srt_text
translated_partial_srt_list = translated_partial_srt_text.split(
'\n\n\n')
translated_lines_list += translated_partial_srt_list
logging.debug(
'BAZARR saving translated subtitles to {}'.format(dest_srt_file))
for i, line in enumerate(subs):
try:
line.plaintext = translated_lines_list[i]
except IndexError:
logging.error(
f'BAZARR is unable to translate malformed subtitles: {source_srt_file}'
)
return False
subs.save(dest_srt_file)
return dest_srt_file
def manual_search(path, language, hi, providers, providers_auth, sceneName, media_type):
if hi == "True":
hi = True
else:
hi = False
language_set = set()
for lang in ast.literal_eval(language):
lang = alpha3_from_alpha2(lang)
if lang == 'pob':
language_set.add(Language('por', 'BR'))
else:
language_set.add(Language(lang))
use_scenename = get_general_settings()[9]
use_postprocessing = get_general_settings()[10]
postprocessing_cmd = get_general_settings()[11]
try:
if sceneName == "None" or use_scenename is False:
used_sceneName = False
video = scan_video(path)
else:
used_sceneName = True
video = Video.fromname(sceneName)
except:
logging.error("Error trying to get video information.")
else:
if media_type == "movie":
max_score = 120.0
elif media_type == "series":
max_score = 360.0
try:
with AsyncProviderPool(max_workers=None, providers=providers, provider_configs=providers_auth) as p:
subtitles = p.list_subtitles(video, language_set)
except Exception as e:
logging.exception("Error trying to get subtitle list from provider")
else:
subtitles_list = []
for s in subtitles:
{s: compute_score(s, video, hearing_impaired=hi)}
if media_type == "movie":
matched = set(s.get_matches(video))
if hi == s.hearing_impaired:
matched.add('hearing_impaired')
not_matched = set(score.movie_scores.keys()) - matched
required = set(['title'])
if any(elem in required for elem in not_matched):
continue
if used_sceneName:
not_matched.remove('hash')
elif media_type == "series":
matched = set(s.get_matches(video))
if hi == s.hearing_impaired:
matched.add('hearing_impaired')
not_matched = set(score.episode_scores.keys()) - matched
required = set(['series', 'season', 'episode'])
if any(elem in required for elem in not_matched):
continue
if used_sceneName:
not_matched.remove('hash')
subtitles_list.append(dict(score=round((compute_score(s, video, hearing_impaired=hi) / max_score * 100), 2), language=alpha2_from_alpha3(s.language.alpha3), hearing_impaired=str(s.hearing_impaired), provider=s.provider_name, subtitle=codecs.encode(pickle.dumps(s), "base64").decode(), url=s.page_link, matches=list(matched), dont_matches=list(not_matched)))
subtitles_dict = {}
subtitles_dict = sorted(subtitles_list, key=lambda x: x['score'], reverse=True)
return(subtitles_dict)
def manual_download_subtitle(path, language, hi, subtitle, provider, providers_auth, sceneName, media_type):
if hi == "True":
hi = True
else:
hi = False
subtitle = pickle.loads(codecs.decode(subtitle.encode(), "base64"))
if media_type == 'series':
type_of_score = 360
elif media_type == 'movie':
type_of_score = 120
use_scenename = get_general_settings()[9]
use_postprocessing = get_general_settings()[10]
postprocessing_cmd = get_general_settings()[11]
language = alpha3_from_alpha2(language)
if language == 'pob':
lang_obj = Language('por', 'BR')
else:
lang_obj = Language(language)
try:
if sceneName is None or use_scenename is False:
used_sceneName = False
video = scan_video(path)
else:
used_sceneName = True
video = Video.fromname(sceneName)
except Exception as e:
logging.exception('Error trying to extract information from this filename: ' + path)
return None
else:
try:
best_subtitle = subtitle
download_subtitles([best_subtitle], providers=provider, provider_configs=providers_auth)
except Exception as e:
logging.exception('Error downloading subtitles for ' + path)
return None
else:
single = get_general_settings()[7]
try:
score = round(float(compute_score(best_subtitle, video, hearing_impaired=hi)) / type_of_score * 100, 2)
if used_sceneName == True:
video = scan_video(path)
if single is True:
result = save_subtitles(video, [best_subtitle], single=True, encoding='utf-8')
else:
result = save_subtitles(video, [best_subtitle], encoding='utf-8')
except Exception as e:
logging.exception('Error saving subtitles file to disk.')
return None
else:
if len(result) > 0:
downloaded_provider = result[0].provider_name
downloaded_language = language_from_alpha3(result[0].language.alpha3)
downloaded_language_code2 = alpha2_from_alpha3(result[0].language.alpha3)
downloaded_language_code3 = result[0].language.alpha3
downloaded_path = get_subtitle_path(path, language=lang_obj)
message = downloaded_language + " subtitles downloaded from " + downloaded_provider + " with a score of " + unicode(score) + "% using manual search."
if use_postprocessing is True:
command = pp_replace(postprocessing_cmd, path, downloaded_path, downloaded_language, downloaded_language_code2, downloaded_language_code3)
try:
if os.name == 'nt':
codepage = subprocess.Popen("chcp", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# wait for the process to terminate
out_codepage, err_codepage = codepage.communicate()
encoding = out_codepage.split(':')[-1].strip()
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# wait for the process to terminate
out, err = process.communicate()
if os.name == 'nt':
out = out.decode(encoding)
except:
if out == "":
logging.error('Post-processing result for file ' + path + ' : Nothing returned from command execution')
else:
logging.error('Post-processing result for file ' + path + ' : ' + out)
else:
if out == "":
logging.info('Post-processing result for file ' + path + ' : Nothing returned from command execution')
else:
logging.info('Post-processing result for file ' + path + ' : ' + out)
return message
else:
return None
def postprocessMovie(item):
postprocess(item)
# Parse audio language
if 'audio_language' in item and item['audio_language'] is not None:
item['audio_language'] = get_audio_profile_languages(movie_id=item['radarrId'])
# Parse alternate titles
if 'alternativeTitles' in item:
if item['alternativeTitles'] is None:
item['alternativeTitles'] = []
else:
item['alternativeTitles'] = ast.literal_eval(item['alternativeTitles'])
# Parse failed attempts
if 'failedAttempts' in item:
if item['failedAttempts']:
item['failedAttempts'] = ast.literal_eval(item['failedAttempts'])
# Parse subtitles
if 'subtitles' in item:
if item['subtitles'] is None:
item['subtitles'] = []
else:
item['subtitles'] = ast.literal_eval(item['subtitles'])
for i, subs in enumerate(item['subtitles']):
language = subs[0].split(':')
item['subtitles'][i] = {"path": path_mappings.path_replace_movie(subs[1]),
"name": language_from_alpha2(language[0]),
"code2": language[0],
"code3": alpha3_from_alpha2(language[0]),
"forced": False,
"hi": False}
if len(language) > 1:
item['subtitles'][i].update({
"forced": True if language[1] == 'forced' else False,
"hi": True if language[1] == 'hi' else False
})
if settings.general.getboolean('embedded_subs_show_desired'):
desired_lang_list = get_desired_languages(item['profileId'])
item['subtitles'] = [x for x in item['subtitles'] if x['code2'] in desired_lang_list or x['path']]
item['subtitles'] = sorted(item['subtitles'], key=itemgetter('name', 'forced'))
# Parse missing subtitles
if 'missing_subtitles' in item:
if item['missing_subtitles'] is None:
item['missing_subtitles'] = []
else:
item['missing_subtitles'] = ast.literal_eval(item['missing_subtitles'])
for i, subs in enumerate(item['missing_subtitles']):
language = subs.split(':')
item['missing_subtitles'][i] = {"name": language_from_alpha2(language[0]),
"code2": language[0],
"code3": alpha3_from_alpha2(language[0]),
"forced": False,
"hi": False}
if len(language) > 1:
item['missing_subtitles'][i].update({
"forced": True if language[1] == 'forced' else False,
"hi": True if language[1] == 'hi' else False
})
# Provide mapped path
if 'path' in item:
if item['path']:
item['path'] = path_mappings.path_replace_movie(item['path'])
if 'subtitles_path' in item:
# Provide mapped subtitles path
item['subtitles_path'] = path_mappings.path_replace_movie(item['subtitles_path'])
# map poster and fanart to server proxy
if 'poster' in item:
poster = item['poster']
item['poster'] = f"{base_url}/images/movies{poster}" if poster else None
if 'fanart' in item:
fanart = item['fanart']
item['fanart'] = f"{base_url}/images/movies{fanart}" if fanart else None
