def test_converter_opensubtitles(self):
self.assertEqual(Language('fra').opensubtitles, Language('fra').alpha3b)
self.assertEqual(Language('por', 'BR').opensubtitles, 'pob')
self.assertEqual(Language.fromopensubtitles('fre'), Language('fra'))
self.assertEqual(Language.fromopensubtitles('pob'), Language('por', 'BR'))
self.assertEqual(Language.fromopensubtitles('pb'), Language('por', 'BR'))
# Montenegrin is not recognized as an ISO language (yet?) but for now it is
# unofficially accepted as Serbian from Montenegro
self.assertEqual(Language.fromopensubtitles('mne'), Language('srp', 'ME'))
self.assertEqual(Language.fromcode('pob', 'opensubtitles'), Language('por', 'BR'))
with self.assertRaises(LanguageReverseError):
Language.fromopensubtitles('zzz')
with self.assertRaises(LanguageConvertError):
Language('aaa').opensubtitles
self.assertEqual(len(get_language_converter('opensubtitles').codes), 606)
# test with all the languages from the opensubtitles api
# downloaded from: http://www.opensubtitles.org/addons/export_languages.php
f = resource_stream('babelfish', 'data/opensubtitles_languages.txt')
f.readline()
for l in f:
idlang, alpha2, _, upload_enabled, web_enabled = l.decode('utf-8').strip().split('\t')
if not int(upload_enabled) and not int(web_enabled):
# do not test languages that are too esoteric / not widely available
continue
self.assertEqual(Language.fromopensubtitles(idlang).opensubtitles, idlang)
if alpha2:
self.assertEqual(Language.fromopensubtitles(idlang), Language.fromopensubtitles(alpha2))
f.close()
def test_check_for_better_checks_languages_with_no_current_subs(self):
superliminal.env.settings.languages.append(Language.fromietf('pt-BR'))
okensub = {
'id': 'okensub',
'language': Language.fromietf('en'),
'series': "Series Title",
'season': 2,
'episode': 3,
'title': "The Episode",
'release_group': "OtherRG",
'content': SUBTITLE_CONTENT
}
self.set_subtitles([okensub])
self.add_video(name="Series.Title.S02E03.720p.WEB-DL.H264-TvRG.mkv")
yield self.wait_until_processed()
betterensub = self.transform_sub(okensub, 'betterensub',
release_group="TvRG", content=SUBTITLE_CONTENT_2)
okbrsub = self.transform_sub(okensub, 'okbrsub',
language=Language.fromietf('pt-BR'), content=SUBTITLE_CONTENT_3)
self.set_subtitles([okensub, betterensub, okbrsub])
SuperliminalCore.check_for_better()
yield self.wait_until_processed()
self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT_2, suffix='.en.srt')
self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT_3, suffix='.pt-BR.srt')
def test_register_converter(self):
class TestConverter(LanguageReverseConverter):
def __init__(self):
self.to_test = {'fra': 'test1', 'eng': 'test2'}
self.from_test = {'test1': 'fra', 'test2': 'eng'}
def convert(self, alpha3, country=None, script=None):
if alpha3 not in self.to_test:
raise LanguageConvertError(alpha3, country, script)
return self.to_test[alpha3]
def reverse(self, test):
if test not in self.from_test:
raise LanguageReverseError(test)
return (self.from_test[test], None)
language = Language('fra')
self.assertFalse(hasattr(language, 'test'))
register_language_converter('test', TestConverter)
self.assertTrue(hasattr(language, 'test'))
self.assertIn('test', LANGUAGE_CONVERTERS)
self.assertEqual(Language('fra').test, 'test1')
self.assertEqual(Language.fromtest('test2').alpha3, 'eng')
unregister_language_converter('test')
self.assertNotIn('test', LANGUAGE_CONVERTERS)
with self.assertRaises(KeyError):
Language.fromtest('test1')
with self.assertRaises(AttributeError):
Language('fra').test
clear_language_converters()
load_language_converters()
def test_check_for_better_checks_all_recent_videos(self):
oktvsub = {
'id': 'oktvsub',
'language': Language.fromietf('en'),
'series': "Series Title",
'season': 2,
'episode': 3,
'title': "The Episode",
'release_group': "OtherRG",
'content': SUBTITLE_CONTENT
}
yield self.add_video(name="Series.Title.S02E03.720p.WEB-DL.H264-TvRG.mkv", subtitle=oktvsub)
movie_filename = self.create_video_file()
okmoviesub = {
'id': 'okmoviesub',
'language': Language.fromietf('en'),
'title': "Movie Title",
'year': 2016,
'release_group': "OtherRG",
'content': SUBTITLE_CONTENT
}
yield self.add_video(path=movie_filename,
name="Movie.Title.2016.720p.WEB-DL.H264-MovieRG.mkv", subtitle=okmoviesub)
bettertvsub = self.transform_sub(oktvsub, 'bettertvsub',
release_group="TvRG", content=SUBTITLE_CONTENT_2)
bettermoviesub = self.transform_sub(okmoviesub, 'bettermoviesub',
release_group="MovieRG", content=SUBTITLE_CONTENT_3)
self.set_subtitles([oktvsub, bettertvsub, okmoviesub, bettermoviesub])
SuperliminalCore.check_for_better()
yield self.wait_until_processed()
self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT_2)
self.assert_subtitle_contents_matches(video_filename=movie_filename,
expected_content=SUBTITLE_CONTENT_3)
def test_downloads_for_all_configured_languages(self):
superliminal.env.settings.languages.append(Language.fromietf('pt-BR'))
self.set_subtitles([{
'id': 'english_sub',
'language': Language.fromietf('en'),
'series': "Series Title",
'season': 2,
'episode': 3,
'title': "The Episode",
'release_group': "TvRG",
'resolution': '720p',
'content': SUBTITLE_CONTENT
},
{
'id': 'brazilian_sub',
'language': Language.fromietf('pt-BR'),
'series': "Series Title",
'season': 2,
'episode': 3,
'title': "The Episode",
'release_group': "TvRG",
'resolution': '720p',
'content': SUBTITLE_CONTENT_2
}])
request = self.get_add_request(name="Series.Title.S02E03.720p.WEB-DL.H264-TvRG.mkv")
response = yield self.http_client.fetch(request)
self.assertEqual(200, response.code)
yield self.wait_until_processed()
self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT, suffix='.en.srt')
self.assert_subtitle_contents_matches(expected_content=SUBTITLE_CONTENT_2, suffix='.pt-BR.srt')
def test_downloads_new_sub_if_new_video_added_for_existing_path(self):
self.set_subtitles([{
'id': 'new_sub',
'language': Language.fromietf('en'),
'series': "Series Title",
'season': 2,
'episode': 3,
'title': "The Episode",
'release_group': "TvRG",
'resolution': '720p',
'content': SUBTITLE_CONTENT
},
{
'id': 'old_sub',
'language': Language.fromietf('en'),
'series': "Series Title",
'season': 2,
'episode': 3,
'title': "The Episode",
'release_group': "OtherRG",
'resolution': '720p',
'content': SUBTITLE_CONTENT_2
}])
request = self.get_add_request(name="Series.Title.S02E03.720p.WEB-DL.H264-OtherRG.mkv")
response = yield self.http_client.fetch(request)
self.assertEqual(200, response.code)
yield self.wait_until_processed()
request = self.get_add_request(name="Series.Title.S02E03.720p.WEB-DL.H264-TvRG.mkv")
response = yield self.http_client.fetch(request)
self.assertEqual(200, response.code)
yield self.wait_until_processed()
self.assert_subtitle_contents_matches()
def test_converter_alpha3t(self):
self.assertEqual(Language('fra').alpha3t, 'fra')
self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
self.assertRaises(LanguageReverseError, lambda: Language.fromalpha3t('zzz'))
self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha3t)
self.assertEqual(len(language_converters['alpha3t'].codes), 418)
def test_converter_alpha2(self):
self.assertEqual(Language('eng').alpha2, 'en')
self.assertEqual(Language.fromalpha2('en'), Language('eng'))
self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
self.assertRaises(LanguageReverseError, lambda: Language.fromalpha2('zz'))
self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha2)
self.assertEqual(len(language_converters['alpha2'].codes), 184)
def subtitlesLanguages(video_path):
"""Return a list detected subtitles for the given video file"""
resultList = []
embedded_subtitle_languages = set()
# Serch for embedded subtitles
if not sickbeard.EMBEDDED_SUBTITLES_ALL:
if video_path.endswith('mkv'):
try:
with open(video_path.encode(sickbeard.SYS_ENCODING), 'rb') as f:
mkv = MKV(f)
if mkv.subtitle_tracks:
for st in mkv.subtitle_tracks:
if st.language:
try:
embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
except BabelfishError:
logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
embedded_subtitle_languages.add(Language('und'))
elif st.name:
try:
embedded_subtitle_languages.add(Language.fromname(st.name))
except BabelfishError:
logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
embedded_subtitle_languages.add(Language('und'))
else:
embedded_subtitle_languages.add(Language('und'))
else:
logger.log('MKV has no subtitle track', logger.DEBUG)
except MalformedMKVError:
logger.log('MKV seems to be malformed, ignoring embedded subtitles', logger.WARNING)
# Search subtitles in the absolute path
if sickbeard.SUBTITLES_DIR and ek(os.path.exists, sickbeard.SUBTITLES_DIR):
video_path = ek(os.path.join, sickbeard.SUBTITLES_DIR, ek(os.path.basename, video_path))
# Search subtitles in the relative path
elif sickbeard.SUBTITLES_DIR:
video_path = ek(os.path.join, ek(os.path.dirname, video_path), sickbeard.SUBTITLES_DIR, ek(os.path.basename, video_path))
external_subtitle_languages = subliminal.video.scan_subtitle_languages(video_path)
subtitle_languages = external_subtitle_languages.union(embedded_subtitle_languages)
if (len(subtitle_languages) is 1 and len(wantedLanguages()) is 1) and Language('und') in subtitle_languages:
subtitle_languages.remove(Language('und'))
subtitle_languages.add(fromietf(wantedLanguages()[0]))
for language in subtitle_languages:
if hasattr(language, 'opensubtitles') and language.opensubtitles:
resultList.append(language.opensubtitles)
elif hasattr(language, 'alpha3') and language.alpha3:
resultList.append(language.alpha3)
elif hasattr(language, 'alpha2') and language.alpha2:
resultList.append(language.alpha2)
defaultLang = wantedLanguages()
if ('pob' in defaultLang or 'pb' in defaultLang) and ('pt' not in defaultLang and 'por' not in defaultLang):
resultList = [x if not x in ['por', 'pt'] else u'pob' for x in resultList]
return sorted(resultList)
def test_register_converter(self):
class TestConverter(LanguageReverseConverter):
def __init__(self):
self.to_test = {'fra': 'test1', 'eng': 'test2'}
self.from_test = {'test1': 'fra', 'test2': 'eng'}
def convert(self, alpha3, country=None, script=None):
if alpha3 not in self.to_test:
raise LanguageConvertError(alpha3, country, script)
return self.to_test[alpha3]
def reverse(self, test):
if test not in self.from_test:
raise LanguageReverseError(test)
return (self.from_test[test], None)
language = Language('fra')
self.assertFalse(hasattr(language, 'test'))
language_converters['test'] = TestConverter()
self.assertTrue(hasattr(language, 'test'))
self.assertIn('test', language_converters)
self.assertEqual(Language('fra').test, 'test1')
self.assertEqual(Language.fromtest('test2').alpha3, 'eng')
del language_converters['test']
self.assertNotIn('test', language_converters)
self.assertRaises(KeyError, lambda: Language.fromtest('test1'))
self.assertRaises(AttributeError, lambda: Language('fra').test)
def check_missing_subtitle_languages(dirname, filename):
log.debug("Checking for missing subtitle")
missing_subtitles = []
# Check embedded subtitles
embedded_subtitles = []
if autosubliminal.SCANEMBEDDEDSUBS:
embedded_subtitles = _get_embedded_subtitles(dirname, filename)
# Check default language
if autosubliminal.DEFAULTLANGUAGE:
default_language = Language.fromietf(autosubliminal.DEFAULTLANGUAGE)
# Check with or without alpha2 code suffix depending on configuration
if autosubliminal.DEFAULTLANGUAGESUFFIX:
srtfile = os.path.splitext(filename)[0] + u"." + autosubliminal.DEFAULTLANGUAGE + u".srt"
else:
srtfile = os.path.splitext(filename)[0] + u".srt"
if not os.path.exists(os.path.join(dirname, srtfile)) and default_language not in embedded_subtitles:
missing_subtitles.append(autosubliminal.DEFAULTLANGUAGE)
# Check additional languages
if autosubliminal.ADDITIONALLANGUAGES:
# Always check with alpha2 code suffix for additional languages
for language in autosubliminal.ADDITIONALLANGUAGES:
additional_language = Language.fromietf(language)
srtfile = os.path.splitext(filename)[0] + u"." + language + u".srt"
if not os.path.exists(os.path.join(dirname, srtfile)) and additional_language not in embedded_subtitles:
missing_subtitles.append(language)
return missing_subtitles
def test_added_subs_are_returned_by_lang(self):
path = "/data/Series/Season 1/01 Title.mkv"
video = Video.fromname("Series.S01E02.Title.720p.WEB-DL.DD5.1.H264-ReleaseGroup.mkv")
self.datastore.add_video(path, video)
provider1 = "davessubs"
sub_id1 = "ABC123"
lang1 = Language.fromietf("en")
score1 = 123
provider2 = "stevesubs"
sub_id2 = "steve123"
lang2 = lang1
score2 = 120
provider3 = "pablosubs"
sub_id3 = "umdoistres"
lang3 = Language.fromietf("pt-BR")
score3 = 150
self.datastore.add_download(path, provider1, sub_id1, lang1, score1)
self.datastore.add_download(path, provider2, sub_id2, lang2, score2)
self.datastore.add_download(path, provider3, sub_id3, lang3, score3)
downloads = self.datastore.get_downloads_for_video(path)
self.assertEqual(
downloads,
{
lang1: [
{"provider": provider1, "sub_id": sub_id1, "lang": lang1, "score": score1},
{"provider": provider2, "sub_id": sub_id2, "lang": lang2, "score": score2},
],
lang3: [{"provider": provider3, "sub_id": sub_id3, "lang": lang3, "score": score3}],
},
)
def getEmbeddedLanguages(video_path):
embedded_subtitle_languages = set()
try:
with ek(io.open, video_path, 'rb') as f:
mkv = MKV(f)
if mkv.subtitle_tracks:
for st in mkv.subtitle_tracks:
if st.language:
try:
embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
except BabelfishError:
logging.debug('Embedded subtitle track is not a valid language')
embedded_subtitle_languages.add(Language('und'))
elif st.name:
try:
embedded_subtitle_languages.add(Language.fromname(st.name))
except BabelfishError:
logging.debug('Embedded subtitle track is not a valid language')
embedded_subtitle_languages.add(Language('und'))
else:
embedded_subtitle_languages.add(Language('und'))
else:
logging.debug('MKV has no subtitle track')
except MalformedMKVError:
logging.info('MKV seems to be malformed ( %s ), ignoring embedded subtitles' % video_path)
return embedded_subtitle_languages
def getAlpha3TCode(code): # We need to make sure that language codes are alpha3T
"""
:param code: Alpha2, Alpha3, or Alpha3b code.
:type code: C{str}
:return: Alpha3t language code (ISO 639-2/T) as C{str}
"""
lang = 'und'
code = code.strip().lower()
if len(code) == 3:
try:
lang = Language(code).alpha3t
except:
try:
lang = Language.fromalpha3b(code).alpha3t
except:
try:
lang = Language.fromalpha3t(code).alpha3t
except:
pass
elif len(code) == 2:
lang = Language.fromalpha2(code).alpha3t
return lang
def test_converter_name(self):
self.assertEqual(Language('eng').name, 'English')
self.assertEqual(Language.fromname('English'), Language('eng'))
self.assertEqual(Language.fromcode('English', 'name'), Language('eng'))
with self.assertRaises(LanguageReverseError):
Language.fromname('Zzzzzzzzz')
self.assertEqual(len(get_language_converter('name').codes), 7874)
def getEmbeddedLanguages(video_path):
embedded_subtitle_languages = set()
try:
with open(video_path, 'rb') as f:
mkv = MKV(f)
if mkv.subtitle_tracks:
for st in mkv.subtitle_tracks:
if st.language:
try:
embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
except BabelfishError:
logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
embedded_subtitle_languages.add(Language('und'))
elif st.name:
try:
embedded_subtitle_languages.add(Language.fromname(st.name))
except BabelfishError:
logger.log('Embedded subtitle track is not a valid language', logger.DEBUG)
embedded_subtitle_languages.add(Language('und'))
else:
embedded_subtitle_languages.add(Language('und'))
else:
logger.log('MKV has no subtitle track', logger.DEBUG)
except MalformedMKVError:
logger.log('MKV seems to be malformed, ignoring embedded subtitles', logger.WARNING)
return embedded_subtitle_languages
def _get_embedded_subtitles(dirname, filename):
"""
Based on subliminal.video.scan_video(...) but only keep the check for embedded subtitles
"""
log.debug("Checking for embedded subtitle")
embedded_subtitle_languages = set()
try:
path = os.path.join(dirname, filename)
if filename.endswith('.mkv'):
with open(path, 'rb') as f:
mkv = MKV(f)
# subtitle tracks
if mkv.subtitle_tracks:
for st in mkv.subtitle_tracks:
if st.language:
try:
embedded_subtitle_languages.add(Language.fromalpha3b(st.language))
except BabelfishError:
log.error('Embedded subtitle track language %r is not a valid language', st.language)
embedded_subtitle_languages.add(Language('und'))
elif st.name:
try:
embedded_subtitle_languages.add(Language.fromname(st.name))
except BabelfishError:
log.debug('Embedded subtitle track name %r is not a valid language', st.name)
embedded_subtitle_languages.add(Language('und'))
else:
embedded_subtitle_languages.add(Language('und'))
log.debug('Found embedded subtitles %r with enzyme', embedded_subtitle_languages)
else:
log.debug('MKV has no subtitle track')
except:
log.exception('Parsing video metadata with enzyme failed')
return embedded_subtitle_languages
def test_converter_alpha3t(self):
self.assertEqual(Language('fra').alpha3t, 'fra')
self.assertEqual(Language.fromalpha3t('fra'), Language('fra'))
self.assertEqual(Language.fromcode('fra', 'alpha3t'), Language('fra'))
with self.assertRaises(LanguageReverseError):
Language.fromalpha3t('zzz')
with self.assertRaises(LanguageConvertError):
Language('aaa').alpha3t
self.assertEqual(len(get_language_converter('alpha3t').codes), 418)
def query(self, searchword=None):
params = {"searchword": searchword}
logger.info("Searching subtitles %r", params)
r = self.session.get("http://sub.makedie.me/sub/?", params=params, timeout=10)
r.raise_for_status()
# loop over
subtitles = []
try:
# BeautifulSoup gives you Unicode, damn it
soup = BeautifulSoup(r.content, "html5lib")
results = soup.find_all("div", attrs={"class": "subitem"})
for it in results:
releases = it.find("a", attrs={"class": "introtitle"})["title"].strip().split("/")
sid = re.search("/xml/sub/\d+/(\d+).xml", it.find("a", attrs={"class": "introtitle"})["href"]).group(1)
download_button = it.find("a", attrs={"id": "downsubbtn"})["onclick"].strip()
if re.search("location.href\s*=\s*", download_button):
link = "http://sub.makedie.me/" + re.search(
"location.href\s*=\s*['\"](.*?)['\"]", download_button
).group(1)
else:
logger.debug("No download link found")
continue
lang = ""
for li in it.find_all("li"):
if re.search("格式：", li.text.strip()):
subtype = re.search("格式：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
if re.search("语言：", li.text.strip()):
lang = re.search("语言：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
if re.search("下载次数：", li.text.strip()):
downloads = re.search("下载次数：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
downloads = re.search("(\d+).*", downloads).group(1)
if re.search("日期：", li.text.strip()):
upload_date = re.search("日期：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
if "简" in lang or "繁" in lang or "双语" in lang:
subtitle = MakeDieSubtitle(
Language.fromalpha2("zh"), releases, sid, link, subtype, downloads, upload_date
)
else:
subtitle = MakeDieSubtitle(
Language.fromalpha2("en"), releases, sid, link, subtype, downloads, upload_date
)
logger.debug("Found subtitle %r", subtitle)
subtitles.append(subtitle)
return subtitles
except:
logger.debug("No subtitle found")
return []
def test_converter_alpha2(self):
self.assertEqual(Language('eng').alpha2, 'en')
self.assertEqual(Language.fromalpha2('en'), Language('eng'))
self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
with self.assertRaises(LanguageReverseError):
Language.fromalpha2('zz')
with self.assertRaises(LanguageConvertError):
Language('aaa').alpha2
self.assertEqual(len(get_language_converter('alpha2').codes), 184)
def scan_subtitle_languages(path):
language_extensions = tuple('.' + c for c in language_converters['opensubtitles'].codes)
dirpath, filename = os.path.split(path)
subtitles = set()
for p in os.listdir(dirpath):
if not isinstance(p, bytes) and p.startswith(os.path.splitext(filename)[0]) and p.endswith(subliminal.video.SUBTITLE_EXTENSIONS):
if os.path.splitext(p)[0].endswith(language_extensions) and len(os.path.splitext(p)[0].rsplit('.', 1)[1]) is 2:
subtitles.add(Language.fromopensubtitles(os.path.splitext(p)[0][-2:]))
elif os.path.splitext(p)[0].endswith(language_extensions) and len(os.path.splitext(p)[0].rsplit('.', 1)[1]) is 3:
subtitles.add(Language.fromopensubtitles(os.path.splitext(p)[0][-3:]))
else:
subtitles.add(Language('und'))
return subtitles
def fix_subtitles_codes(self):
sqlResults = self.connection.select(
"SELECT subtitles, episode_id FROM tv_episodes WHERE subtitles != '' AND subtitles_lastsearch < ?;",
[datetime.datetime(2015, 7, 15, 17, 20, 44, 326380).strftime(dateTimeFormat)])
validLanguages = [Language.fromopensubtitles(language).opensubtitles for language in language_converters['opensubtitles'].codes if len(language) == 3]
if not sqlResults:
return
for sqlResult in sqlResults:
langs = []
logger.log("Checking subtitle codes for episode_id: %s, codes: %s" %
(sqlResult['episode_id'], sqlResult['subtitles']), logger.DEBUG)
for subcode in sqlResult['subtitles'].split(','):
if not len(subcode) is 3 or not subcode in validLanguages:
logger.log("Fixing subtitle codes for episode_id: %s, invalid code: %s" %
(sqlResult['episode_id'], subcode), logger.DEBUG)
continue
langs.append(subcode)
self.connection.action("UPDATE tv_episodes SET subtitles = ?, subtitles_lastsearch = ? WHERE episode_id = ?;",
[','.join(langs), datetime.datetime.now().strftime(dateTimeFormat), sqlResult['episode_id']])
def find_possible_languages(string):
"""Find possible languages in the string
:return: list of tuple (property, Language, lang_word, word)
"""
words = find_words(string)
valid_words = []
for word in words:
lang_word = word.lower()
key = 'language'
for prefix in subtitle_prefixes:
if lang_word.startswith(prefix):
lang_word = lang_word[len(prefix):]
key = 'subtitleLanguage'
for suffix in subtitle_suffixes:
if lang_word.endswith(suffix):
lang_word = lang_word[:len(suffix)]
key = 'subtitleLanguage'
for prefix in lang_prefixes:
if lang_word.startswith(prefix):
lang_word = lang_word[len(prefix):]
if not lang_word in LNG_COMMON_WORDS:
try:
lang = Language.fromguessit(lang_word)
# Keep language with alpha2 equivalent. Others are probably
# uncommon languages.
if lang == 'mul' or hasattr(lang, 'alpha2'):
valid_words.append((key, lang, lang_word, word))
except babelfish.Error:
pass
return valid_words
def query(self, series, season, episode, year=None):
# search the show id
show_id = self.search_show_id(series, year)
if show_id is None:
logger.error("No show id found for %r (%r)", series, {"year": year})
return []
# get the episode ids
episode_ids = self.get_episode_ids(show_id, season)
if episode not in episode_ids:
logger.error("Episode %d not found", episode)
return []
# get the episode page
logger.info("Getting the page for episode %d", episode_ids[episode])
r = self.session.get(self.server_url + "episode-%d.html" % episode_ids[episode], timeout=10)
soup = ParserBeautifulSoup(r.content, ["lxml", "html.parser"])
# loop over subtitles rows
subtitles = []
for row in soup.select(".subtitlen"):
# read the item
language = Language.fromtvsubtitles(row.h5.img["src"][13:-4])
subtitle_id = int(row.parent["href"][10:-5])
page_link = self.server_url + "subtitle-%d.html" % subtitle_id
rip = row.find("p", title="rip").text.strip() or None
release = row.find("p", title="release").text.strip() or None
subtitle = TVsubtitlesSubtitle(
language, page_link, subtitle_id, series, season, episode, year, rip, release
)
logger.debug("Found subtitle %s", subtitle)
subtitles.append(subtitle)
return subtitles
def _detect_subtitle_language(srt_path):
log.debug('Detecting subtitle language')
# Load srt file (try first iso-8859-1 with fallback to utf-8)
try:
subtitle = pysrt.open(path=srt_path, encoding='iso-8859-1')
except Exception:
try:
subtitle = pysrt.open(path=srt_path, encoding='utf-8')
except Exception:
# If we can't read it, we can't detect, so return
return None
# Read first 5 subtitle lines to determine the language
if len(subtitle) >= 5:
text = ''
for sub in subtitle[0:5]:
text += sub.text
# Detect the language with highest probability and return it if it's more than the required minimum probability
detected_languages = langdetect.detect_langs(text)
log.debug('Detected subtitle language(s): %s', detected_languages)
if len(detected_languages) > 0:
# Get first detected language (list is sorted according to probability, highest first)
detected_language = detected_languages[0]
language_probability = detected_language.prob
if language_probability >= autosubliminal.DETECTEDLANGUAGEPROBABILITY:
log.debug('Probability of detected subtitle language accepted: %s', detected_language)
return Language.fromietf(detected_language.lang)
else:
log.debug('Probability of detected subtitle language too low: %s', detected_language)
return None
def query(self, series, season, episode, year=None):
# search the show id
show_id = self.search_show_id(series, year)
if show_id is None:
logger.error('No show id found for %r (%r)', series, {'year': year})
return []
# get the episode ids
episode_ids = self.get_episode_ids(show_id, season)
if episode not in episode_ids:
logger.error('Episode %d not found', episode)
return []
# get the episode page
logger.info('Getting the page for episode %d', episode_ids[episode])
r = self.session.get(self.server_url + 'episode-%d.html' % episode_ids[episode], timeout=10)
soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
# loop over subtitles rows
subtitles = []
for row in soup.select('.subtitlen'):
# read the item
language = Language.fromtvsubtitles(row.h5.img['src'][13:-4])
subtitle_id = int(row.parent['href'][10:-5])
page_link = self.server_url + 'subtitle-%d.html' % subtitle_id
rip = row.find('p', title='rip').text.strip() or None
release = row.find('p', title='release').text.strip() or None
subtitle = TVsubtitlesSubtitle(language, page_link, subtitle_id, series, season, episode, year, rip,
release)
logger.debug('Found subtitle %s', subtitle)
subtitles.append(subtitle)
return subtitles
def from_english(self, language):
if language in EXCEPTIONS:
return Language.fromietf(EXCEPTIONS[language])
language_match = re.search('(\w[\w\s]*\w)', language)
if not language_match:
return
language_english = language_match.group(1)
language_alpha3 = xbmc.convertLanguage(language_english, format=xbmc.ISO_639_2)
if not language_alpha3:
return
result = Language.fromcode(language_alpha3, 'alpha3b')
result.country = self.get_country(language)
return result
def search_external_subtitles(path):
"""Search for external subtitles from a video `path` and their associated language.
:param str path: path to the video.
:return: found subtitles with their languages.
:rtype: dict
"""
dirpath, filename = os.path.split(path)
dirpath = dirpath or '.'
fileroot, fileext = os.path.splitext(filename)
subtitles = {}
for p in os.listdir(dirpath):
# keep only valid subtitle filenames
if not p.startswith(fileroot) or not p.endswith(SUBTITLE_EXTENSIONS):
continue
# extract the potential language code
language_code = p[len(fileroot):-len(os.path.splitext(p)[1])].replace(fileext, '').replace('_', '-')[1:]
# default language is undefined
language = Language('und')
# attempt to parse
if language_code:
try:
language = Language.fromietf(language_code)
except ValueError:
logger.error('Cannot parse language code %r', language_code)
subtitles[p] = language
logger.debug('Found subtitles %r', subtitles)
return subtitles
def reverse(self, name):
with_country = (GuessitConverter._with_country_regexp.match(name) or
GuessitConverter._with_country_regexp2.match(name))
name = u(name.lower())
if with_country:
lang = Language.fromguessit(with_country.group(1).strip())
lang.country = babelfish.Country.fromguessit(with_country.group(2).strip())
return (lang.alpha3, lang.country.alpha2 if lang.country else None, lang.script or None)
# exceptions come first, as they need to override a potential match
# with any of the other guessers
try:
return self.guessit_exceptions[name]
except KeyError:
pass
for conv in [babelfish.Language,
babelfish.Language.fromalpha3b,
babelfish.Language.fromalpha2,
babelfish.Language.fromname,
babelfish.Language.fromopensubtitles]:
try:
c = conv(name)
return c.alpha3, c.country, c.script
except (ValueError, babelfish.LanguageReverseError):
pass
raise babelfish.LanguageReverseError(name)
def query(self, languages, hash=None, size=None, imdb_id=None, query=None, season=None, episode=None, tag=None):
# fill the search criteria
criteria = []
if hash and size:
criteria.append({'moviehash': hash, 'moviebytesize': str(size)})
if imdb_id:
if season and episode:
criteria.append({'imdbid': imdb_id[2:], 'season': season, 'episode': episode})
else:
criteria.append({'imdbid': imdb_id[2:]})
if tag:
criteria.append({'tag': tag})
if query and season and episode:
criteria.append({'query': query.replace('\'', ''), 'season': season, 'episode': episode})
elif query:
criteria.append({'query': query.replace('\'', '')})
if not criteria:
raise ValueError('Not enough information')
# add the language
for criterion in criteria:
criterion['sublanguageid'] = ','.join(sorted(l.opensubtitles for l in languages))
# query the server
logger.info('Searching subtitles %r', criteria)
response = checked(self.server.SearchSubtitles(self.token, criteria))
subtitles = []
# exit if no data
if not response['data']:
logger.debug('No subtitles found')
return subtitles
# loop over subtitle items
for subtitle_item in response['data']:
# read the item
language = Language.fromopensubtitles(subtitle_item['SubLanguageID'])
hearing_impaired = bool(int(subtitle_item['SubHearingImpaired']))
page_link = subtitle_item['SubtitlesLink']
subtitle_id = int(subtitle_item['IDSubtitleFile'])
matched_by = subtitle_item['MatchedBy']
movie_kind = subtitle_item['MovieKind']
hash = subtitle_item['MovieHash']
movie_name = subtitle_item['MovieName']
movie_release_name = subtitle_item['MovieReleaseName']
movie_year = int(subtitle_item['MovieYear']) if subtitle_item['MovieYear'] else None
movie_imdb_id = 'tt' + subtitle_item['IDMovieImdb']
series_season = int(subtitle_item['SeriesSeason']) if subtitle_item['SeriesSeason'] else None
series_episode = int(subtitle_item['SeriesEpisode']) if subtitle_item['SeriesEpisode'] else None
filename = subtitle_item['SubFileName']
encoding = subtitle_item.get('SubEncoding') or None
subtitle = self.subtitle_class(language, hearing_impaired, page_link, subtitle_id, matched_by, movie_kind,
hash, movie_name, movie_release_name, movie_year, movie_imdb_id,
series_season, series_episode, filename, encoding)
logger.debug('Found subtitle %r by %s', subtitle, matched_by)
subtitles.append(subtitle)
return subtitles
def test_ne_with_country(self):
self.assertNotEqual(Language('eng', 'US'),
Language('eng', Country('GB')))
def get_language(lang_short):
return Language.fromietf(lang_short)
def test_eq_with_country(self):
self.assertEqual(Language('eng', 'US'), Language('eng', Country('US')))
def test_converter_type(self):
self.assertEqual(language_converters['type'].codes,
set(['A', 'C', 'E', 'H', 'L', 'S']))
self.assertEqual(Language('eng').type, 'living')
self.assertEqual(Language('und').type, 'special')
def __init__(self, directory, filename, logger=None):
# Setup logging
if logger:
log = logger
else:
log = logging.getLogger(__name__)
# Setup encoding to avoid UTF-8 errors
if sys.version[0] == '2':
SYS_ENCODING = None
try:
locale.setlocale(locale.LC_ALL, "")
SYS_ENCODING = locale.getpreferredencoding()
except (locale.Error, IOError):
pass
# For OSes that are poorly configured just force UTF-8
if not SYS_ENCODING or SYS_ENCODING in ('ANSI_X3.4-1968',
'US-ASCII', 'ASCII'):
SYS_ENCODING = 'UTF-8'
if not hasattr(sys, "setdefaultencoding"):
reload(sys)
try:
# pylint: disable=E1101
# On non-unicode builds this will raise an AttributeError, if encoding type is not valid it throws a LookupError
sys.setdefaultencoding(SYS_ENCODING)
except:
log.exception(
"Sorry, your environment is not setup correctly for utf-8 support. Please fix your setup and try again"
)
sys.exit(
"Sorry, your environment is not setup correctly for utf-8 support. Please fix your setup and try again"
)
log.info(sys.executable)
# Default settings for SickBeard
sb_defaults = {
'host': 'localhost',
'port': '8081',
'ssl': "False",
'api_key': '',
'web_root': '',
'username': '',
'password': ''
}
# Default MP4 conversion settings
mp4_defaults = {
'ffmpeg': 'ffmpeg.exe',
'ffprobe': 'ffprobe.exe',
'threads': 'auto',
'output_directory': '',
'copy_to': '',
'move_to': '',
'output_extension': 'mp4',
'output_format': 'mp4',
'delete_original': 'True',
'relocate_moov': 'True',
'ios-audio': 'True',
'ios-first-track-only': 'False',
'ios-audio-filter': '',
'max-audio-channels': '',
'audio-language': '',
'audio-default-language': '',
'audio-codec': 'ac3',
'audio-filter': '',
'audio-channel-bitrate': '256',
'video-codec': 'h264, x264',
'video-bitrate': '',
'video-max-width': '',
'h264-max-level': '',
'aac_adtstoasc': 'False',
'use-qsv-decoder-with-encoder': 'True',
'subtitle-codec': 'mov_text',
'subtitle-language': '',
'subtitle-default-language': '',
'subtitle-encoding': '',
'convert-mp4': 'False',
'fullpathguess': 'True',
'tagfile': 'True',
'tag-language': 'en',
'download-artwork': 'poster',
'download-subs': 'False',
'embed-subs': 'True',
'sub-providers': 'addic7ed, podnapisi, thesubdb, opensubtitles',
'permissions': '777',
'post-process': 'False',
'pix-fmt': ''
}
# Default settings for CouchPotato
cp_defaults = {
'host': 'localhost',
'port': '5050',
'username': '',
'password': '',
'apikey': '',
'delay': '65',
'method': 'renamer',
'delete_failed': 'False',
'ssl': 'False',
'web_root': ''
}
# Default settings for Sonarr
sonarr_defaults = {
'host': 'localhost',
'port': '8989',
'apikey': '',
'ssl': 'False',
'web_root': ''
}
# Default uTorrent settings
utorrent_defaults = {
'couchpotato-label': 'couchpotato',
'sickbeard-label': 'sickbeard',
'sickrage-label': 'sickrage',
'sonarr-label': 'sonarr',
'bypass-label': 'bypass',
'convert': 'True',
'webui': 'False',
'action_before': 'stop',
'action_after': 'removedata',
'host': 'http://localhost:8080/',
'username': '',
'password': ''
}
# Default SAB settings
sab_defaults = {
'convert': 'True',
'Sickbeard-category': 'sickbeard',
'Sickrage-category': 'sickrage',
'Couchpotato-category': 'couchpotato',
'Sonarr-category': 'sonarr',
'Bypass-category': 'bypass'
}
# Default Sickrage Settings
sr_defaults = {
'host': 'localhost',
'port': '8081',
'ssl': "False",
'api_key': '',
'web_root': '',
'username': '',
'password': ''
}
# Default deluge settings
deluge_defaults = {
'couchpotato-label': 'couchpotato',
'sickbeard-label': 'sickbeard',
'sickrage-label': 'sickrage',
'sonarr-label': 'sonarr',
'bypass-label': 'bypass',
'convert': 'True',
'host': 'localhost',
'port': '58846',
'username': '',
'password': ''
}
# Default Plex Settings
plex_defaults = {
'host': 'localhost',
'port': '32400',
'refresh': 'true',
'token': ''
}
defaults = {
'SickBeard': sb_defaults,
'CouchPotato': cp_defaults,
'Sonarr': sonarr_defaults,
'MP4': mp4_defaults,
'uTorrent': utorrent_defaults,
'SABNZBD': sab_defaults,
'Sickrage': sr_defaults,
'Deluge': deluge_defaults,
'Plex': plex_defaults
}
write = False # Will be changed to true if a value is missing from the config file and needs to be written
config = configparser.SafeConfigParser()
configFile = os.path.join(directory, filename)
if os.path.isfile(configFile):
config.read(configFile)
else:
log.error("Config file not found, creating %s." % configFile)
# config.filename = filename
write = True
# Make sure all sections and all keys for each section are present
for s in defaults:
if not config.has_section(s):
config.add_section(s)
write = True
for k in defaults[s]:
if not config.has_option(s, k):
config.set(s, k, defaults[s][k])
write = True
# If any keys are missing from the config file, write them
if write:
self.writeConfig(config, configFile)
# Read relevant MP4 section information
section = "MP4"
self.ffmpeg = os.path.normpath(self.raw(config.get(
section, "ffmpeg"))) # Location of FFMPEG.exe
self.ffprobe = os.path.normpath(
self.raw(config.get(section,
"ffprobe"))) # Location of FFPROBE.exe
self.threads = config.get(section,
"threads") # Number of FFMPEG threads
try:
if int(self.threads) < 1:
self.threads = "auto"
except:
self.threads = "auto"
self.output_dir = config.get(section, "output_directory")
if self.output_dir == '':
self.output_dir = None
else:
self.output_dir = os.path.normpath(self.raw(
self.output_dir)) # Output directory
self.copyto = config.get(
section,
"copy_to") # Directories to make copies of the final product
if self.copyto == '':
self.copyto = None
else:
self.copyto = self.copyto.split('|')
for i in range(len(self.copyto)):
self.copyto[i] = os.path.normpath(self.copyto[i])
if not os.path.isdir(self.copyto[i]):
try:
os.makedirs(self.copyto[i])
except:
log.exception("Error making directory %s." %
(self.copyto[i]))
self.moveto = config.get(
section, "move_to") # Directory to move final product to
if self.moveto == '':
self.moveto = None
else:
self.moveto = os.path.normpath(self.moveto)
if not os.path.isdir(self.moveto):
try:
os.makedirs(self.moveto)
except:
log.exception("Error making directory %s." % (self.moveto))
self.moveto = None
self.output_extension = config.get(
section, "output_extension") # Output extension
self.output_format = config.get(section,
"output_format") # Output format
if self.output_format not in valid_formats:
self.output_format = 'mov'
self.delete = config.getboolean(
section, "delete_original") # Delete original file
self.relocate_moov = config.getboolean(
section, "relocate_moov") # Relocate MOOV atom to start of file
if self.relocate_moov:
try:
import qtfaststart
except:
log.error(
"Please install QTFastStart via PIP, relocate_moov will be disabled without this module."
)
self.relocate_moov = False
self.acodec = config.get(section, "audio-codec").lower(
) # Gets the desired audio codec, if no valid codec selected, default to AC3
if self.acodec == '':
self.acodec == ['ac3']
else:
self.acodec = self.acodec.lower().replace(' ', '').split(',')
self.abitrate = config.get(section, "audio-channel-bitrate")
try:
self.abitrate = int(self.abitrate)
except:
self.abitrate = 256
log.warning(
"Audio bitrate was invalid, defaulting to 256 per channel.")
if self.abitrate > 256:
log.warning(
"Audio bitrate >256 may create errors with common codecs.")
self.afilter = config.get(
section, "audio-filter").lower().strip() # Audio filter
if self.afilter == '':
self.afilter = None
self.iOS = config.get(
section, "ios-audio"
) # Creates a second audio channel if the standard output methods are different from this for iOS compatability
if self.iOS == "" or self.iOS.lower() in ['false', 'no', 'f', '0']:
self.iOS = False
else:
if self.iOS.lower() in ['true', 'yes', 't', '1']:
self.iOS = ['aac']
else:
self.iOS = self.iOS.lower().replace(' ', '').split(',')
self.iOSFirst = config.getboolean(
section, "ios-first-track-only"
) # Enables the iOS audio option only for the first track
self.iOSfilter = config.get(
section, "ios-audio-filter").lower().strip() # iOS audio filter
if self.iOSfilter == '':
self.iOSfilter = None
self.downloadsubs = config.getboolean(
section, "download-subs"
) # Enables downloading of subtitles from the internet sources using subliminal
if self.downloadsubs:
try:
import subliminal
except Exception as e:
self.downloadsubs = False
log.exception(
"Subliminal is not installed, automatically downloading of subs has been disabled."
)
self.subproviders = config.get(section, 'sub-providers').lower()
if self.subproviders == '':
self.downloadsubs = False
log.warning(
"You must specifiy at least one subtitle provider to downlaod subs automatically, subtitle downloading disabled."
)
else:
self.subproviders = self.subproviders.lower().replace(
' ', '').split(',')
self.embedsubs = config.getboolean(section, 'embed-subs')
self.permissions = config.get(section, 'permissions')
try:
self.permissions = int(self.permissions, 8)
except:
self.log.exception("Invalid permissions, defaulting to 777.")
self.permissions = int("0777", 8)
try:
self.postprocess = config.getboolean(section, 'post-process')
except:
self.postprocess = False
self.aac_adtstoasc = config.getboolean(section, 'aac_adtstoasc')
# Setup variable for maximum audio channels
self.maxchannels = config.get(section, 'max-audio-channels')
if self.maxchannels == "":
self.maxchannels = None
else:
try:
self.maxchannels = int(self.maxchannels)
except:
log.exception("Invalid number of audio channels specified.")
self.maxchannels = None
if self.maxchannels is not None and self.maxchannels < 1:
log.warning("Must have at least 1 audio channel.")
self.maxchannels = None
self.vcodec = config.get(section, "video-codec")
if self.vcodec == '':
self.vcodec == ['h264', 'x264']
else:
self.vcodec = self.vcodec.lower().replace(' ', '').split(',')
self.vbitrate = config.get(section, "video-bitrate")
if self.vbitrate == '':
self.vbitrate = None
else:
try:
self.vbitrate = int(self.vbitrate)
if not (self.vbitrate > 0):
self.vbitrate = None
log.warning(
"Video bitrate must be greater than 0, defaulting to no video bitrate cap."
)
except:
log.exception(
"Invalid video bitrate, defaulting to no video bitrate cap."
)
self.vbitrate = None
self.vwidth = config.get(section, "video-max-width")
if self.vwidth == '':
self.vwidth = None
else:
try:
self.vwidth = int(self.vwidth)
except:
log.exception("Invalid video width, defaulting to none.")
self.vwidth = None
self.h264_level = config.get(section, "h264-max-level")
if self.h264_level == '':
self.h264_level = None
else:
try:
self.h264_level = float(self.h264_level)
except:
log.exception("Invalid h264 level, defaulting to none.")
self.h264_level = None
self.qsv_decoder = config.getboolean(
section, "use-qsv-decoder-with-encoder"
) # Use Intel QuickSync Decoder when using QuickSync Encoder
self.pix_fmt = config.get(section, "pix-fmt").strip().lower()
if self.pix_fmt == '':
self.pix_fmt = None
else:
self.pix_fmt = self.pix_fmt.replace(' ', '').split(',')
self.awl = config.get(section, 'audio-language').strip().lower(
) # List of acceptable languages for audio streams to be carried over from the original file, separated by a comma. Blank for all
if self.awl == '':
self.awl = None
else:
self.awl = self.awl.replace(' ', '').split(',')
self.scodec = config.get(section, 'subtitle-codec').strip().lower()
if not self.scodec or self.scodec == "":
if self.embedsubs:
self.scodec = ['mov_text']
else:
self.scodec = ['srt']
log.warning("Invalid subtitle codec, defaulting to '%s'." %
self.scodec)
else:
self.scodec = self.scodec.replace(' ', '').split(',')
if self.embedsubs:
if len(self.scodec) > 1:
log.warning(
"Can only embed one subtitle type, defaulting to 'mov_text'."
)
self.scodec = ['mov_text']
if self.scodec[0] not in valid_internal_subcodecs:
log.warning(
"Invalid interal subtitle codec %s, defaulting to 'mov_text'."
% self.scodec[0])
self.scodec = ['mov_text']
else:
for codec in self.scodec:
if codec not in valid_external_subcodecs:
log.warning(
"Invalid external subtitle codec %s, ignoring." %
codec)
self.scodec.remove(codec)
if len(self.scodec) == 0:
log.warning(
"No valid subtitle formats found, defaulting to 'srt'.")
self.scodec = ['srt']
self.swl = config.get(section, 'subtitle-language').strip().lower(
) # List of acceptable languages for subtitle streams to be carried over from the original file, separated by a comma. Blank for all
if self.swl == '':
self.swl = None
else:
self.swl = self.swl.replace(' ', '').split(',')
self.subencoding = config.get(section,
'subtitle-encoding').strip().lower()
if self.subencoding == '':
self.subencoding = None
self.adl = config.get(section, 'audio-default-language').strip().lower(
) # What language to default an undefinied audio language tag to. If blank, it will remain undefined. This is useful for single language releases which tend to leave things tagged as und
if self.adl == "" or len(self.adl) > 3:
self.adl = None
self.sdl = config.get(section, 'subtitle-default-language').strip(
).lower(
) # What language to default an undefinied subtitle language tag to. If blank, it will remain undefined. This is useful for single language releases which tend to leave things tagged as und
if self.sdl == "" or len(self.sdl) > 3:
self.sdl = None
# Prevent incompatible combination of settings
if self.output_dir == "" and self.delete is False:
log.error(
"You must specify an alternate output directory if you aren't going to delete the original file."
)
sys.exit()
# Create output directory if it does not exist
if self.output_dir is not None:
if not os.path.isdir(self.output_dir):
os.makedirs(self.output_dir)
self.processMP4 = config.getboolean(
section, "convert-mp4"
) # Determine whether or not to reprocess mp4 files or just tag them
self.fullpathguess = config.getboolean(
section, "fullpathguess") # Guess using the full path or not
self.tagfile = config.getboolean(section,
"tagfile") # Tag files with metadata
self.taglanguage = config.get(
section, "tag-language").strip().lower() # Language to tag files
if len(self.taglanguage) > 2:
try:
babel = Language(self.taglanguage)
self.taglanguage = babel.alpha2
except:
log.exception(
"Unable to set tag language, defaulting to English.")
self.taglanguage = 'en'
elif len(self.taglanguage) < 2:
log.exception("Unable to set tag language, defaulting to English.")
self.taglanguage = 'en'
self.artwork = config.get(
section, "download-artwork").lower() # Download and embed artwork
if self.artwork == "poster":
self.artwork = True
self.thumbnail = False
elif self.artwork == "thumb" or self.artwork == "thumbnail":
self.artwork = True
self.thumbnail = True
else:
self.thumbnail = False
try:
self.artwork = config.getboolean(section, "download-artwork")
except:
self.artwork = True
self.log.error(
"Invalid download-artwork value, defaulting to 'poster'.")
# Read relevant CouchPotato section information
section = "CouchPotato"
self.CP = {}
self.CP['host'] = config.get(section, "host")
self.CP['port'] = config.get(section, "port")
self.CP['username'] = config.get(section, "username")
self.CP['password'] = config.get(section, "password")
self.CP['apikey'] = config.get(section, "apikey")
self.CP['delay'] = config.get(section, "delay")
self.CP['method'] = config.get(section, "method")
self.CP['web_root'] = config.get(section, "web_root")
try:
self.CP['delay'] = float(self.CP['delay'])
except ValueError:
self.CP['delay'] = 60
try:
self.CP['delete_failed'] = config.getboolean(
section, "delete_failed")
except (configparser.NoOptionError, ValueError):
self.CP['delete_failed'] = False
try:
if config.getboolean(section, 'ssl'):
self.CP['protocol'] = "https://"
else:
self.CP['protocol'] = "http://"
except (configparser.NoOptionError, ValueError):
self.CP['protocol'] = "http://"
# Read relevant uTorrent section information
section = "uTorrent"
self.uTorrent = {}
self.uTorrent['cp'] = config.get(section, "couchpotato-label").lower()
self.uTorrent['sb'] = config.get(section, "sickbeard-label").lower()
self.uTorrent['sr'] = config.get(section, "sickrage-label").lower()
self.uTorrent['sonarr'] = config.get(section, "sonarr-label").lower()
self.uTorrent['bypass'] = config.get(section, "bypass-label").lower()
try:
self.uTorrent['convert'] = config.getboolean(section, "convert")
except:
self.uTorrent['convert'] = False
self.uTorrentWebUI = config.getboolean(section, "webui")
self.uTorrentActionBefore = config.get(section,
"action_before").lower()
self.uTorrentActionAfter = config.get(section, "action_after").lower()
self.uTorrentHost = config.get(section, "host").lower()
self.uTorrentUsername = config.get(section, "username")
self.uTorrentPassword = config.get(section, "password")
# Read relevant Deluge section information
section = "Deluge"
self.deluge = {}
self.deluge['cp'] = config.get(section, "couchpotato-label").lower()
self.deluge['sb'] = config.get(section, "sickbeard-label").lower()
self.deluge['sr'] = config.get(section, "sickrage-label").lower()
self.deluge['sonarr'] = config.get(section, "sonarr-label").lower()
self.deluge['bypass'] = config.get(section, "bypass-label").lower()
try:
self.deluge['convert'] = config.getboolean(section, "convert")
except:
self.deluge['convert'] = False
self.deluge['host'] = config.get(section, "host").lower()
self.deluge['port'] = config.get(section, "port")
self.deluge['user'] = config.get(section, "username")
self.deluge['pass'] = config.get(section, "password")
# Read relevant Sonarr section information
section = "Sonarr"
self.Sonarr = {}
self.Sonarr['host'] = config.get(section, "host")
self.Sonarr['port'] = config.get(section, "port")
self.Sonarr['apikey'] = config.get(section, "apikey")
self.Sonarr['ssl'] = config.get(section, "ssl")
self.Sonarr['web_root'] = config.get(section, "web_root")
# Read Sickbeard section information
section = "SickBeard"
self.Sickbeard = {}
self.Sickbeard['host'] = config.get(section, "host") # Server Address
self.Sickbeard['port'] = config.get(section, "port") # Server Port
self.Sickbeard['api_key'] = config.get(section,
"api_key") # Sickbeard API key
self.Sickbeard['web_root'] = config.get(
section, "web_root") # Sickbeard webroot
self.Sickbeard['ssl'] = config.getboolean(section, "ssl") # SSL
self.Sickbeard['user'] = config.get(section, "username")
self.Sickbeard['pass'] = config.get(section, "password")
# Read Sickrage section information
section = "Sickrage"
self.Sickrage = {}
self.Sickrage['host'] = config.get(section, "host") # Server Address
self.Sickrage['port'] = config.get(section, "port") # Server Port
self.Sickrage['api_key'] = config.get(section,
"api_key") # Sickbeard API key
self.Sickrage['web_root'] = config.get(section,
"web_root") # Sickbeard webroot
self.Sickrage['ssl'] = config.getboolean(section, "ssl") # SSL
self.Sickrage['user'] = config.get(section, "username")
self.Sickrage['pass'] = config.get(section, "password")
# Read SAB section information
section = "SABNZBD"
self.SAB = {}
try:
self.SAB['convert'] = config.getboolean(section,
"convert") # Convert
except:
self.SAB['convert'] = False
self.SAB['cp'] = config.get(section, "Couchpotato-category").lower()
self.SAB['sb'] = config.get(section, "Sickbeard-category").lower()
self.SAB['sr'] = config.get(section, "Sickrage-category").lower()
self.SAB['sonarr'] = config.get(section, "Sonarr-category").lower()
self.SAB['bypass'] = config.get(section, "Bypass-category").lower()
# Read Plex section information
section = "Plex"
self.Plex = {}
self.Plex['host'] = config.get(section, "host")
self.Plex['port'] = config.get(section, "port")
try:
self.Plex['refresh'] = config.getboolean(section, "refresh")
except:
self.Plex['refresh'] = False
self.Plex['token'] = config.get(section, "token")
if self.Plex['token'] == '':
self.Plex['token'] = None
# Pass the values on
self.config = config
self.configFile = configFile
def convert(self, value, param, ctx):
try:
return Language.fromietf(value)
except BabelfishError:
self.fail('%s is not a valid language' % value)
def test_wrong_language(self):
self.assertRaises(ValueError, lambda: Language('zzz'))
def test_fromietf_wrong_country(self):
self.assertRaises(ValueError, lambda: Language.fromietf('fra-YZ'))
def test_fromietf_wrong_script(self):
self.assertRaises(ValueError, lambda: Language.fromietf('fra-FR-Wxyz'))
def test_fromietf_alpha2_language(self):
language = Language.fromietf('fr-Latn')
self.assertEqual(language.alpha3, 'fra')
self.assertIsNone(language.country)
self.assertEqual(language.script, Script('Latn'))
def test_fromietf_wrong_language(self):
self.assertRaises(ValueError, lambda: Language.fromietf('xyz-FR'))
def test_fromietf_no_country_no_script(self):
language = Language.fromietf('fra-FR')
self.assertEqual(language.alpha3, 'fra')
self.assertEqual(language.country, Country('FR'))
self.assertIsNone(language.script)
def test_fromietf_country_script(self):
language = Language.fromietf('fra-FR-Latn')
self.assertEqual(language.alpha3, 'fra')
self.assertEqual(language.country, Country('FR'))
self.assertEqual(language.script, Script('Latn'))
def test_converter_opensubtitles(self):
self.assertEqual(
Language('fra').opensubtitles,
Language('fra').alpha3b)
self.assertEqual(Language('por', 'BR').opensubtitles, 'pob')
self.assertEqual(Language.fromopensubtitles('fre'), Language('fra'))
self.assertEqual(Language.fromopensubtitles('pob'),
Language('por', 'BR'))
self.assertEqual(Language.fromopensubtitles('pb'),
Language('por', 'BR'))
# Montenegrin is not recognized as an ISO language (yet?) but for now it is
# unofficially accepted as Serbian from Montenegro
self.assertEqual(Language.fromopensubtitles('mne'),
Language('srp', 'ME'))
self.assertEqual(Language.fromcode('pob', 'opensubtitles'),
Language('por', 'BR'))
self.assertRaises(LanguageReverseError,
lambda: Language.fromopensubtitles('zzz'))
self.assertRaises(LanguageConvertError,
lambda: Language('aaa').opensubtitles)
self.assertEqual(len(language_converters['opensubtitles'].codes), 606)
# test with all the LANGUAGES from the opensubtitles api
# downloaded from: http://www.opensubtitles.org/addons/export_languages.php
f = resource_stream('babelfish', 'data/opensubtitles_languages.txt')
f.readline()
for l in f:
idlang, alpha2, _, upload_enabled, web_enabled = l.decode(
'utf-8').strip().split('\t')
if not int(upload_enabled) and not int(web_enabled):
# do not test LANGUAGES that are too esoteric / not widely available
continue
self.assertEqual(
Language.fromopensubtitles(idlang).opensubtitles, idlang)
if alpha2:
self.assertEqual(Language.fromopensubtitles(idlang),
Language.fromopensubtitles(alpha2))
f.close()
def test_script(self):
self.assertEqual(Language('srp', script='Latn').script, Script('Latn'))
self.assertEqual(
Language('srp', script=Script('Cyrl')).script, Script('Cyrl'))
def test_unknown_language(self):
self.assertEqual(Language('zzzz', unknown='und'), Language('und'))
def languages(self):
return {
Language.fromietf(l)
for l in json.loads(self.config.get('general', 'languages'))
}
class OpenSubtitlesProvider(Provider):
"""OpenSubtitles Provider.
:param str username: username.
:param str password: password.
"""
languages = {
Language.fromopensubtitles(l)
for l in language_converters['opensubtitles'].codes
}
def __init__(self, username=None, password=None):
self.server = ServerProxy('https://api.opensubtitles.org/xml-rpc',
TimeoutSafeTransport(10))
if username and not password or not username and password:
raise ConfigurationError('Username and password must be specified')
# None values not allowed for logging in, so replace it by ''
self.username = username or ''
self.password = password or ''
self.token = None
def initialize(self):
logger.info('Logging in')
response = checked(
self.server.LogIn(self.username, self.password, 'eng',
'Grabber v1'))
self.token = response['token']
logger.debug('Logged in with token %r', self.token)
def terminate(self):
logger.info('Logging out')
checked(self.server.LogOut(self.token))
self.server.close()
self.token = None
logger.debug('Logged out')
def no_operation(self):
logger.debug('No operation')
checked(self.server.NoOperation(self.token))
def query(self,
languages,
hash=None,
size=None,
imdb_id=None,
query=None,
season=None,
episode=None,
tag=None):
# fill the search criteria
criteria = []
if hash and size:
criteria.append({'moviehash': hash, 'moviebytesize': str(size)})
if imdb_id:
criteria.append({'imdbid': imdb_id[2:]})
if tag:
criteria.append({'tag': tag})
if query and season and episode:
criteria.append({
'query': query.replace('\'', ''),
'season': season,
'episode': episode
})
elif query:
criteria.append({'query': query.replace('\'', '')})
if not criteria:
raise ValueError('Not enough information')
# add the language
for criterion in criteria:
criterion['sublanguageid'] = ','.join(
sorted(l.opensubtitles for l in languages))
# query the server
logger.info('Searching subtitles %r', criteria)
response = checked(self.server.SearchSubtitles(self.token, criteria))
subtitles = []
# exit if no data
if not response['data']:
logger.debug('No subtitles found')
return subtitles
# loop over subtitle items
for subtitle_item in response['data']:
# read the item
language = Language.fromopensubtitles(
subtitle_item['SubLanguageID'])
hearing_impaired = bool(int(subtitle_item['SubHearingImpaired']))
page_link = subtitle_item['SubtitlesLink']
subtitle_id = int(subtitle_item['IDSubtitleFile'])
matched_by = subtitle_item['MatchedBy']
movie_kind = subtitle_item['MovieKind']
hash = subtitle_item['MovieHash']
movie_name = subtitle_item['MovieName']
movie_release_name = subtitle_item['MovieReleaseName']
movie_year = int(subtitle_item['MovieYear']
) if subtitle_item['MovieYear'] else None
movie_imdb_id = 'tt' + subtitle_item['IDMovieImdb']
series_season = int(subtitle_item['SeriesSeason']
) if subtitle_item['SeriesSeason'] else None
series_episode = int(subtitle_item['SeriesEpisode']
) if subtitle_item['SeriesEpisode'] else None
filename = subtitle_item['SubFileName']
encoding = subtitle_item.get('SubEncoding') or None
subtitle = OpenSubtitlesSubtitle(
language, hearing_impaired, page_link, subtitle_id, matched_by,
movie_kind, hash, movie_name, movie_release_name, movie_year,
movie_imdb_id, series_season, series_episode, filename,
encoding)
logger.debug('Found subtitle %r by %s', subtitle, matched_by)
subtitles.append(subtitle)
return subtitles
def list_subtitles(self, video, languages):
season = episode = None
if isinstance(video, Episode):
query = video.series
season = video.season
episode = video.episode
else:
query = video.title
return self.query(languages,
hash=video.hashes.get('opensubtitles'),
size=video.size,
imdb_id=video.imdb_id,
query=query,
season=season,
episode=episode,
tag=os.path.basename(video.name))
def download_subtitle(self, subtitle):
logger.info('Downloading subtitle %r', subtitle)
response = checked(
self.server.DownloadSubtitles(self.token,
[str(subtitle.subtitle_id)]))
subtitle.content = fix_line_ending(
zlib.decompress(base64.b64decode(response['data'][0]['data']), 47))
def query(self,
languages,
hash=None,
size=None,
imdb_id=None,
query=None,
season=None,
episode=None,
tag=None):
# fill the search criteria
criteria = []
if hash and size:
criteria.append({'moviehash': hash, 'moviebytesize': str(size)})
if imdb_id:
criteria.append({'imdbid': imdb_id[2:]})
if tag:
criteria.append({'tag': tag})
if query and season and episode:
criteria.append({
'query': query.replace('\'', ''),
'season': season,
'episode': episode
})
elif query:
criteria.append({'query': query.replace('\'', '')})
if not criteria:
raise ValueError('Not enough information')
# add the language
for criterion in criteria:
criterion['sublanguageid'] = ','.join(
sorted(l.opensubtitles for l in languages))
# query the server
logger.info('Searching subtitles %r', criteria)
response = checked(self.server.SearchSubtitles(self.token, criteria))
subtitles = []
# exit if no data
if not response['data']:
logger.debug('No subtitles found')
return subtitles
# loop over subtitle items
for subtitle_item in response['data']:
# read the item
language = Language.fromopensubtitles(
subtitle_item['SubLanguageID'])
hearing_impaired = bool(int(subtitle_item['SubHearingImpaired']))
page_link = subtitle_item['SubtitlesLink']
subtitle_id = int(subtitle_item['IDSubtitleFile'])
matched_by = subtitle_item['MatchedBy']
movie_kind = subtitle_item['MovieKind']
hash = subtitle_item['MovieHash']
movie_name = subtitle_item['MovieName']
movie_release_name = subtitle_item['MovieReleaseName']
movie_year = int(subtitle_item['MovieYear']
) if subtitle_item['MovieYear'] else None
movie_imdb_id = 'tt' + subtitle_item['IDMovieImdb']
series_season = int(subtitle_item['SeriesSeason']
) if subtitle_item['SeriesSeason'] else None
series_episode = int(subtitle_item['SeriesEpisode']
) if subtitle_item['SeriesEpisode'] else None
filename = subtitle_item['SubFileName']
encoding = subtitle_item.get('SubEncoding') or None
subtitle = OpenSubtitlesSubtitle(
language, hearing_impaired, page_link, subtitle_id, matched_by,
movie_kind, hash, movie_name, movie_release_name, movie_year,
movie_imdb_id, series_season, series_episode, filename,
encoding)
logger.debug('Found subtitle %r by %s', subtitle, matched_by)
subtitles.append(subtitle)
return subtitles
class SubsCenterProvider(Provider):
"""SubsCenter Provider."""
languages = {Language.fromalpha2(l) for l in ['he']}
server_url = 'http://subscenter.cinemast.com/he/'
def __init__(self, username=None, password=None):
if username is not None and password is None or username is None and password is not None:
raise ConfigurationError('Username and password must be specified')
self.username = username
self.password = password
self.logged_in = False
def initialize(self):
self.session = Session()
self.session.headers[
'User-Agent'] = 'Subliminal/%s' % __short_version__
# login
if self.username is not None and self.password is not None:
logger.debug('Logging in')
url = self.server_url + 'subscenter/accounts/login/'
# retrieve CSRF token
self.session.get(url)
csrf_token = self.session.cookies['csrftoken']
# actual login
data = {
'username': self.username,
'password': self.password,
'csrfmiddlewaretoken': csrf_token
}
r = self.session.post(url, data, allow_redirects=False, timeout=10)
if r.status_code != 302:
raise AuthenticationError(self.username)
logger.info('Logged in')
self.logged_in = True
def terminate(self):
# logout
if self.logged_in:
logger.info('Logging out')
r = self.session.get(self.server_url +
'subscenter/accounts/logout/',
timeout=10)
r.raise_for_status()
logger.info('Logged out')
self.logged_in = False
self.session.close()
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
def _search_url_titles(self, title):
"""Search the URL titles by kind for the given `title`.
:param str title: title to search for.
:return: the URL titles by kind.
:rtype: collections.defaultdict
"""
# make the search
logger.info('Searching title name for %r', title)
r = self.session.get(self.server_url + 'subtitle/search/',
params={'q': title},
timeout=10)
r.raise_for_status()
# get the suggestions
soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
links = soup.select('#processes div.generalWindowTop a')
logger.debug('Found %d suggestions', len(links))
url_titles = defaultdict(list)
for link in links:
parts = link.attrs['href'].split('/')
url_titles[parts[-3]].append(parts[-2])
return url_titles
def query(self, title, season=None, episode=None):
# search for the url title
url_titles = self._search_url_titles(title)
# episode
if season and episode:
if 'series' not in url_titles:
logger.error('No URL title found for series %r', title)
return []
url_title = url_titles['series'][0]
logger.debug('Using series title %r', url_title)
url = self.server_url + 'cinemast/data/series/sb/{}/{}/{}/'.format(
url_title, season, episode)
page_link = self.server_url + 'subtitle/series/{}/{}/{}/'.format(
url_title, season, episode)
else:
if 'movie' not in url_titles:
logger.error('No URL title found for movie %r', title)
return []
url_title = url_titles['movie'][0]
logger.debug('Using movie title %r', url_title)
url = self.server_url + 'cinemast/data/movie/sb/{}/'.format(
url_title)
page_link = self.server_url + 'subtitle/movie/{}/'.format(
url_title)
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
results = json.loads(r.text)
# loop over results
subtitles = {}
for language_code, language_data in results.items():
for quality_data in language_data.values():
for quality, subtitles_data in quality_data.items():
for subtitle_item in subtitles_data.values():
# read the item
language = Language.fromalpha2(language_code)
hearing_impaired = bool(
subtitle_item['hearing_impaired'])
subtitle_id = subtitle_item['id']
subtitle_key = subtitle_item['key']
downloaded = subtitle_item['downloaded']
release = subtitle_item['subtitle_version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug(
'Found additional release %r for subtitle %d',
release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases,
release) # deterministic order
subtitles[subtitle_id].downloaded += downloaded
continue
# otherwise create it
subtitle = SubsCenterSubtitle(
language, hearing_impaired, page_link, title,
season, episode, title, subtitle_id, subtitle_key,
downloaded, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def list_subtitles(self, video, languages):
season = episode = None
title = video.title
if isinstance(video, Episode):
title = video.series
season = video.season
episode = video.episode
return [
s for s in self.query(title, season, episode)
if s.language in languages
]
def download_subtitle(self, subtitle):
# download
url = self.server_url + 'subtitle/download/{}/{}/'.format(
subtitle.language.alpha2, subtitle.subtitle_id)
params = {'v': subtitle.releases[0], 'key': subtitle.subtitle_key}
r = self.session.get(url,
params=params,
headers={'Referer': subtitle.page_link},
timeout=10)
r.raise_for_status()
# open the zip
with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
# remove some filenames from the namelist
namelist = [n for n in zf.namelist() if not n.endswith('.txt')]
if len(namelist) > 1:
raise ProviderError('More than one file to unzip')
subtitle.content = fix_line_ending(zf.read(namelist[0]))
def test_eq(self):
self.assertEqual(Language('eng'), Language('eng'))
import glob, os
from shutil import copyfile, move
from subliminal import download_best_subtitles, save_subtitles, scan_videos
from babelfish import Language
#Vuse Torrent Default: userProf + "/Documents/Vuze Downloads"
#BiglyBT Torrent Default: userProf + "/Documents/BiglyBT Downloads"
__author__ = "TooSlepy"
__PyVer__ = "3.6"
userProf = os.getenv("userprofile")
videoSaveLoc = userProf + "/Documents/Vuze Downloads"
saveLoc = "D:/Moviex/"
lang = "eng"
os.chdir(videoSaveLoc)
for x in ("*.avi", "*.mkv", "*.mp4", "*.mpg"):
for file in glob.glob(x):
fileDir = file.rsplit('.', 1)[0]
if not os.path.exists(fileDir):
os.mkdir(fileDir)
move(file, fileDir + "/" + file)
# scan for videos in a folder
videos = scan_videos(fileDir)
# download best subtitles
subtitles = download_best_subtitles(videos, {Language(lang)})
# save them to disk, next to the video
for v in videos:
save_subtitles(v, subtitles[v])
move(fileDir, saveLoc + "/" + fileDir)
print(file + " has been moved and subtitled.")
def test_ne(self):
self.assertNotEqual(Language('fra'), Language('eng'))
self.assertIsNotNone(Language('fra'))
def test_nonzero(self):
self.assertFalse(bool(Language('und')))
self.assertTrue(bool(Language('eng')))
def generateOptions(self, inputfile, original=None):
# Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
# Video stream
self.log.info("Reading video stream.")
self.log.info("Video codec detected: %s." % info.video.codec)
try:
vbr = self.estimateVideoBitrate(info)
except:
vbr = info.format.bitrate / 1000
if info.video.codec.lower() in self.video_codec:
vcodec = 'copy'
else:
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate if self.video_bitrate else vbr
self.log.info("Pix Fmt: %s." % info.video.pix_fmt)
if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt:
self.log.debug(
"Overriding video pix_fmt. Codec cannot be copied because pix_fmt is not approved."
)
vcodec = self.video_codec[0]
pix_fmt = self.pix_fmt[0]
if self.video_profile:
vprofile = self.video_profile[0]
elif self.pix_fmt:
pix_fmt = self.pix_fmt[0]
else:
pix_fmt = None
if self.video_bitrate is not None and vbr > self.video_bitrate:
self.log.debug(
"Overriding video bitrate. Codec cannot be copied because video bitrate is too high."
)
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate
if self.video_width is not None and self.video_width < info.video.video_width:
self.log.debug(
"Video width is over the max width, it will be downsampled. Video stream can no longer be copied."
)
vcodec = self.video_codec[0]
vwidth = self.video_width
else:
vwidth = self.video_width
if '264' in info.video.codec.lower(
) and self.h264_level and info.video.video_level and (
info.video.video_level / 10 > self.h264_level):
self.log.info("Video level %0.1f." % (info.video.video_level / 10))
vcodec = self.video_codec[0]
self.log.debug("Video codec: %s." % vcodec)
self.log.debug("Video bitrate: %s." % vbitrate)
self.log.info("Profile: %s." % info.video.profile)
if self.video_profile and info.video.profile.lower().replace(
" ", "") not in self.video_profile:
self.log.debug(
"Video profile is not supported. Video stream can no longer be copied."
)
vcodec = self.video_codec[0]
vprofile = self.video_profile[0]
if self.pix_fmt:
pix_fmt = self.pix_fmt[0]
elif self.video_profile:
vprofile = self.video_profile[0]
else:
vprofile = None
# Audio streams
self.log.info("Reading audio streams.")
overrideLang = True
for a in info.audio:
try:
if a.metadata['language'].strip(
) == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
if (a.metadata['language'] == 'und' and self.adl) or (
self.awl and a.metadata['language'].lower() in self.awl):
overrideLang = False
break
if overrideLang:
self.awl = None
self.log.info(
"No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present."
)
audio_settings = {}
blocked_audio_languages = []
l = 0
for a in info.audio:
try:
if a.metadata['language'].strip(
) == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
self.log.info("Audio detected for stream #%s: %s [%s]." %
(a.index, a.codec, a.metadata['language']))
if self.output_extension in valid_tagging_extensions and a.codec.lower(
) == 'truehd' and self.ignore_truehd and len(
info.audio
) > 1: # Need to skip it early so that it flags the next track as default.
self.log.info(
"MP4 containers do not support truehd audio, and converting it is inconsistent due to video/audio sync issues. Skipping stream %s as typically the 2nd audio track is the AC3 core of the truehd stream."
% a.index)
continue
# Set undefined language to default language if specified
if self.adl is not None and a.metadata['language'] == 'und':
self.log.debug(
"Undefined language detected, defaulting to [%s]." %
self.adl)
a.metadata['language'] = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
iosdata = None
if self.awl is None or (a.metadata['language'].lower() in self.awl
and a.metadata['language'].lower()
not in blocked_audio_languages):
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS and a.audio_channels > 2:
iOSbitrate = 256 if (self.audio_bitrate * 2) > 256 else (
self.audio_bitrate * 2)
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug(
"Attempting to set ios stream bitrate based on source stream bitrate."
)
try:
iOSbitrate = (
(a.bitrate / 1000) / a.audio_channels) * 2
except:
self.log.warning(
"Unable to determine iOS audio bitrate from source stream %s, defaulting to 256 per channel."
% a.index)
iOSbitrate = audio_channels * 256
self.log.info(
"Creating audio stream %s from source audio stream %s [iOS-audio]."
% (str(l), a.index))
self.log.debug("Audio codec: %s." % self.iOS[0])
self.log.debug("Channels: 2.")
self.log.debug("Filter: %s." % self.iOS_filter)
self.log.debug("Bitrate: %s." % iOSbitrate)
self.log.debug("Language: %s." % a.metadata['language'])
iosdata = {
'map': a.index,
'codec': self.iOS[0],
'channels': 2,
'bitrate': iOSbitrate,
'filter': self.iOS_filter,
'language': a.metadata['language'],
'disposition': 'none',
}
if not self.iOSLast:
audio_settings.update({l: iosdata})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
self.log.info(
"Creating audio stream %s from source stream %s." %
(str(l), a.index))
if self.iOS and a.audio_channels <= 2:
self.log.debug(
"Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio]."
)
acodec = 'copy' if a.codec in self.iOS else self.iOS[0]
audio_channels = a.audio_channels
afilter = self.iOS_filter
abitrate = a.audio_channels * 128 if (
a.audio_channels * self.audio_bitrate) > (
a.audio_channels * 128) else (a.audio_channels *
self.audio_bitrate)
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = 'copy' if a.codec.lower(
) in self.audio_codec else self.audio_codec[0]
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == 'copy':
acodec = self.audio_codec[0]
abitrate = self.maxchannels * self.audio_bitrate
else:
audio_channels = a.audio_channels
abitrate = a.audio_channels * self.audio_bitrate
afilter = self.audio_filter
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug(
"Attempting to set bitrate based on source stream bitrate."
)
try:
abitrate = ((a.bitrate / 1000) /
a.audio_channels) * audio_channels
except:
self.log.warning(
"Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel."
% a.index)
abitrate = audio_channels * 256
self.log.debug("Audio codec: %s." % acodec)
self.log.debug("Channels: %s." % audio_channels)
self.log.debug("Bitrate: %s." % abitrate)
self.log.debug("Language: %s" % a.metadata['language'])
self.log.debug("Filter: %s" % afilter)
# If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed
if self.iOS and self.iOSFirst:
self.log.debug(
"Not creating any additional iOS audio streams.")
self.iOS = False
audio_settings.update({
l: {
'map': a.index,
'codec': acodec,
'channels': audio_channels,
'bitrate': abitrate,
'filter': afilter,
'language': a.metadata['language'],
'disposition': 'none',
}
})
if acodec == 'copy' and a.codec == 'aac' and self.aac_adtstoasc:
audio_settings[l]['bsf'] = 'aac_adtstoasc'
l += 1
# Add the iOS track last instead
if self.iOSLast and iosdata:
audio_settings.update({l: iosdata})
l += 1
if self.audio_copyoriginal and acodec != 'copy':
self.log.info(
"Adding copy of original audio track in format %s" %
a.codec)
audio_settings.update({
l: {
'map': a.index,
'codec': 'copy',
'language': a.metadata['language'],
'disposition': 'none',
}
})
# Remove the language if we only want the first track from a given language
if self.audio_first_language_track and self.awl:
try:
blocked_audio_languages.append(
a.metadata['language'].lower())
self.log.debug(
"Removing language from whitelist to prevent multiple tracks of the same: %s."
% a.metadata['language'])
except:
self.log.error(
"Unable to remove language %s from whitelist." %
a.metadata['language'])
# Audio Default
if len(audio_settings) > 0 and self.adl:
try:
default_track = [
x for x in audio_settings.values()
if x['language'] == self.adl
][0]
default_track['disposition'] = 'default'
except:
audio_settings[0]['disposition'] = 'default'
else:
self.log.error("Audio language array is empty.")
# Subtitle streams
subtitle_settings = {}
l = 0
self.log.info("Reading subtitle streams.")
for s in info.subtitle:
try:
if s.metadata['language'].strip(
) == "" or s.metadata['language'] is None:
s.metadata['language'] = 'und'
except KeyError:
s.metadata['language'] = 'und'
self.log.info("Subtitle detected for stream #%s: %s [%s]." %
(s.index, s.codec, s.metadata['language']))
# Set undefined language to default language if specified
if self.sdl is not None and s.metadata['language'] == 'und':
self.log.debug(
"Undefined language detected, defaulting to [%s]." %
self.sdl)
s.metadata['language'] = self.sdl
# Make sure its not an image based codec
if self.embedsubs and s.codec.lower(
) not in self.bad_internal_subtitle_codecs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.metadata['language'].lower(
) in self.swl:
subtitle_settings.update({
l: {
'map': s.index,
'codec': self.scodec[0],
'language': s.metadata['language'],
'encoding': self.subencoding,
'disposition': 'none',
# 'forced': s.sub_forced,
# 'default': s.sub_default
}
})
self.log.info(
"Creating subtitle stream %s from source stream %s." %
(l, s.index))
l = l + 1
elif not self.embedsubs and s.codec.lower(
) not in self.bad_external_subtitle_codecs:
if self.swl is None or s.metadata['language'].lower(
) in self.swl:
for codec in self.scodec:
ripsub = {
0: {
'map': s.index,
'codec': codec,
'language': s.metadata['language']
}
}
options = {
'format': codec,
'subtitle': ripsub,
}
try:
extension = subtitle_codec_extensions[codec]
except:
self.log.info(
"Wasn't able to determine subtitle file extension, defaulting to '.srt'."
)
extension = 'srt'
forced = ".forced" if s.sub_forced else ""
input_dir, filename, input_extension = self.parseFile(
inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(
output_dir, filename + "." +
s.metadata['language'] + forced + "." + extension)
i = 2
while os.path.isfile(outputfile):
self.log.debug(
"%s exists, appending %s to filename." %
(outputfile, i))
outputfile = os.path.join(
output_dir,
filename + "." + s.metadata['language'] +
forced + "." + str(i) + "." + extension)
i += 1
try:
self.log.info(
"Ripping %s subtitle from source stream %s into external file."
% (s.metadata['language'], s.index))
conv = Converter(self.FFMPEG_PATH,
self.FFPROBE_PATH).convert(
inputfile,
outputfile,
options,
timeout=None)
for timecode in conv:
pass
self.log.info("%s created." % outputfile)
except FFMpegConvertError:
self.log.error(
"Unabled to create external %s subtitle file for stream %s, may be an incompatible format."
% (extension, s.index))
self.removeFile(outputfile)
continue
except:
self.log.exception(
"Unabled to create external subtitle file for stream %s."
% (s.index))
try:
os.chmod(outputfile, self.permissions
) # Set permissions of newly created file
except:
self.log.exception(
"Unable to set new file permissions.")
# Attempt to download subtitles if they are missing using subliminal
languages = set()
try:
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
self.log.error(
"No valid subtitle language specified, cannot download subtitles."
)
except:
self.log.exception(
"Unable to verify subtitle languages for download.")
self.downloadsubs = False
if self.downloadsubs:
import subliminal
self.log.info("Attempting to download subtitles.")
# Attempt to set the dogpile cache
try:
subliminal.region.configure('dogpile.cache.memory')
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile),
subtitles=True,
embedded_subtitles=True)
subtitles = subliminal.download_best_subtitles(
[video],
languages,
hearing_impaired=False,
providers=self.subproviders)
try:
subliminal.save_subtitles(video, subtitles[video])
except:
# Support for older versions of subliminal
subliminal.save_subtitles(subtitles)
self.log.info(
"Please update to the latest version of subliminal.")
except Exception as e:
self.log.info("Unable to download subtitles.", exc_info=True)
self.log.debug("Unable to download subtitles.", exc_info=True)
# External subtitle import
if self.embedsubs and not self.embedonlyinternalsubs: # Don't bother if we're not embeddeding subtitles and external subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
self.log.info(
"External %s subtitle file detected." % lang)
if self.swl is None or lang in self.swl:
self.log.info(
"Creating subtitle stream %s by importing %s."
% (l, fname))
subtitle_settings.update({
l: {
'path': os.path.join(dirName, fname),
'source': src,
'map': 0,
'codec': 'mov_text',
'disposition': 'none',
'language': lang
}
})
self.log.debug("Path: %s." %
os.path.join(dirName, fname))
self.log.debug("Source: %s." % src)
self.log.debug("Codec: mov_text.")
self.log.debug("Langauge: %s." % lang)
l = l + 1
src = src + 1
self.deletesubs.add(
os.path.join(dirName, fname))
else:
self.log.info(
"Ignoring %s external subtitle stream due to language %s."
% (fname, lang))
# Subtitle Default
if len(subtitle_settings) > 0 and self.sdl:
try:
default_track = [
x for x in subtitle_settings.values()
if x['language'] == self.sdl
][0]
default_track['disposition'] = 'default'
except:
subtitle_settings[0]['disposition'] = 'default'
else:
self.log.warning("Subtitle stream array is empty.")
# Collect all options
options = {
'format': self.output_format,
'video': {
'codec': vcodec,
'map': info.video.index,
'bitrate': vbitrate,
'level': self.h264_level,
'profile': vprofile,
'pix_fmt': pix_fmt
},
'audio': audio_settings,
'subtitle': subtitle_settings,
'preopts': [],
'postopts': ['-threads', self.threads]
}
# If a CRF option is set, override the determine bitrate
if self.vcrf:
del options['video']['bitrate']
options['video']['crf'] = self.vcrf
if len(options['subtitle']) > 0:
options['preopts'].append('-fix_sub_duration')
if self.preopts:
options['preopts'].extend(self.preopts)
if self.postopts:
options['postopts'].extend(self.postopts)
if self.dxva2_decoder: # DXVA2 will fallback to CPU decoding when it hits a file that it cannot handle, so we don't need to check if the file is supported.
options['preopts'].extend(['-hwaccel', 'dxva2'])
elif info.video.codec.lower() == "hevc" and self.hevc_qsv_decoder:
options['preopts'].extend(['-vcodec', 'hevc_qsv'])
elif vcodec == "h264qsv" and info.video.codec.lower(
) == "h264" and self.qsv_decoder and (info.video.video_level / 10) < 5:
options['preopts'].extend(['-vcodec', 'h264_qsv'])
# Add width option
if vwidth:
options['video']['width'] = vwidth
# HEVC Tagging for copied streams
if info.video.codec.lower() in ['x265', 'h265', 'hevc'
] and vcodec == 'copy':
options['postopts'].extend(['-tag:v', 'hvc1'])
self.log.info("Tagging copied video stream as hvc1")
self.options = options
return options
def test_language_hasattr(self):
self.assertTrue(hasattr(Language('fra'), 'alpha3'))
self.assertTrue(hasattr(Language('fra'), 'alpha2'))
self.assertFalse(hasattr(Language('bej'), 'alpha2'))
def query(self, title, season=None, episode=None):
# search for the url title
url_titles = self._search_url_titles(title)
# episode
if season and episode:
if 'series' not in url_titles:
logger.error('No URL title found for series %r', title)
return []
url_title = url_titles['series'][0]
logger.debug('Using series title %r', url_title)
url = self.server_url + 'cinemast/data/series/sb/{}/{}/{}/'.format(
url_title, season, episode)
page_link = self.server_url + 'subtitle/series/{}/{}/{}/'.format(
url_title, season, episode)
else:
if 'movie' not in url_titles:
logger.error('No URL title found for movie %r', title)
return []
url_title = url_titles['movie'][0]
logger.debug('Using movie title %r', url_title)
url = self.server_url + 'cinemast/data/movie/sb/{}/'.format(
url_title)
page_link = self.server_url + 'subtitle/movie/{}/'.format(
url_title)
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
results = json.loads(r.text)
# loop over results
subtitles = {}
for language_code, language_data in results.items():
for quality_data in language_data.values():
for quality, subtitles_data in quality_data.items():
for subtitle_item in subtitles_data.values():
# read the item
language = Language.fromalpha2(language_code)
hearing_impaired = bool(
subtitle_item['hearing_impaired'])
subtitle_id = subtitle_item['id']
subtitle_key = subtitle_item['key']
downloaded = subtitle_item['downloaded']
release = subtitle_item['subtitle_version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug(
'Found additional release %r for subtitle %d',
release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases,
release) # deterministic order
subtitles[subtitle_id].downloaded += downloaded
continue
# otherwise create it
subtitle = SubsCenterSubtitle(
language, hearing_impaired, page_link, title,
season, episode, title, subtitle_id, subtitle_key,
downloaded, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def test_country(self):
self.assertEqual(Language('por', 'BR').country, Country('BR'))
self.assertEqual(Language('eng', Country('US')).country, Country('US'))
def test_converter_scope(self):
self.assertEqual(language_converters['scope'].codes,
set(['I', 'S', 'M']))
self.assertEqual(Language('eng').scope, 'individual')
self.assertEqual(Language('und').scope, 'special')
def on_task_output(self, task, config):
"""
Configuration::
subliminal:
languages: List of languages (as IETF codes) in order of preference. At least one is required.
alternatives: List of second-choice languages; subs will be downloaded but entries rejected.
exact_match: Use file hash only to search for subs, otherwise Subliminal will try to guess by filename.
providers: List of providers from where to download subtitles.
single: Download subtitles in single mode (no language code added to subtitle filename).
"""
if not task.accepted:
log.debug('nothing accepted, aborting')
return
from babelfish import Language
from dogpile.cache.exception import RegionAlreadyConfigured
import subliminal
try:
subliminal.region.configure('dogpile.cache.dbm',
arguments={
'filename':
os.path.join(
tempfile.gettempdir(),
'cachefile.dbm'),
'lock_factory':
subliminal.cli.MutexLock
})
except RegionAlreadyConfigured:
pass
logging.getLogger("subliminal").setLevel(logging.CRITICAL)
logging.getLogger("enzyme").setLevel(logging.WARNING)
langs = set(
[Language.fromietf(s) for s in config.get('languages', [])])
alts = set(
[Language.fromietf(s) for s in config.get('alternatives', [])])
# keep all downloaded subtitles and save to disk when done (no need to write every time)
downloaded_subtitles = collections.defaultdict(list)
providers_list = config.get('providers', None)
# test if only one language was provided, if so we will download in single mode
# (aka no language code added to subtitle filename)
# unless we are forced not to by configuration
# if we pass 'yes' for single in configuration but choose more than one language
# we ignore the configuration and add the language code to the
# potentially downloaded files
single_mode = config.get('single', '') and len(langs | alts) <= 1
for entry in task.accepted:
if 'location' not in entry:
log.warning(
'Cannot act on entries that do not represent a local file.'
)
elif not os.path.exists(entry['location']):
entry.fail('file not found: %s' % entry['location'])
elif '$RECYCLE.BIN' not in entry[
'location']: # ignore deleted files in Windows shares
try:
entry_langs = entry.get('subtitle_languages', [])
if not entry_langs:
entry_langs = langs
video = subliminal.scan_video(entry['location'])
if isinstance(video, subliminal.Episode):
title = video.series
else:
title = video.title
log.info('Name computed for %s was %s' %
(entry['location'], title))
msc = video.scores['hash'] if config['exact_match'] else 0
if entry_langs & video.subtitle_languages:
entry['subtitles_missing'] = set()
continue # subs for preferred lang(s) already exists
else:
subtitle = subliminal.download_best_subtitles(
[video],
entry_langs,
providers=providers_list,
min_score=msc)
if subtitle:
downloaded_subtitles.update(subtitle)
log.info('Subtitles found for %s' %
entry['location'])
else:
# TODO check performance hit -- this explicit check may be better on slower devices
# but subliminal already handles it for us, but it loops over all providers before stopping
remaining_alts = alts - video.subtitle_languages
if remaining_alts:
# only try to download for alternatives that aren't alread downloaded
subtitle = subliminal.download_best_subtitles(
[video],
remaining_alts,
providers=providers_list,
min_score=msc)
# this potentially just checks an already checked assignment bleh
if subtitle:
downloaded_subtitles.update(subtitle)
entry.fail(
'subtitles found for a second-choice language.'
)
else:
entry.fail(
'cannot find any subtitles for now.')
downloaded_languages = set([
Language.fromietf(unicode(l.language))
for l in subtitle[video]
])
if entry_langs:
entry[
'subtitles_missing'] = entry_langs - downloaded_languages
except Exception as err:
# don't want to abort the entire task for errors in a
# single video file or for occasional network timeouts
if err.args:
msg = unicode(err.args[0])
else:
# Subliminal errors don't always have a message, just use the name
msg = 'subliminal error: %s' % err.__class__.__name__
log.debug(msg)
entry.fail(msg)
if downloaded_subtitles:
# save subtitles to disk
for k, v in downloaded_subtitles.iteritems():
if v:
subliminal.save_subtitles(k, v, single=single_mode)
