def test_converter_alpha2(self):
self.assertEqual(Language('eng').alpha2, 'en')
self.assertEqual(Language.fromalpha2('en'), Language('eng'))
self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
self.assertRaises(LanguageReverseError, lambda: Language.fromalpha2('zz'))
self.assertRaises(LanguageConvertError, lambda: Language('aaa').alpha2)
self.assertEqual(len(language_converters['alpha2'].codes), 184)
def get_subtitle_list(movie_title, source_language_code,
destination_language_code):
video = Video.fromname(movie_title)
return list_subtitles(
[video], {
Language.fromalpha2(source_language_code),
Language.fromalpha2(destination_language_code)
}, FreePool)[video]
def test_converter_alpha2(self):
self.assertEqual(Language('eng').alpha2, 'en')
self.assertEqual(Language.fromalpha2('en'), Language('eng'))
self.assertEqual(Language.fromcode('en', 'alpha2'), Language('eng'))
with self.assertRaises(LanguageReverseError):
Language.fromalpha2('zz')
with self.assertRaises(LanguageConvertError):
Language('aaa').alpha2
self.assertEqual(len(get_language_converter('alpha2').codes), 184)
def query(self, searchword=None):
params = {"searchword": searchword}
logger.info("Searching subtitles %r", params)
r = self.session.get("http://sub.makedie.me/sub/?", params=params, timeout=10)
r.raise_for_status()
# loop over
subtitles = []
try:
# BeautifulSoup gives you Unicode, damn it
soup = BeautifulSoup(r.content, "html5lib")
results = soup.find_all("div", attrs={"class": "subitem"})
for it in results:
releases = it.find("a", attrs={"class": "introtitle"})["title"].strip().split("/")
sid = re.search("/xml/sub/\d+/(\d+).xml", it.find("a", attrs={"class": "introtitle"})["href"]).group(1)
download_button = it.find("a", attrs={"id": "downsubbtn"})["onclick"].strip()
if re.search("location.href\s*=\s*", download_button):
link = "http://sub.makedie.me/" + re.search(
"location.href\s*=\s*['\"](.*?)['\"]", download_button
).group(1)
else:
logger.debug("No download link found")
continue
lang = ""
for li in it.find_all("li"):
if re.search("格式：", li.text.strip()):
subtype = re.search("格式：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
if re.search("语言：", li.text.strip()):
lang = re.search("语言：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
if re.search("下载次数：", li.text.strip()):
downloads = re.search("下载次数：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
downloads = re.search("(\d+).*", downloads).group(1)
if re.search("日期：", li.text.strip()):
upload_date = re.search("日期：\s*([^\(]+)(?:\(\?\))*", li.text.strip()).group(1)
if "简" in lang or "繁" in lang or "双语" in lang:
subtitle = MakeDieSubtitle(
Language.fromalpha2("zh"), releases, sid, link, subtype, downloads, upload_date
)
else:
subtitle = MakeDieSubtitle(
Language.fromalpha2("en"), releases, sid, link, subtype, downloads, upload_date
)
logger.debug("Found subtitle %r", subtitle)
subtitles.append(subtitle)
return subtitles
except:
logger.debug("No subtitle found")
return []
def checkLanguage(self, language):
if not language:
return None
if len(language) < 2:
self.log.error("Unable to set tag language [tag-language].")
return None
try:
from babelfish import Language
except:
self.log.exception(
"Unable to important Language from babelfish [tag-language].")
return None
if len(language) == 2:
try:
return Language.fromalpha2(language).alpha3
self.log.exception(
"Unable to set tag language [tag-language].")
except:
return None
try:
return Language(language).alpha3
except:
self.log.exception("Unable to set tag language [tag-language].")
return None
def getAlpha3TCode(code): # We need to make sure that language codes are alpha3T
"""
:param code: Alpha2, Alpha3, or Alpha3b code.
:type code: C{str}
:return: Alpha3t language code (ISO 639-2/T) as C{str}
"""
lang = 'und'
code = code.strip().lower()
if len(code) == 3:
try:
lang = Language(code).alpha3t
except:
try:
lang = Language.fromalpha3b(code).alpha3t
except:
try:
lang = Language.fromalpha3t(code).alpha3t
except:
pass
elif len(code) == 2:
lang = Language.fromalpha2(code).alpha3t
return lang
def query(self,
title,
season=None,
episode=None,
year=None,
filename=None,
imdb_id=None):
# search for the IMDB ID if needed.
is_movie = not (season and episode)
imdb_id = imdb_id or self._search_imdb_id(title, year, is_movie)
if not imdb_id:
return {}
# search
logger.debug('Using IMDB ID %r', imdb_id)
url = 'https://json.{}/{}.json'.format(self.server_url, imdb_id)
page_link = 'https://{}/#/{}/{}'.format(
self.server_url, 'movies' if is_movie else 'series', imdb_id)
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
try:
results = r.json()
except ValueError:
return {}
# filter irrelevant results
if not is_movie:
results = results.get('subs', {}).get(str(season),
{}).get(str(episode), [])
else:
results = results.get('subs', [])
# loop over results
subtitles = {}
for result in results:
language = Language.fromalpha2('he')
hearing_impaired = False
subtitle_id = result['id']
release = result['version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug('Found additional release %r for subtitle %d',
release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases,
release) # deterministic order
subtitles[subtitle_id].downloaded += 1
continue
# otherwise create it
subtitle = WizdomSubtitle(language, hearing_impaired, page_link,
title, season, episode, title, imdb_id,
subtitle_id, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def getAlpha3TCode(code): # We need to make sure that language codes are alpha3T
"""
:param code: Alpha2, Alpha3, or Alpha3b code.
:type code: C{str}
:return: Alpha3t language code (ISO 639-2/T) as C{str}
"""
lang = 'und'
code = code.strip().lower()
if len(code) == 3:
try:
lang = Language(code).alpha3t
except:
try:
lang = Language.fromalpha3b(code).alpha3t
except:
try:
lang = Language.fromalpha3t(code).alpha3t
except:
pass
elif len(code) == 2:
lang = Language.fromalpha2(code).alpha3t
return lang
def test_get_matches_episode(episodes):
subtitle = ArgenteamSubtitle(Language.fromalpha2('es'), None,
'Game of Thrones', 3, 10, 'EVOLVE', '720p')
matches = subtitle.get_matches(episodes['got_s03e10'])
assert matches == {
'resolution', 'series', 'season', 'episode', 'year', 'country'
}
class TheSubDBProvider(Provider):
languages = {
Language.fromalpha2(l)
for l in ['en', 'es', 'fr', 'it', 'nl', 'pl', 'pt', 'ro', 'sv', 'tr']
}
required_hash = 'thesubdb'
server_url = 'http://api.thesubdb.com/'
def initialize(self):
self.session = Session()
self.session.headers = {
'User-Agent':
'SubDB/1.0 (subliminal/%s; https://github.com/Diaoul/subliminal)' %
get_version(__version__)
}
def terminate(self):
self.session.close()
def query(self, hash):
# make the query
params = {'action': 'search', 'hash': hash}
logger.info('Searching subtitles %r', params)
r = self.session.get(self.server_url, params=params, timeout=10)
# handle subtitles not found and errors
if r.status_code == 404:
logger.debug('No subtitles found')
return []
r.raise_for_status()
# loop over languages
subtitles = []
for language_code in r.text.split(','):
language = Language.fromalpha2(language_code)
subtitle = TheSubDBSubtitle(language, hash)
logger.info('Found subtitle %r', subtitle)
subtitles.append(subtitle)
return subtitles
def list_subtitles(self, video, languages):
return [
s for s in self.query(video.hashes['thesubdb'])
if s.language in languages
]
def download_subtitle(self, subtitle):
logger.info('Downloading subtitle %r')
params = {
'action': 'download',
'hash': subtitle.hash,
'language': subtitle.language.alpha2
}
r = self.session.get(self.server_url, params=params, timeout=10)
r.raise_for_status()
subtitle.content = fix_line_ending(r.content)
def test_download_subtitle(episodes):
video = episodes['bbt_s07e05']
languages = {Language.fromalpha2('es')}
with ArgenteamProvider() as provider:
subtitles = provider.list_subtitles(video, languages)
provider.download_subtitle(subtitles[0])
assert subtitles[0].content is not None
assert subtitles[0].is_valid() is True
def query(self, title, season=None, episode=None):
# search for the url title
url_titles = self._search_url_titles(title)
# episode
if season and episode:
if 'series' not in url_titles:
logger.error('No URL title found for series %r', title)
return []
url_title = url_titles['series'][0]
logger.debug('Using series title %r', url_title)
url = self.server_url + 'cst/data/series/sb/{}/{}/{}/'.format(url_title, season, episode)
page_link = self.server_url + 'subtitle/series/{}/{}/{}/'.format(url_title, season, episode)
else:
if 'movie' not in url_titles:
logger.error('No URL title found for movie %r', title)
return []
url_title = url_titles['movie'][0]
logger.debug('Using movie title %r', url_title)
url = self.server_url + 'cst/data/movie/sb/{}/'.format(url_title)
page_link = self.server_url + 'subtitle/movie/{}/'.format(url_title)
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
results = json.loads(r.text)
# loop over results
subtitles = {}
for language_code, language_data in results.items():
for quality_data in language_data.values():
for quality, subtitles_data in quality_data.items():
for subtitle_item in subtitles_data.values():
# read the item
language = Language.fromalpha2(language_code)
hearing_impaired = bool(subtitle_item['hearing_impaired'])
subtitle_id = subtitle_item['id']
subtitle_key = subtitle_item['key']
subtitle_version = subtitle_item['h_version']
downloaded = subtitle_item['downloaded']
release = subtitle_item['subtitle_version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug('Found additional release %r for subtitle %d', release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases, release) # deterministic order
subtitles[subtitle_id].downloaded += downloaded
continue
# otherwise create it
subtitle = SubsCenterSubtitle(language, hearing_impaired, page_link, title, season, episode,
title, subtitle_id, subtitle_key, subtitle_version, downloaded,
[release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def query(self, title, season=None, episode=None):
# search for the url title
url_titles = self._search_url_titles(title)
# episode
if season and episode:
if 'series' not in url_titles:
logger.error('No URL title found for series %r', title)
return []
url_title = url_titles['series'][0]
logger.debug('Using series title %r', url_title)
url = self.server_url + 'cst/data/series/sb/{}/{}/{}/'.format(url_title, season, episode)
page_link = self.server_url + 'subtitle/series/{}/{}/{}/'.format(url_title, season, episode)
else:
if 'movie' not in url_titles:
logger.error('No URL title found for movie %r', title)
return []
url_title = url_titles['movie'][0]
logger.debug('Using movie title %r', url_title)
url = self.server_url + 'cst/data/movie/sb/{}/'.format(url_title)
page_link = self.server_url + 'subtitle/movie/{}/'.format(url_title)
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
results = json.loads(r.text)
# loop over results
subtitles = {}
for language_code, language_data in results.items():
for quality_data in language_data.values():
for quality, subtitles_data in quality_data.items():
for subtitle_item in subtitles_data.values():
# read the item
language = Language.fromalpha2(language_code)
hearing_impaired = bool(subtitle_item['hearing_impaired'])
subtitle_id = subtitle_item['id']
subtitle_key = subtitle_item['key']
subtitle_version = subtitle_item['h_version']
downloaded = subtitle_item['downloaded']
release = subtitle_item['subtitle_version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug('Found additional release %r for subtitle %d', release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases, release) # deterministic order
subtitles[subtitle_id].downloaded += downloaded
continue
# otherwise create it
subtitle = SubsCenterSubtitle(language, hearing_impaired, page_link, title, season, episode,
title, subtitle_id, subtitle_key, subtitle_version, downloaded,
[release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def query(self, series=None, season=None, episode=None, title=None):
# set the correct parameters depending on the kind
if series and season and episode:
url_series = self._search_url_title(series, 'series')
url = self.server_url + 'cinemast/data/series/sb/{}/{}/{}/'.format(
url_series, season, episode)
page_link = self.server_url + 'subtitle/series/{}/{}/{}/'.format(
url_series, season, episode)
elif title:
url_title = self._search_url_title(title, 'movie')
url = self.server_url + 'cinemast/data/movie/sb/{}/'.format(
url_title)
page_link = self.server_url + 'subtitle/movie/{}/'.format(
url_title)
else:
raise ValueError('One or more parameters are missing')
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
results = json.loads(r.text)
# loop over results
subtitles = {}
for language_code, language_data in results.items():
for quality_data in language_data.values():
for quality, subtitles_data in quality_data.items():
for subtitle_item in subtitles_data.values():
# read the item
language = Language.fromalpha2(language_code)
hearing_impaired = bool(
subtitle_item['hearing_impaired'])
subtitle_id = subtitle_item['id']
subtitle_key = subtitle_item['key']
downloaded = subtitle_item['downloaded']
release = subtitle_item['subtitle_version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug(
'Found additional release %r for subtitle %d',
release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases,
release) # deterministic order
subtitles[subtitle_id].downloaded += downloaded
continue
# otherwise create it
subtitle = SubsCenterSubtitle(
language, hearing_impaired, page_link, series,
season, episode, title, subtitle_id, subtitle_key,
downloaded, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
class NapiProjektProvider(Provider):
"""NapiProjekt Provider."""
languages = {Language.fromalpha2(l) for l in ['pl']}
required_hash = 'napiprojekt'
server_url = 'http://napiprojekt.pl/unit_napisy/dl.php'
subtitle_class = NapiProjektSubtitle
def __init__(self):
self.session = None
def initialize(self):
self.session = Session()
self.session.headers[
'User-Agent'] = 'Subliminal/%s' % __short_version__
def terminate(self):
self.session.close()
def query(self, language, hash):
params = {
'v': 'dreambox',
'kolejka': 'false',
'nick': '',
'pass': '',
'napios': 'Linux',
'l': language.alpha2.upper(),
'f': hash,
't': get_subhash(hash)
}
logger.info('Searching subtitle %r', params)
r = self.session.get(self.server_url, params=params, timeout=10)
r.raise_for_status()
# handle subtitles not found and errors
if r.content[:4] == b'NPc0':
logger.debug('No subtitles found')
return None
subtitle = self.subtitle_class(language, hash)
subtitle.content = r.content
logger.debug('Found subtitle %r', subtitle)
return subtitle
def list_subtitles(self, video, languages):
return [
s for s in
[self.query(l, video.hashes['napiprojekt']) for l in languages]
if s is not None
]
def download_subtitle(self, subtitle):
# there is no download step, content is already filled from listing subtitles
pass
def test_list_subtitles_episode_alternative_series(episodes):
video = episodes['turn_s04e03']
languages = {Language.fromalpha2('es')}
expected_subtitles = {
'http://www.argenteam.net/subtitles/67263/TURN.Washingtons.Spies.%282014%29.S04E03'
'-Blood.for.Blood.HDTV.x264-SVA',
'http://www.argenteam.net/subtitles/67264/TURN.Washingtons.Spies.%282014%29.S04E03'
'-Blood.for.Blood.HDTV.x264.720p-AVS'
}
with ArgenteamProvider() as provider:
subtitles = provider.list_subtitles(video, languages)
assert {s.id for s in subtitles} == expected_subtitles
def query(self, title, season=None, episode=None, year=None, filename=None, imdb_id=None):
# search for the IMDB ID if needed.
is_movie = not (season and episode)
imdb_id = imdb_id or self._search_imdb_id(title, year, is_movie)
if not imdb_id:
return {}
# search
logger.debug('Using IMDB ID %r', imdb_id)
url = 'http://json.{}/{}.json'.format(self.server_url, imdb_id)
page_link = 'http://{}/#/{}/{}'.format(self.server_url, 'movies' if is_movie else 'series', imdb_id)
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
try:
results = r.json()
except ValueError:
return {}
# filter irrelevant results
if not is_movie:
results = results.get('subs', {}).get(str(season), {}).get(str(episode), [])
else:
results = results.get('subs', [])
# loop over results
subtitles = {}
for result in results:
language = Language.fromalpha2('he')
hearing_impaired = False
subtitle_id = result['id']
release = result['version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug('Found additional release %r for subtitle %d', release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases, release) # deterministic order
subtitles[subtitle_id].downloaded += 1
continue
# otherwise create it
subtitle = WizdomSubtitle(language, hearing_impaired, page_link, title, season, episode, title, imdb_id,
subtitle_id, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def query(self, series=None, season=None, episode=None, title=None):
# set the correct parameters depending on the kind
if series and season and episode:
url_series = self._search_url_title(series, 'series')
url = self.server + 'cinemast/data/series/sb/{}/{}/{}/'.format(url_series, season, episode)
page_link = self.server + 'subtitle/series/{}/{}/{}/'.format(url_series, season, episode)
elif title:
url_title = self._search_url_title(title, 'movie')
url = self.server + 'cinemast/data/movie/sb/{}/'.format(url_title)
page_link = self.server + 'subtitle/movie/{}/'.format(url_title)
else:
raise ValueError('One or more parameters are missing')
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
results = json.loads(r.text)
# loop over results
subtitles = {}
for language_code, language_data in results.items():
for quality_data in language_data.values():
for quality, subtitles_data in quality_data.items():
for subtitle_item in subtitles_data.values():
# read the item
language = Language.fromalpha2(language_code)
hearing_impaired = bool(subtitle_item['hearing_impaired'])
subtitle_id = subtitle_item['id']
subtitle_key = subtitle_item['key']
downloaded = subtitle_item['downloaded']
release = subtitle_item['subtitle_version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug('Found additional release %r for subtitle %d', release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases, release) # deterministic order
subtitles[subtitle_id].downloaded += downloaded
continue
# otherwise create it
subtitle = SubsCenterSubtitle(language, hearing_impaired, page_link, series, season, episode,
title, subtitle_id, subtitle_key, downloaded, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def query(self, hash):
# make the query
params = {'action': 'search', 'hash': hash}
logger.info('Searching subtitles %r', params)
r = self.session.get(self.server_url, params=params, timeout=10)
# handle subtitles not found and errors
if r.status_code == 404:
logger.debug('No subtitles found')
return []
r.raise_for_status()
# loop over languages
subtitles = []
for language_code in r.text.split(','):
language = Language.fromalpha2(language_code)
subtitle = TheSubDBSubtitle(language, hash)
logger.info('Found subtitle %r', subtitle)
subtitles.append(subtitle)
return subtitles
def query(self, hash):
# make the query
params = {'action': 'search', 'hash': hash}
logger.info('Searching subtitles %r', params)
r = self.session.get(self.server_url, params=params, timeout=10)
# handle subtitles not found and errors
if r.status_code == 404:
logger.debug('No subtitles found')
return []
r.raise_for_status()
# loop over languages
subtitles = []
for language_code in r.text.split(','):
language = Language.fromalpha2(language_code)
subtitle = TheSubDBSubtitle(language, hash)
logger.info('Found subtitle %r', subtitle)
subtitles.append(subtitle)
return subtitles
def getAlpha3TCode(code, default=None):
lang = default or UNDEFINED
if not code or code == UNDEFINED:
return lang
code = code.strip().lower().replace('.', '')
if len(code) == 3:
try:
lang = Language(code).alpha3t
except:
try:
lang = Language.fromalpha3b(code).alpha3t
except:
try:
lang = Language.fromalpha3t(code).alpha3t
except:
pass
elif len(code) == 2:
try:
lang = Language.fromalpha2(code).alpha3t
except:
pass
return lang
def query(self, hash):
# shooter has many DNS mirrors, e.g. splayer[1-9], but one is enough
params = {"pathinfo": "temp", "format": "json", "filehash": hash}
logger.info("Searching subtitles %r", params)
r = self.session.get("https://www.shooter.cn/api/subapi.php", params=params, timeout=10)
r.raise_for_status()
# loop over, server always returns found or not
subtitles = []
try:
for it in r.json():
# It normally contains one File, but can contain multiple
link = it["Files"][0]["Link"]
subtype = it["Files"][0]["Ext"]
subtitle = ShooterSubtitle(Language.fromalpha2("zh"), hash, link, subtype)
logger.debug("Found subtitle %r", subtitle)
subtitles.append(subtitle)
return subtitles
except:
logger.debug("No subtitle found")
return []
def build_commands(file, new_dir, movie_name, bitbucket):
if isinstance(file, string_types):
input_file = file
if 'concat:' in file:
file = file.split('|')[0].replace('concat:', '')
video_details, result = get_video_details(file)
directory, name = os.path.split(file)
name, ext = os.path.splitext(name)
check = re.match('VTS_([0-9][0-9])_[0-9]+', name)
if check and core.CONCAT:
name = movie_name
elif check:
name = ('{0}.cd{1}'.format(movie_name, check.groups()[0]))
elif core.CONCAT and re.match('(.+)[cC][dD][0-9]', name):
name = re.sub('([ ._=:-]+[cC][dD][0-9])', '', name)
if ext == core.VEXTENSION and new_dir == directory: # we need to change the name to prevent overwriting itself.
core.VEXTENSION = '-transcoded{ext}'.format(
ext=core.VEXTENSION) # adds '-transcoded.ext'
new_file = file
else:
img, data = next(iteritems(file))
name = data['name']
new_file = []
rem_vid = []
for vid in data['files']:
video_details, result = get_video_details(vid, img, bitbucket)
if not check_vid_file(
video_details, result
): #lets not transcode menu or other clips that don't have audio and video.
rem_vid.append(vid)
data['files'] = [f for f in data['files'] if f not in rem_vid]
new_file = {img: {'name': data['name'], 'files': data['files']}}
video_details, result = get_video_details(data['files'][0], img,
bitbucket)
input_file = '-'
file = '-'
newfile_path = os.path.normpath(
os.path.join(new_dir, name) + core.VEXTENSION)
map_cmd = []
video_cmd = []
audio_cmd = []
audio_cmd2 = []
sub_cmd = []
meta_cmd = []
other_cmd = []
if not video_details or not video_details.get(
'streams'
): # we couldn't read streams with ffprobe. Set defaults to try transcoding.
video_streams = []
audio_streams = []
sub_streams = []
map_cmd.extend(['-map', '0'])
if core.VCODEC:
video_cmd.extend(['-c:v', core.VCODEC])
if core.VCODEC == 'libx264' and core.VPRESET:
video_cmd.extend(['-pre', core.VPRESET])
else:
video_cmd.extend(['-c:v', 'copy'])
if core.VFRAMERATE:
video_cmd.extend(['-r', str(core.VFRAMERATE)])
if core.VBITRATE:
video_cmd.extend(['-b:v', str(core.VBITRATE)])
if core.VRESOLUTION:
video_cmd.extend(
['-vf', 'scale={vres}'.format(vres=core.VRESOLUTION)])
if core.VPRESET:
video_cmd.extend(['-preset', core.VPRESET])
if core.VCRF:
video_cmd.extend(['-crf', str(core.VCRF)])
if core.VLEVEL:
video_cmd.extend(['-level', str(core.VLEVEL)])
if core.ACODEC:
audio_cmd.extend(['-c:a', core.ACODEC])
if core.ACODEC in [
'aac', 'dts'
]: # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg
audio_cmd.extend(['-strict', '-2'])
else:
audio_cmd.extend(['-c:a', 'copy'])
if core.ACHANNELS:
audio_cmd.extend(['-ac', str(core.ACHANNELS)])
if core.ABITRATE:
audio_cmd.extend(['-b:a', str(core.ABITRATE)])
if core.OUTPUTQUALITYPERCENT:
audio_cmd.extend(['-q:a', str(core.OUTPUTQUALITYPERCENT)])
if core.SCODEC and core.ALLOWSUBS:
sub_cmd.extend(['-c:s', core.SCODEC])
elif core.ALLOWSUBS: # Not every subtitle codec can be used for every video container format!
sub_cmd.extend(['-c:s', 'copy'])
else: # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options
sub_cmd.extend(['-sn']) # Don't copy the subtitles over
if core.OUTPUTFASTSTART:
other_cmd.extend(['-movflags', '+faststart'])
else:
video_streams = [
item for item in video_details['streams']
if item['codec_type'] == 'video'
]
audio_streams = [
item for item in video_details['streams']
if item['codec_type'] == 'audio'
]
sub_streams = [
item for item in video_details['streams']
if item['codec_type'] == 'subtitle'
]
if core.VEXTENSION not in ['.mkv', '.mpegts']:
sub_streams = [
item for item in video_details['streams']
if item['codec_type'] == 'subtitle' and item['codec_name'] !=
'hdmv_pgs_subtitle' and item['codec_name'] != 'pgssub'
]
for video in video_streams:
codec = video['codec_name']
fr = video.get('avg_frame_rate', 0)
width = video.get('width', 0)
height = video.get('height', 0)
scale = core.VRESOLUTION
if codec in core.VCODEC_ALLOW or not core.VCODEC:
video_cmd.extend(['-c:v', 'copy'])
else:
video_cmd.extend(['-c:v', core.VCODEC])
if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <=
core.VFRAMERATE * 1.001):
video_cmd.extend(['-r', str(core.VFRAMERATE)])
if scale:
w_scale = width / float(scale.split(':')[0])
h_scale = height / float(scale.split(':')[1])
if w_scale > h_scale: # widescreen, Scale by width only.
scale = '{width}:{height}'.format(
width=scale.split(':')[0],
height=int((height / w_scale) / 2) * 2,
)
if w_scale > 1:
video_cmd.extend(
['-vf', 'scale={width}'.format(width=scale)])
else: # lower or matching ratio, scale by height only.
scale = '{width}:{height}'.format(
width=int((width / h_scale) / 2) * 2,
height=scale.split(':')[1],
)
if h_scale > 1:
video_cmd.extend(
['-vf', 'scale={height}'.format(height=scale)])
if core.VBITRATE:
video_cmd.extend(['-b:v', str(core.VBITRATE)])
if core.VPRESET:
video_cmd.extend(['-preset', core.VPRESET])
if core.VCRF:
video_cmd.extend(['-crf', str(core.VCRF)])
if core.VLEVEL:
video_cmd.extend(['-level', str(core.VLEVEL)])
no_copy = ['-vf', '-r', '-crf', '-level', '-preset', '-b:v']
if video_cmd[1] == 'copy' and any(i in video_cmd for i in no_copy):
video_cmd[1] = core.VCODEC
if core.VCODEC == 'copy': # force copy. therefore ignore all other video transcoding.
video_cmd = ['-c:v', 'copy']
map_cmd.extend(['-map', '0:{index}'.format(index=video['index'])])
break # Only one video needed
used_audio = 0
a_mapped = []
commentary = []
if audio_streams:
for i, val in reversed(list(enumerate(audio_streams))):
try:
if 'Commentary' in val.get('tags').get(
'title'): # Split out commentry tracks.
commentary.append(val)
del audio_streams[i]
except Exception:
continue
try:
audio1 = [
item for item in audio_streams
if item['tags']['language'] == core.ALANGUAGE
]
except Exception: # no language tags. Assume only 1 language.
audio1 = audio_streams
try:
audio2 = [
item for item in audio1
if item['codec_name'] in core.ACODEC_ALLOW
]
except Exception:
audio2 = []
try:
audio3 = [
item for item in audio_streams
if item['tags']['language'] != core.ALANGUAGE
]
except Exception:
audio3 = []
try:
audio4 = [
item for item in audio3
if item['codec_name'] in core.ACODEC_ALLOW
]
except Exception:
audio4 = []
if audio2: # right (or only) language and codec...
map_cmd.extend(
['-map', '0:{index}'.format(index=audio2[0]['index'])])
a_mapped.extend([audio2[0]['index']])
bitrate = int(float(audio2[0].get('bit_rate', 0))) / 1000
channels = int(float(audio2[0].get('channels', 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio1: # right (or only) language, wrong codec.
map_cmd.extend(
['-map', '0:{index}'.format(index=audio1[0]['index'])])
a_mapped.extend([audio1[0]['index']])
bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000
channels = int(float(audio1[0].get('channels', 0)))
audio_cmd.extend([
'-c:a:{0}'.format(used_audio),
core.ACODEC if core.ACODEC else 'copy'
])
elif audio4: # wrong language, right codec.
map_cmd.extend(
['-map', '0:{index}'.format(index=audio4[0]['index'])])
a_mapped.extend([audio4[0]['index']])
bitrate = int(float(audio4[0].get('bit_rate', 0))) / 1000
channels = int(float(audio4[0].get('channels', 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio3: # wrong language, wrong codec. just pick the default audio track
map_cmd.extend(
['-map', '0:{index}'.format(index=audio3[0]['index'])])
a_mapped.extend([audio3[0]['index']])
bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000
channels = int(float(audio3[0].get('channels', 0)))
audio_cmd.extend([
'-c:a:{0}'.format(used_audio),
core.ACODEC if core.ACODEC else 'copy'
])
if core.ACHANNELS and channels and channels > core.ACHANNELS:
audio_cmd.extend(
['-ac:a:{0}'.format(used_audio),
str(core.ACHANNELS)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate <
core.ABITRATE * 1.1):
audio_cmd.extend(
['-b:a:{0}'.format(used_audio),
str(core.ABITRATE)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if core.OUTPUTQUALITYPERCENT:
audio_cmd.extend([
'-q:a:{0}'.format(used_audio),
str(core.OUTPUTQUALITYPERCENT)
])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if audio_cmd[1] in ['aac', 'dts']:
audio_cmd[2:2] = ['-strict', '-2']
if core.ACODEC2_ALLOW:
used_audio += 1
try:
audio5 = [
item for item in audio1
if item['codec_name'] in core.ACODEC2_ALLOW
]
except Exception:
audio5 = []
try:
audio6 = [
item for item in audio3
if item['codec_name'] in core.ACODEC2_ALLOW
]
except Exception:
audio6 = []
if audio5: # right language and codec.
map_cmd.extend(
['-map', '0:{index}'.format(index=audio5[0]['index'])])
a_mapped.extend([audio5[0]['index']])
bitrate = int(float(audio5[0].get('bit_rate', 0))) / 1000
channels = int(float(audio5[0].get('channels', 0)))
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio1: # right language wrong codec.
map_cmd.extend(
['-map', '0:{index}'.format(index=audio1[0]['index'])])
a_mapped.extend([audio1[0]['index']])
bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000
channels = int(float(audio1[0].get('channels', 0)))
if core.ACODEC2:
audio_cmd2.extend(
['-c:a:{0}'.format(used_audio), core.ACODEC2])
else:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio6: # wrong language, right codec
map_cmd.extend(
['-map', '0:{index}'.format(index=audio6[0]['index'])])
a_mapped.extend([audio6[0]['index']])
bitrate = int(float(audio6[0].get('bit_rate', 0))) / 1000
channels = int(float(audio6[0].get('channels', 0)))
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio3: # wrong language, wrong codec just pick the default audio track
map_cmd.extend(
['-map', '0:{index}'.format(index=audio3[0]['index'])])
a_mapped.extend([audio3[0]['index']])
bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000
channels = int(float(audio3[0].get('channels', 0)))
if core.ACODEC2:
audio_cmd2.extend(
['-c:a:{0}'.format(used_audio), core.ACODEC2])
else:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
if core.ACHANNELS2 and channels and channels > core.ACHANNELS2:
audio_cmd2.extend(
['-ac:a:{0}'.format(used_audio),
str(core.ACHANNELS2)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate <
core.ABITRATE2 * 1.1):
audio_cmd2.extend(
['-b:a:{0}'.format(used_audio),
str(core.ABITRATE2)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if core.OUTPUTQUALITYPERCENT:
audio_cmd2.extend([
'-q:a:{0}'.format(used_audio),
str(core.OUTPUTQUALITYPERCENT)
])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if audio_cmd2[1] in ['aac', 'dts']:
audio_cmd2[2:2] = ['-strict', '-2']
if a_mapped[1] == a_mapped[0] and audio_cmd2[1:] == audio_cmd[
1:]: # check for duplicate output track.
del map_cmd[-2:]
else:
audio_cmd.extend(audio_cmd2)
if core.AINCLUDE and core.ACODEC3:
audio_streams.extend(commentary) # add commentry tracks back here.
for audio in audio_streams:
if audio['index'] in a_mapped:
continue
used_audio += 1
map_cmd.extend(
['-map', '0:{index}'.format(index=audio['index'])])
audio_cmd3 = []
bitrate = int(float(audio.get('bit_rate', 0))) / 1000
channels = int(float(audio.get('channels', 0)))
if audio['codec_name'] in core.ACODEC3_ALLOW:
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])
else:
if core.ACODEC3:
audio_cmd3.extend(
['-c:a:{0}'.format(used_audio), core.ACODEC3])
else:
audio_cmd3.extend(
['-c:a:{0}'.format(used_audio), 'copy'])
if core.ACHANNELS3 and channels and channels > core.ACHANNELS3:
audio_cmd3.extend(
['-ac:a:{0}'.format(used_audio),
str(core.ACHANNELS3)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate <
core.ABITRATE3 * 1.1):
audio_cmd3.extend(
['-b:a:{0}'.format(used_audio),
str(core.ABITRATE3)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if core.OUTPUTQUALITYPERCENT > 0:
audio_cmd3.extend([
'-q:a:{0}'.format(used_audio),
str(core.OUTPUTQUALITYPERCENT)
])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if audio_cmd3[1] in ['aac', 'dts']:
audio_cmd3[2:2] = ['-strict', '-2']
audio_cmd.extend(audio_cmd3)
s_mapped = []
burnt = 0
n = 0
for lan in core.SLANGUAGES:
try:
subs1 = [
item for item in sub_streams if item['tags']['language'] == lan
]
except Exception:
subs1 = []
if core.BURN and not subs1 and not burnt and os.path.isfile(file):
for subfile in get_subs(file):
if lan in os.path.split(subfile)[1]:
video_cmd.extend(
['-vf', 'subtitles={subs}'.format(subs=subfile)])
burnt = 1
for sub in subs1:
if core.BURN and not burnt and os.path.isfile(input_file):
subloc = 0
for index in range(len(sub_streams)):
if sub_streams[index]['index'] == sub['index']:
subloc = index
break
video_cmd.extend([
'-vf', 'subtitles={sub}:si={loc}'.format(sub=input_file,
loc=subloc)
])
burnt = 1
if not core.ALLOWSUBS:
break
if sub['codec_name'] in [
'dvd_subtitle', 'VobSub'
] and core.SCODEC == 'mov_text': # We can't convert these.
continue
map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])])
s_mapped.extend([sub['index']])
if core.SINCLUDE:
for sub in sub_streams:
if not core.ALLOWSUBS:
break
if sub['index'] in s_mapped:
continue
if sub['codec_name'] in [
'dvd_subtitle', 'VobSub'
] and core.SCODEC == 'mov_text': # We can't convert these.
continue
map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])])
s_mapped.extend([sub['index']])
if core.OUTPUTFASTSTART:
other_cmd.extend(['-movflags', '+faststart'])
if core.OTHEROPTS:
other_cmd.extend(core.OTHEROPTS)
command = [core.FFMPEG, '-loglevel', 'warning']
if core.HWACCEL:
command.extend(['-hwaccel', 'auto'])
if core.GENERALOPTS:
command.extend(core.GENERALOPTS)
command.extend(['-i', input_file])
if core.SEMBED and os.path.isfile(file):
for subfile in get_subs(file):
sub_details, result = get_video_details(subfile)
if not sub_details or not sub_details.get('streams'):
continue
if core.SCODEC == 'mov_text':
subcode = [
stream['codec_name'] for stream in sub_details['streams']
]
if set(subcode).intersection(['dvd_subtitle', 'VobSub'
]): # We can't convert these.
continue
command.extend(['-i', subfile])
lan = os.path.splitext(
os.path.splitext(subfile)[0])[1][1:].split('-')[0]
lan = text_type(lan)
metlan = None
try:
if len(lan) == 3:
metlan = Language(lan)
if len(lan) == 2:
metlan = Language.fromalpha2(lan)
except Exception:
pass
if metlan:
meta_cmd.extend([
'-metadata:s:s:{x}'.format(x=len(s_mapped) + n),
'language={lang}'.format(lang=metlan.alpha3)
])
n += 1
map_cmd.extend(['-map', '{x}:0'.format(x=n)])
if not core.ALLOWSUBS or (not s_mapped and not n):
sub_cmd.extend(['-sn'])
else:
if core.SCODEC:
sub_cmd.extend(['-c:s', core.SCODEC])
else:
sub_cmd.extend(['-c:s', 'copy'])
command.extend(map_cmd)
command.extend(video_cmd)
command.extend(audio_cmd)
command.extend(sub_cmd)
command.extend(meta_cmd)
command.extend(other_cmd)
command.append(newfile_path)
if platform.system() != 'Windows':
command = core.NICENESS + command
return command, new_file
class PodnapisiProvider(Provider):
"""Podnapisi Provider."""
languages = ({Language('por', 'BR'),
Language('srp', script='Latn')} | {
Language.fromalpha2(l)
for l in language_converters['alpha2'].codes
})
server_url = 'https://www.podnapisi.net/subtitles/'
subtitle_class = PodnapisiSubtitle
def __init__(self):
self.session = None
def initialize(self):
self.session = Session()
self.session.mount('https://', SecLevelOneTLSAdapter())
self.session.headers['User-Agent'] = self.user_agent
self.session.headers['Accept'] = 'application/json'
def terminate(self):
self.session.close()
def query(self, language, keyword, season=None, episode=None, year=None):
# set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652
params = {'keywords': keyword, 'language': str(language)}
is_episode = False
if season is not None and episode:
is_episode = True
params['seasons'] = season
params['episodes'] = episode
params['movie_type'] = ['tv-series', 'mini-series']
else:
params['movie_type'] = 'movie'
if year:
params['year'] = year
# loop over paginated results
logger.info('Searching subtitles %r', params)
subtitles = []
pids = set()
while True:
# query the server
r = self.session.get(self.server_url + 'search/advanced',
params=params,
timeout=10)
r.raise_for_status()
result = json.loads(r.text)
# loop over subtitles
for data in result['data']:
# read xml elements
pid = data['id']
# ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321
if pid in pids:
logger.debug('Ignoring duplicate %r', pid)
continue
if is_episode and data['movie']['type'] == 'movie':
logger.error('Wrong type detected: movie for episode')
continue
language = Language.fromietf(data['language'])
hearing_impaired = 'hearing_impaired' in data['flags']
page_link = data['url']
releases = data['releases'] + data['custom_releases']
title = data['movie']['title']
season = int(data['movie']['episode_info'].get(
'season')) if is_episode else None
episode = int(data['movie']['episode_info'].get(
'episode')) if is_episode else None
year = int(data['movie']['year'])
if is_episode:
subtitle = self.subtitle_class(language,
hearing_impaired,
page_link,
pid,
releases,
title,
season=season,
episode=episode,
year=year)
else:
subtitle = self.subtitle_class(language,
hearing_impaired,
page_link,
pid,
releases,
title,
year=year)
logger.debug('Found subtitle %r', subtitle)
subtitles.append(subtitle)
pids.add(pid)
# stop on last page
if int(result['page']) >= int(result['all_pages']):
break
# increment current page
params['page'] = int(result['page']) + 1
logger.debug('Getting page %d', params['page'])
return subtitles
def list_subtitles(self, video, languages):
season = episode = None
if isinstance(video, Episode):
titles = [video.series] + video.alternative_series
season = video.season
episode = video.episode
else:
titles = [video.title] + video.alternative_titles
for title in titles:
subtitles = [
s for l in languages for s in self.query(
l, title, season=season, episode=episode, year=video.year)
]
if subtitles:
return subtitles
return []
def download_subtitle(self, subtitle):
# download as a zip
logger.info('Downloading subtitle %r', subtitle)
r = self.session.get(self.server_url + subtitle.pid + '/download',
params={'container': 'zip'},
timeout=10)
r.raise_for_status()
# open the zip
with ZipFile(io.BytesIO(r.content)) as zf:
if len(zf.namelist()) > 1:
raise ProviderError('More than one file to unzip')
subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))
class SubsCenterProvider(Provider):
"""SubsCenter Provider."""
languages = {Language.fromalpha2(l) for l in ['he']}
server_url = 'http://subscenter.cinemast.com/he/'
def __init__(self, username=None, password=None):
if username is not None and password is None or username is None and password is not None:
raise ConfigurationError('Username and password must be specified')
self.username = username
self.password = password
self.logged_in = False
def initialize(self):
self.session = Session()
self.session.headers[
'User-Agent'] = 'Subliminal/%s' % __short_version__
# login
if self.username is not None and self.password is not None:
logger.debug('Logging in')
url = self.server_url + 'subscenter/accounts/login/'
# retrieve CSRF token
self.session.get(url)
csrf_token = self.session.cookies['csrftoken']
# actual login
data = {
'username': self.username,
'password': self.password,
'csrfmiddlewaretoken': csrf_token
}
r = self.session.post(url, data, allow_redirects=False, timeout=10)
if r.status_code != 302:
raise AuthenticationError(self.username)
logger.info('Logged in')
self.logged_in = True
def terminate(self):
# logout
if self.logged_in:
logger.info('Logging out')
r = self.session.get(self.server_url +
'subscenter/accounts/logout/',
timeout=10)
r.raise_for_status()
logger.info('Logged out')
self.logged_in = False
self.session.close()
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
def _search_url_title(self, title, kind):
"""Search the URL title for the given `title`.
:param str title: title to search for.
:param str kind: kind of the title, ``movie`` or ``series``.
:return: the URL version of the title.
:rtype: str
"""
# make the search
logger.info('Searching title name for %r', title)
r = self.session.get(self.server_url + 'subtitle/search/',
params={'q': title},
allow_redirects=False,
timeout=10)
r.raise_for_status()
# if redirected, get the url title from the Location header
if r.is_redirect:
parts = r.headers['Location'].split('/')
# check kind
if parts[-3] == kind:
return parts[-2]
return None
# otherwise, get the first valid suggestion
soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
suggestions = soup.select('#processes div.generalWindowTop a')
logger.debug('Found %d suggestions', len(suggestions))
for suggestion in suggestions:
parts = suggestion.attrs['href'].split('/')
# check kind
if parts[-3] == kind:
return parts[-2]
def query(self, series=None, season=None, episode=None, title=None):
# set the correct parameters depending on the kind
if series and season and episode:
url_series = self._search_url_title(series, 'series')
url = self.server_url + 'cinemast/data/series/sb/{}/{}/{}/'.format(
url_series, season, episode)
page_link = self.server_url + 'subtitle/series/{}/{}/{}/'.format(
url_series, season, episode)
elif title:
url_title = self._search_url_title(title, 'movie')
url = self.server_url + 'cinemast/data/movie/sb/{}/'.format(
url_title)
page_link = self.server_url + 'subtitle/movie/{}/'.format(
url_title)
else:
raise ValueError('One or more parameters are missing')
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
results = json.loads(r.text)
# loop over results
subtitles = {}
for language_code, language_data in results.items():
for quality_data in language_data.values():
for quality, subtitles_data in quality_data.items():
for subtitle_item in subtitles_data.values():
# read the item
language = Language.fromalpha2(language_code)
hearing_impaired = bool(
subtitle_item['hearing_impaired'])
subtitle_id = subtitle_item['id']
subtitle_key = subtitle_item['key']
downloaded = subtitle_item['downloaded']
release = subtitle_item['subtitle_version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug(
'Found additional release %r for subtitle %d',
release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases,
release) # deterministic order
subtitles[subtitle_id].downloaded += downloaded
continue
# otherwise create it
subtitle = SubsCenterSubtitle(
language, hearing_impaired, page_link, series,
season, episode, title, subtitle_id, subtitle_key,
downloaded, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def list_subtitles(self, video, languages):
series = None
season = None
episode = None
title = video.title
if isinstance(video, Episode):
series = video.series
season = video.season
episode = video.episode
return [
s for s in self.query(series, season, episode, title)
if s.language in languages
]
def download_subtitle(self, subtitle):
# download
url = self.server_url + 'subtitle/download/{}/{}/'.format(
subtitle.language.alpha2, subtitle.subtitle_id)
params = {'v': subtitle.releases[0], 'key': subtitle.subtitle_key}
r = self.session.get(url,
params=params,
headers={'Referer': subtitle.page_link},
timeout=10)
r.raise_for_status()
# open the zip
with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
# remove some filenames from the namelist
namelist = [n for n in zf.namelist() if not n.endswith('.txt')]
if len(namelist) > 1:
raise ProviderError('More than one file to unzip')
subtitle.content = fix_line_ending(zf.read(namelist[0]))
def generateOptions(self, inputfile, original=None, force_transcode=False):
# Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
# Video stream
self.log.info("Reading video stream.")
self.log.info("Video codec detected: %s." % info.video.codec)
try:
vbr = self.estimateVideoBitrate(info)
except:
vbr = info.format.bitrate / 1000
if info.video.codec.lower(
) in self.video_codec and not force_transcode:
vcodec = 'copy'
else:
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate if self.video_bitrate else vbr
self.log.info("Pix Fmt: %s." % info.video.pix_fmt)
if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt:
self.log.debug(
"Overriding video pix_fmt. Codec cannot be copied because pix_fmt is not approved."
)
vcodec = self.video_codec[0]
pix_fmt = self.pix_fmt[0]
if self.video_profile:
vprofile = self.video_profile[0]
else:
pix_fmt = None
if self.video_bitrate is not None and vbr > self.video_bitrate:
self.log.debug(
"Overriding video bitrate. Codec cannot be copied because video bitrate is too high."
)
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate
if self.video_width is not None and self.video_width < info.video.video_width:
self.log.debug(
"Video width is over the max width, it will be downsampled. Video stream can no longer be copied."
)
vcodec = self.video_codec[0]
vwidth = self.video_width
else:
vwidth = None
if '264' in info.video.codec.lower(
) and self.h264_level and info.video.video_level and (
info.video.video_level / 10 > self.h264_level):
self.log.info("Video level %0.1f." % (info.video.video_level / 10))
vcodec = self.video_codec[0]
self.log.debug("Video codec: %s." % vcodec)
self.log.debug("Video bitrate: %s." % vbitrate)
self.log.info("Profile: %s." % info.video.profile)
if self.video_profile and info.video.profile.lower().replace(
" ", "") not in self.video_profile:
self.log.debug(
"Video profile is not supported. Video stream can no longer be copied."
)
vcodec = self.video_codec[0]
vprofile = self.video_profile[0]
if self.pix_fmt:
pix_fmt = self.pix_fmt[0]
else:
vprofile = None
# Audio streams
self.log.info("Reading audio streams.")
overrideLang = True
for a in info.audio:
try:
if a.metadata['language'].strip(
) == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
if (a.metadata['language'] == 'und' and self.adl) or (
self.awl and a.metadata['language'].lower() in self.awl):
overrideLang = False
break
if overrideLang:
self.awl = None
self.log.info(
"No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present."
)
audio_settings = {}
blocked_audio_languages = []
l = 0
for a in info.audio:
try:
if a.metadata['language'].strip(
) == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
self.log.info("Audio detected for stream #%s: %s [%s]." %
(a.index, a.codec, a.metadata['language']))
if self.output_extension in valid_tagging_extensions and a.codec.lower(
) == 'truehd' and self.ignore_truehd: # Need to skip it early so that it flags the next track as default.
self.log.info(
"MP4 containers do not support truehd audio, and converting it is inconsistent due to video/audio sync issues. Skipping stream %s as typically the 2nd audio track is the AC3 core of the truehd stream."
% a.index)
continue
# Set undefined language to default language if specified
if self.adl is not None and a.metadata['language'] == 'und':
self.log.debug(
"Undefined language detected, defaulting to [%s]." %
self.adl)
a.metadata['language'] = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
iosdata = None
if self.awl is None or (a.metadata['language'].lower() in self.awl
and a.metadata['language'].lower()
not in blocked_audio_languages):
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS and a.audio_channels > 2:
iOSbitrate = 256 if (self.audio_bitrate * 2) > 256 else (
self.audio_bitrate * 2)
self.log.info(
"Creating audio stream %s from source audio stream %s [iOS-audio]."
% (str(l), a.index))
self.log.debug("Audio codec: %s." % self.iOS[0])
self.log.debug("Channels: 2.")
self.log.debug("Filter: %s." % self.iOS_filter)
self.log.debug("Bitrate: %s." % iOSbitrate)
self.log.debug("Language: %s." % a.metadata['language'])
iosdata = {
'map': a.index,
'codec': self.iOS[0],
'channels': 2,
'bitrate': iOSbitrate,
'filter': self.iOS_filter,
'language': a.metadata['language'],
'disposition': 'none',
}
if not self.iOSLast:
audio_settings.update({l: iosdata})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
self.log.info(
"Creating audio stream %s from source stream %s." %
(str(l), a.index))
if self.iOS and a.audio_channels <= 2:
self.log.debug(
"Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio]."
)
acodec = 'copy' if a.codec in self.iOS else self.iOS[0]
audio_channels = a.audio_channels
afilter = self.iOS_filter
abitrate = a.audio_channels * 128 if (
a.audio_channels * self.audio_bitrate) > (
a.audio_channels * 128) else (a.audio_channels *
self.audio_bitrate)
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = 'copy' if a.codec.lower(
) in self.audio_codec else self.audio_codec[0]
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == 'copy':
acodec = self.audio_codec[0]
abitrate = self.maxchannels * self.audio_bitrate
else:
audio_channels = a.audio_channels
abitrate = a.audio_channels * self.audio_bitrate
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug(
"Attempting to set bitrate based on source stream bitrate."
)
try:
abitrate = a.bitrate / 1000
except:
self.log.warning(
"Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel."
% a.index)
abitrate = a.audio_channels * 256
afilter = self.audio_filter
self.log.debug("Audio codec: %s." % acodec)
self.log.debug("Channels: %s." % audio_channels)
self.log.debug("Bitrate: %s." % abitrate)
self.log.debug("Language: %s" % a.metadata['language'])
self.log.debug("Filter: %s" % afilter)
# If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed
if self.iOS and self.iOSFirst:
self.log.debug(
"Not creating any additional iOS audio streams.")
self.iOS = False
audio_settings.update({
l: {
'map': a.index,
'codec': acodec,
'channels': audio_channels,
'bitrate': abitrate,
'filter': afilter,
'language': a.metadata['language'],
'disposition': 'none',
}
})
if acodec == 'copy' and a.codec == 'aac' and self.aac_adtstoasc:
audio_settings[l]['bsf'] = 'aac_adtstoasc'
l += 1
# Add the iOS track last instead
if self.iOSLast and iosdata:
audio_settings.update({l: iosdata})
l += 1
if self.audio_copyoriginal and acodec != 'copy':
self.log.info(
"Adding copy of original audio track in format %s" %
a.codec)
audio_settings.update({
l: {
'map': a.index,
'codec': 'copy',
'language': a.metadata['language'],
'disposition': 'none',
}
})
# Remove the language if we only want the first track from a given language
if self.audio_first_language_track and self.awl:
try:
blocked_audio_languages.append(
a.metadata['language'].lower())
self.log.debug(
"Removing language from whitelist to prevent multiple tracks of the same: %s."
% a.metadata['language'])
except:
self.log.error(
"Unable to remove language %s from whitelist." %
a.metadata['language'])
# Audio Default
if len(audio_settings) > 0 and self.adl:
try:
default_track = [
x for x in audio_settings.values()
if x['language'] == self.adl
][0]
default_track['disposition'] = 'default'
except:
audio_settings[0]['disposition'] = 'default'
else:
self.log.error("Audio language array is empty.")
# Subtitle streams
subtitle_settings = {}
l = 0
self.log.info("Reading subtitle streams.")
for s in info.subtitle:
try:
if s.metadata['language'].strip(
) == "" or s.metadata['language'] is None:
s.metadata['language'] = 'und'
except KeyError:
s.metadata['language'] = 'und'
self.log.info("Subtitle detected for stream #%s: %s [%s]." %
(s.index, s.codec, s.metadata['language']))
# Set undefined language to default language if specified
if self.sdl is not None and s.metadata['language'] == 'und':
self.log.debug(
"Undefined language detected, defaulting to [%s]." %
self.sdl)
s.metadata['language'] = self.sdl
# Make sure its not an image based codec
if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.metadata['language'].lower(
) in self.swl:
subtitle_settings.update({
l: {
'map': s.index,
'codec': self.scodec[0],
'language': s.metadata['language'],
'encoding': self.subencoding,
'disposition': 'none',
# 'forced': s.sub_forced,
# 'default': s.sub_default
}
})
self.log.info(
"Creating subtitle stream %s from source stream %s." %
(l, s.index))
l = l + 1
elif s.codec.lower(
) not in bad_subtitle_codecs and not self.embedsubs:
if self.swl is None or s.metadata['language'].lower(
) in self.swl:
for codec in self.scodec:
ripsub = {
0: {
'map': s.index,
'codec': codec,
'language': s.metadata['language']
}
}
options = {
'format': codec,
'subtitle': ripsub,
}
try:
extension = subtitle_codec_extensions[codec]
except:
self.log.info(
"Wasn't able to determine subtitle file extension, defaulting to '.srt'."
)
extension = 'srt'
forced = ".forced" if s.sub_forced else ""
input_dir, filename, input_extension = self.parseFile(
inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(
output_dir, filename + "." +
s.metadata['language'] + forced + "." + extension)
i = 2
while os.path.isfile(outputfile):
self.log.debug(
"%s exists, appending %s to filename." %
(outputfile, i))
outputfile = os.path.join(
output_dir,
filename + "." + s.metadata['language'] +
forced + "." + str(i) + "." + extension)
i += 1
try:
self.log.info(
"Ripping %s subtitle from source stream %s into external file."
% (s.metadata['language'], s.index))
conv = Converter(self.FFMPEG_PATH,
self.FFPROBE_PATH).convert(
inputfile,
outputfile,
options,
timeout=None)
for timecode in conv:
pass
self.log.info("%s created." % outputfile)
except:
self.log.exception(
"Unabled to create external subtitle file for stream %s."
% (s.index))
try:
os.chmod(outputfile, self.permissions
) # Set permissions of newly created file
except:
self.log.exception(
"Unable to set new file permissions.")
# Attempt to download subtitles if they are missing using subliminal
languages = set()
try:
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
self.log.error(
"No valid subtitle language specified, cannot download subtitles."
)
except:
self.log.exception(
"Unable to verify subtitle languages for download.")
self.downloadsubs = False
if self.downloadsubs:
import subliminal
self.log.info("Attempting to download subtitles.")
# Attempt to set the dogpile cache
try:
subliminal.region.configure('dogpile.cache.memory')
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile),
subtitles=True,
embedded_subtitles=True)
subtitles = subliminal.download_best_subtitles(
[video],
languages,
hearing_impaired=False,
providers=self.subproviders)
try:
subliminal.save_subtitles(video, subtitles[video])
except:
# Support for older versions of subliminal
subliminal.save_subtitles(subtitles)
self.log.info(
"Please update to the latest version of subliminal.")
except Exception as e:
self.log.info("Unable to download subtitles.", exc_info=True)
self.log.debug("Unable to download subtitles.", exc_info=True)
# External subtitle import
if self.embedsubs and not self.embedonlyinternalsubs: # Don't bother if we're not embeddeding subtitles and external subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
self.log.info(
"External %s subtitle file detected." % lang)
if self.swl is None or lang in self.swl:
self.log.info(
"Creating subtitle stream %s by importing %s."
% (l, fname))
subtitle_settings.update({
l: {
'path': os.path.join(dirName, fname),
'source': src,
'map': 0,
'codec': 'mov_text',
'disposition': 'none',
'language': lang
}
})
self.log.debug("Path: %s." %
os.path.join(dirName, fname))
self.log.debug("Source: %s." % src)
self.log.debug("Codec: mov_text.")
self.log.debug("Langauge: %s." % lang)
l = l + 1
src = src + 1
self.deletesubs.add(
os.path.join(dirName, fname))
else:
self.log.info(
"Ignoring %s external subtitle stream due to language %s."
% (fname, lang))
# Subtitle Default
if len(subtitle_settings) > 0 and self.sdl:
try:
default_track = [
x for x in subtitle_settings.values()
if x['language'] == self.sdl
][0]
default_track['disposition'] = 'default'
except:
subtitle_settings[0]['disposition'] = 'default'
else:
self.log.warning("Subtitle language array is empty.")
# Collect all options
options = {
'format': self.output_format,
'video': {
'codec': vcodec,
'map': info.video.index,
'bitrate': vbitrate,
'level': self.h264_level,
'profile': vprofile,
'pix_fmt': pix_fmt
},
'audio': audio_settings,
'subtitle': subtitle_settings,
'preopts': [],
'postopts': ['-threads', self.threads]
}
# If a CRF option is set, override the determine bitrate
if self.vcrf:
del options['video']['bitrate']
options['video']['crf'] = self.vcrf
if len(options['subtitle']) > 0:
options['preopts'].append('-fix_sub_duration')
if self.preopts:
options['preopts'].extend(self.preopts)
if self.postopts:
options['postopts'].extend(self.postopts)
if self.dxva2_decoder: # DXVA2 will fallback to CPU decoding when it hits a file that it cannot handle, so we don't need to check if the file is supported.
options['preopts'].extend(['-hwaccel', 'dxva2'])
elif info.video.codec.lower() == "hevc" and self.hevc_qsv_decoder:
options['preopts'].extend(['-vcodec', 'hevc_qsv'])
elif vcodec == "h264qsv" and info.video.codec.lower(
) == "h264" and self.qsv_decoder and (info.video.video_level / 10) < 5:
options['preopts'].extend(['-vcodec', 'h264_qsv'])
if self.auto_crop:
options['video']['mode'] = 'auto_crop'
# Add width option
if vwidth:
options['video']['width'] = vwidth
# HEVC Tagging for copied streams
if info.video.codec.lower() in ['x265', 'h265', 'hevc'
] and vcodec == 'copy':
options['postopts'].extend(['-tag:v', 'hvc1'])
self.log.info("Tagging copied video stream as hvc1")
self.options = options
return options
def rename_subs(path):
filepaths = []
sub_ext = ['.srt', '.sub', '.idx']
vidfiles = core.list_media_files(path,
media=True,
audio=False,
meta=False,
archives=False)
if not vidfiles or len(
vidfiles
) > 1: # If there is more than 1 video file, or no video files, we can't rename subs.
return
name = os.path.splitext(os.path.split(vidfiles[0])[1])[0]
for directory, _, filenames in os.walk(path):
for filename in filenames:
filepaths.extend([os.path.join(directory, filename)])
subfiles = [
item for item in filepaths if os.path.splitext(item)[1] in sub_ext
]
subfiles.sort(
) #This should sort subtitle names by language (alpha) and Number (where multiple)
renamed = []
for sub in subfiles:
subname, ext = os.path.splitext(os.path.basename(sub))
if name in subname: # The sub file name already includes the video name.
continue
words = re.findall('[a-zA-Z]+',
str(subname)) # find whole words in string
# parse the words for language descriptors.
lan = None
for word in words:
try:
if len(word) == 2:
lan = Language.fromalpha2(word.lower())
elif len(word) == 3:
lan = Language(word.lower())
elif len(word) > 3:
lan = Language.fromname(word.lower())
if lan:
break
except: #if we didn't find a language, try next word.
continue
# rename the sub file as name.lan.ext
if not lan:
# could call ffprobe to parse the sub information and get language if lan unknown here.
new_sub_name = name
else:
new_sub_name = '{name}.{lan}'.format(name=name, lan=str(lan))
new_sub = os.path.join(directory,
new_sub_name) # full path and name less ext
if '{new_sub}{ext}'.format(
new_sub=new_sub, ext=ext
) in renamed: # If duplicate names, add unique number before ext.
for i in range(1, len(renamed) + 1):
if '{new_sub}.{i}{ext}'.format(new_sub=new_sub, i=i,
ext=ext) in renamed:
continue
new_sub = '{new_sub}.{i}'.format(new_sub=new_sub, i=i)
break
new_sub = '{new_sub}{ext}'.format(new_sub=new_sub,
ext=ext) # add extension now
if os.path.isfile(new_sub): # Don't copy over existing - final check.
logger.debug(
'Unable to rename sub file {old} as destination {new} already exists'
.format(old=sub, new=new_sub))
continue
logger.debug('Renaming sub file from {old} to {new}'.format(
old=sub, new=new_sub))
renamed.append(new_sub)
try:
os.rename(sub, new_sub)
except Exception as error:
logger.error('Unable to rename sub file due to: {error}'.format(
error=error))
return
def generateOptions(self, inputfile, original=None):
# Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
# Video stream
self.log.info("Reading video stream.")
self.log.info("Video codec detected: %s." % info.video.codec)
try:
vbr = self.estimateVideoBitrate(info)
except:
vbr = info.format.bitrate / 1000
if info.video.codec.lower() in self.video_codec:
vcodec = 'copy'
else:
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate if self.video_bitrate else vbr
self.log.info("Pix Fmt: %s." % info.video.pix_fmt)
if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt:
vcodec = self.video_codec[0]
if self.video_bitrate is not None and vbr > self.video_bitrate:
self.log.debug("Overriding video bitrate. Codec cannot be copied because video bitrate is too high.")
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate
if self.video_width is not None and self.video_width < info.video.video_width:
self.log.debug("Video width is over the max width, it will be downsampled. Video stream can no longer be copied.")
vcodec = self.video_codec[0]
vwidth = self.video_width
else:
vwidth = None
if '264' in info.video.codec.lower() and self.h264_level and info.video.video_level and (info.video.video_level / 10 > self.h264_level):
self.log.info("Video level %0.1f." % (info.video.video_level / 10))
vcodec = self.video_codec[0]
self.log.debug("Video codec: %s." % vcodec)
self.log.debug("Video bitrate: %s." % vbitrate)
# Audio streams
self.log.info("Reading audio streams.")
overrideLang = True
for a in info.audio:
try:
if a.metadata['language'].strip() == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
if (a.metadata['language'] == 'und' and self.adl) or (self.awl and a.metadata['language'].lower() in self.awl):
overrideLang = False
break
if overrideLang:
self.awl = None
self.log.info("No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present.")
audio_settings = {}
l = 0
for a in info.audio:
try:
if a.metadata['language'].strip() == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
self.log.info("Audio detected for stream #%s: %s [%s]." % (a.index, a.codec, a.metadata['language']))
if a.codec.lower() == 'truehd': # Need to skip it early so that it flags the next track as default.
self.log.info( "MP4 containers do not support truehd audio, and converting it is inconsistent due to video/audio sync issues. Skipping stream %s as typically the 2nd audio track is the AC3 core of the truehd stream." % a.index )
continue
# Set undefined language to default language if specified
if self.adl is not None and a.metadata['language'] == 'und':
self.log.debug("Undefined language detected, defaulting to [%s]." % self.adl)
a.metadata['language'] = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
iosdata = None
if self.awl is None or a.metadata['language'].lower() in self.awl:
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS and a.audio_channels > 2:
vbrUsable = True if self.iOS[0] in valid_vbr_audio_codecs else False
iOSbitrate = 256 if (self.audio_bitrate * 2) > 256 else (self.audio_bitrate * 2)
self.log.info("Creating audio stream %s from source audio stream %s [iOS-audio]." % (str(l), a.index))
self.log.debug("Audio codec: %s." % self.iOS[0])
self.log.debug("Channels: 2.")
self.log.debug("Filter: %s." % self.iOS_filter)
self.log.debug("Profile: %s." % self.iOS_profile)
self.log.debug("Bitrate: %s." % "Variable" if vbrUsable else iOSbitrate)
self.log.debug("Language: %s." % a.metadata['language'])
if l == 0:
disposition = 'default'
self.log.info("Audio track is number %s setting disposition to %s" % (str(l), disposition))
else:
disposition = 'none'
self.log.info("Audio track is number %s setting disposition to %s" % (str(l), disposition))
iosdata = {
'map': a.index,
'codec': self.iOS[0],
'channels': 2,
'bitrate': None if vbrUsable else iOSbitrate,
'vbr': self.avbr if vbrUsable else None,
'filter': self.iOS_filter,
'profile': self.iOS_profile,
'language': a.metadata['language'],
'disposition': disposition,
}
if not self.iOSLast:
audio_settings.update({l: iosdata})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
self.log.info("Creating audio stream %s from source stream %s." % (str(l), a.index))
if self.iOS and a.audio_channels <= 2:
self.log.debug("Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio].")
acodec = 'copy' if a.codec in self.iOS else self.iOS[0]
audio_channels = a.audio_channels
afilter = self.iOS_filter
aprofile = self.iOS_profile
abitrate = a.audio_channels * 128 if (a.audio_channels * self.audio_bitrate) > (a.audio_channels * 128) else (a.audio_channels * self.audio_bitrate)
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = 'copy' if a.codec.lower() in self.audio_codec else self.audio_codec[0]
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == 'copy':
acodec = self.audio_codec[0]
abitrate = self.maxchannels * self.audio_bitrate
else:
audio_channels = a.audio_channels
abitrate = a.audio_channels * self.audio_bitrate
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug("Attempting to set bitrate based on source stream bitrate.")
try:
abitrate = a.bitrate / 1000
except:
self.log.warning("Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel." % a.index)
abitrate = a.audio_channels * 256
afilter = self.audio_filter
aprofile = self.audio_profile
vbrUsable = True if acodec in valid_vbr_audio_codecs else False
self.log.debug("Audio codec: %s." % acodec)
self.log.debug("Channels: %s." % audio_channels)
self.log.debug("Bitrate: %s." % "Variable" if vbrUsable else abitrate)
self.log.debug("Language: %s" % a.metadata['language'])
self.log.debug("Filter: %s" % afilter)
self.log.debug("Profile: %s" % aprofile)
# If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed
if self.iOS and self.iOSFirst:
self.log.debug("Not creating any additional iOS audio streams.")
self.iOS = False
# Set first track as default disposition
if l == 0:
disposition = 'default'
self.log.info("Audio Track is number %s setting disposition to %s" % (a.index, disposition))
else:
disposition = 'none'
self.log.info("Audio Track is number %s setting disposition to %s" % (a.index, disposition))
# If a VBR option is set, override the bitrate
audio_settings.update({l: {
'map': a.index,
'codec': acodec,
'channels': audio_channels,
'bitrate': None if vbrUsable else abitrate,
'vbr': self.avbr if vbrUsable else None,
'filter': afilter,
'profile': aprofile,
'language': a.metadata['language'],
'disposition': disposition,
}})
if acodec == 'copy' and a.codec == 'aac' and self.aac_adtstoasc:
audio_settings[l]['bsf'] = 'aac_adtstoasc'
l += 1
# Add the iOS track last instead
if self.iOSLast and iosdata:
iosdata['disposition'] = 'none'
audio_settings.update({l: iosdata})
l += 1
if self.audio_copyoriginal and acodec != 'copy':
self.log.info("Adding copy of original audio track in format %s" % a.codec)
audio_settings.update({l: {
'map': a.index,
'codec': 'copy',
'language': a.metadata['language'],
'disposition': 'none',
}})
# Subtitle streams
subtitle_settings = {}
l = 0
self.log.info("Reading subtitle streams.")
for s in info.subtitle:
try:
if s.metadata['language'].strip() == "" or s.metadata['language'] is None:
s.metadata['language'] = 'und'
except KeyError:
s.metadata['language'] = 'und'
self.log.info("Subtitle detected for stream #%s: %s [%s]." % (s.index, s.codec, s.metadata['language']))
# Set undefined language to default language if specified
if self.sdl is not None and s.metadata['language'] == 'und':
self.log.debug("Undefined language detected, defaulting to [%s]." % self.sdl)
s.metadata['language'] = self.sdl
# Make sure its not an image based codec
if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.metadata['language'].lower() in self.swl:
subtitle_settings.update({l: {
'map': s.index,
'codec': self.scodec[0],
'language': s.metadata['language'],
'encoding': self.subencoding,
# 'forced': s.sub_forced,
# 'default': s.sub_default
}})
self.log.info("Creating subtitle stream %s from source stream %s." % (l, s.index))
l = l + 1
elif s.codec.lower() not in bad_subtitle_codecs and not self.embedsubs:
if self.swl is None or s.metadata['language'].lower() in self.swl:
for codec in self.scodec:
ripsub = {0: {
'map': s.index,
'codec': codec,
'language': s.metadata['language']
}}
options = {
'format': codec,
'subtitle': ripsub,
}
try:
extension = subtitle_codec_extensions[codec]
except:
self.log.info("Wasn't able to determine subtitle file extension, defaulting to '.srt'.")
extension = 'srt'
forced = ".forced" if s.sub_forced else ""
input_dir, filename, input_extension = self.parseFile(inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + forced + "." + extension)
i = 2
while os.path.isfile(outputfile):
self.log.debug("%s exists, appending %s to filename." % (outputfile, i))
outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + forced + "." + str(i) + "." + extension)
i += 1
try:
self.log.info("Ripping %s subtitle from source stream %s into external file." % (s.metadata['language'], s.index))
conv = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).convert(inputfile, outputfile, options, timeout=None)
for timecode in conv:
pass
self.log.info("%s created." % outputfile)
except:
self.log.exception("Unabled to create external subtitle file for stream %s." % (s.index))
# Attempt to download subtitles if they are missing using subliminal
languages = set()
try:
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
self.log.error("No valid subtitle language specified, cannot download subtitles.")
except:
self.log.exception("Unable to verify subtitle languages for download.")
self.downloadsubs = False
if self.downloadsubs:
import subliminal
self.log.info("Attempting to download subtitles.")
# Attempt to set the dogpile cache
try:
subliminal.region.configure('dogpile.cache.memory')
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile), subtitles=True, embedded_subtitles=True)
subtitles = subliminal.download_best_subtitles([video], languages, hearing_impaired=False, providers=self.subproviders)
try:
subliminal.save_subtitles(video, subtitles[video])
except:
# Support for older versions of subliminal
subliminal.save_subtitles(subtitles)
self.log.info("Please update to the latest version of subliminal.")
except Exception as e:
self.log.info("Unable to download subtitles.", exc_info=True)
self.log.debug("Unable to download subtitles.", exc_info=True)
# External subtitle import
if self.embedsubs and not self.embedonlyinternalsubs: # Don't bother if we're not embeddeding subtitles and external subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
self.log.info("External %s subtitle file detected." % lang)
if self.swl is None or lang in self.swl:
self.log.info("Creating subtitle stream %s by importing %s." % (l, fname))
subtitle_settings.update({l: {
'path': os.path.join(dirName, fname),
'source': src,
'map': 0,
'codec': 'mov_text',
'language': lang}})
self.log.debug("Path: %s." % os.path.join(dirName, fname))
self.log.debug("Source: %s." % src)
self.log.debug("Codec: mov_text.")
self.log.debug("Langauge: %s." % lang)
l = l + 1
src = src + 1
self.deletesubs.add(os.path.join(dirName, fname))
else:
self.log.info("Ignoring %s external subtitle stream due to language %s." % (fname, lang))
# Collect all options
options = {
'format': self.output_format,
'video': {
'codec': vcodec,
'map': info.video.index,
'bitrate': vbitrate,
'level': self.h264_level
},
'audio': audio_settings,
'subtitle': subtitle_settings,
'preopts': [],
'postopts': ['-threads', self.threads]
}
# If a CRF option is set, override the determine bitrate
if self.vcrf:
del options['video']['bitrate']
options['video']['crf'] = self.vcrf
if len(options['subtitle']) > 0:
options['preopts'].append('-fix_sub_duration')
if self.preopts:
options['preopts'].extend(self.preopts)
if self.postopts:
options['postopts'].extend(self.postopts)
if self.dxva2_decoder: # DXVA2 will fallback to CPU decoding when it hits a file that it cannot handle, so we don't need to check if the file is supported.
options['preopts'].extend(['-hwaccel', 'dxva2'])
elif info.video.codec.lower() == "hevc" and self.hevc_qsv_decoder:
options['preopts'].extend(['-vcodec', 'hevc_qsv'])
elif vcodec == "h264qsv" and info.video.codec.lower() == "h264" and self.qsv_decoder and (info.video.video_level / 10) < 5:
options['preopts'].extend(['-vcodec', 'h264_qsv'])
# Add width option
if vwidth:
options['video']['width'] = vwidth
# Add pix_fmt
if self.pix_fmt:
options['video']['pix_fmt'] = self.pix_fmt[0]
# HEVC Tagging for copied streams
if info.video.codec.lower() in ['x265', 'h265', 'hevc'] and vcodec == 'copy':
options['postopts'].extend(['-tag:v', 'hvc1'])
self.log.info("Tagging copied video stream as hvc1")
self.options = options
return options
class WizdomProvider(Provider):
"""Wizdom Provider."""
languages = {Language.fromalpha2(l) for l in ['he']}
server_url = 'wizdom.xyz'
def __init__(self):
self.session = None
def initialize(self):
self.session = Session()
def terminate(self):
self.session.close()
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
def _search_imdb_id(self, title, year, is_movie):
"""Search the IMDB ID for the given `title` and `year`.
:param str title: title to search for.
:param int year: year to search for (or 0 if not relevant).
:param bool is_movie: If True, IMDB ID will be searched for in TMDB instead of Wizdom.
:return: the IMDB ID for the given title and year (or None if not found).
:rtype: str
"""
# make the search
logger.info('Searching IMDB ID for %r%r', title,
'' if not year else ' ({})'.format(year))
category = 'movie' if is_movie else 'tv'
title = title.replace('\'', '')
# get TMDB ID first
r = self.session.get(
'https://api.tmdb.org/3/search/{}?api_key={}&query={}{}&language=en'
.format(category, sickbeard.TMDB_API_KEY, title,
'' if not year else '&year={}'.format(year)))
r.raise_for_status()
tmdb_results = r.json().get('results')
if tmdb_results:
tmdb_id = tmdb_results[0].get('id')
if tmdb_id:
# get actual IMDB ID from TMDB
r = self.session.get(
'https://api.tmdb.org/3/{}/{}{}?api_key={}&language=en'.
format(category, tmdb_id,
'' if is_movie else '/external_ids',
sickbeard.TMDB_API_KEY))
r.raise_for_status()
return str(r.json().get('imdb_id', '')) or None
return None
def query(self,
title,
season=None,
episode=None,
year=None,
filename=None,
imdb_id=None):
# search for the IMDB ID if needed.
is_movie = not (season and episode)
imdb_id = imdb_id or self._search_imdb_id(title, year, is_movie)
if not imdb_id:
return {}
# search
logger.debug('Using IMDB ID %r', imdb_id)
url = 'https://json.{}/{}.json'.format(self.server_url, imdb_id)
page_link = 'https://{}/#/{}/{}'.format(
self.server_url, 'movies' if is_movie else 'series', imdb_id)
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
try:
results = r.json()
except ValueError:
return {}
# filter irrelevant results
if not is_movie:
results = results.get('subs', {}).get(str(season),
{}).get(str(episode), [])
else:
results = results.get('subs', [])
# loop over results
subtitles = {}
for result in results:
language = Language.fromalpha2('he')
hearing_impaired = False
subtitle_id = result['id']
release = result['version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug('Found additional release %r for subtitle %d',
release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases,
release) # deterministic order
subtitles[subtitle_id].downloaded += 1
continue
# otherwise create it
subtitle = WizdomSubtitle(language, hearing_impaired, page_link,
title, season, episode, title, imdb_id,
subtitle_id, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def list_subtitles(self, video, languages):
season = episode = None
title = video.title
year = video.year
filename = video.name
imdb_id = video.imdb_id
if isinstance(video, Episode):
title = video.series
season = video.season
episode = video.episode
imdb_id = video.series_imdb_id
return [
s for s in self.query(title, season, episode, year, filename,
imdb_id) if s.language in languages
]
def download_subtitle(self, subtitle):
# download
url = 'https://zip.{}/{}.zip'.format(self.server_url,
subtitle.subtitle_id)
r = self.session.get(url,
headers={'Referer': subtitle.page_link},
timeout=10)
r.raise_for_status()
# open the zip
with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
# remove some filenames from the namelist
namelist = [
n for n in zf.namelist()
if os.path.splitext(n)[1] in ['.srt', '.sub']
]
if len(namelist) > 1:
raise ProviderError('More than one file to unzip')
subtitle.content = fix_line_ending(zf.read(namelist[0]))
class PodnapisiProvider(Provider):
languages = ({Language('por', 'BR'), Language('srp', script='Latn')} |
{Language.fromalpha2(l) for l in language_converters['alpha2'].codes})
video_types = (Episode, Movie)
server_url = 'http://podnapisi.net/subtitles/'
def initialize(self):
self.session = Session()
self.session.headers = {'User-Agent': 'Subliminal/%s' % get_version(__version__)}
def terminate(self):
self.session.close()
def query(self, language, keyword, season=None, episode=None, year=None):
# set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652
params = {'sXML': 1, 'sL': str(language), 'sK': keyword}
is_episode = False
if season and episode:
is_episode = True
params['sTS'] = season
params['sTE'] = episode
if year:
params['sY'] = year
# loop over paginated results
logger.info('Searching subtitles %r', params)
subtitles = []
pids = set()
while True:
# query the server
xml = etree.fromstring(self.session.get(self.server_url + 'search/old', params=params, timeout=10).content)
# exit if no results
if not int(xml.find('pagination/results').text):
logger.debug('No subtitles found')
break
# loop over subtitles
for subtitle_xml in xml.findall('subtitle'):
# read xml elements
language = Language.fromietf(subtitle_xml.find('language').text)
hearing_impaired = 'n' in (subtitle_xml.find('flags').text or '')
page_link = subtitle_xml.find('url').text
pid = subtitle_xml.find('pid').text
releases = []
if subtitle_xml.find('release').text:
for release in subtitle_xml.find('release').text.split():
releases.append(re.sub(r'\.+$', '', release)) # remove trailing dots
title = subtitle_xml.find('title').text
season = int(subtitle_xml.find('tvSeason').text)
episode = int(subtitle_xml.find('tvEpisode').text)
year = int(subtitle_xml.find('year').text)
if is_episode:
subtitle = PodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title,
season=season, episode=episode, year=year)
else:
subtitle = PodnapisiSubtitle(language, hearing_impaired, page_link, pid, releases, title,
year=year)
# ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321
if pid in pids:
continue
logger.debug('Found subtitle %r', subtitle)
subtitles.append(subtitle)
pids.add(pid)
# stop on last page
if int(xml.find('pagination/current').text) >= int(xml.find('pagination/count').text):
break
# increment current page
params['page'] = int(xml.find('pagination/current').text) + 1
logger.debug('Getting page %d', params['page'])
return subtitles
def list_subtitles(self, video, languages):
if isinstance(video, Episode):
return [s for l in languages for s in self.query(l, video.series, season=video.season,
episode=video.episode, year=video.year)]
elif isinstance(video, Movie):
return [s for l in languages for s in self.query(l, video.title, year=video.year)]
def download_subtitle(self, subtitle):
# download as a zip
logger.info('Downloading subtitle %r')
r = self.session.get(self.server_url + subtitle.pid + '/download', params={'container': 'zip'}, timeout=10)
r.raise_for_status()
# open the zip
with ZipFile(io.BytesIO(r.content)) as zf:
if len(zf.namelist()) > 1:
raise ProviderError('More than one file to unzip')
subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))
print(len(section))
fork, fork_params = auto_fork('SickBeard', 'tv')
if server_responding('http://127.0.0.1:5050'):
print('CouchPotato Running')
if server_responding('http://127.0.0.1:7073'):
print('SickBeard Running')
if server_responding('http://127.0.0.1:8181'):
print('HeadPhones Running')
if server_responding('http://127.0.0.1:8085'):
print('Gamez Running')
if server_responding('http://127.0.0.1:8090'):
print('Mylar Running')
lan = 'pt'
lan = Language.fromalpha2(lan)
print(lan.alpha3)
vidName = '/volume1/Public/Movies/A Few Good Men/A Few Good Men(1992).mkv'
inputName = 'in.the.name.of.ben.hur.2016.bdrip.x264-rusted.nzb'
guess = guessit.guessit(inputName)
if guess:
# Movie Title
title = None
if 'title' in guess:
title = guess['title']
# Movie Year
year = None
if 'year' in guess:
year = guess['year']
url = 'http://www.omdbapi.com'
r = requests.get(url,
fork, fork_params = autoFork('SickBeard', 'tv')
if server_responding("http://127.0.0.1:5050"):
print "CouchPotato Running"
if server_responding("http://127.0.0.1:7073"):
print "SickBeard Running"
if server_responding("http://127.0.0.1:8181"):
print "HeadPhones Running"
if server_responding("http://127.0.0.1:8085"):
print "Gamez Running"
if server_responding("http://127.0.0.1:8090"):
print "Mylar Running"
from babelfish import Language
lan = 'pt'
lan = Language.fromalpha2(lan)
print lan.alpha3
vidName = "/volume1/Public/Movies/A Few Good Men/A Few Good Men(1992).mkv"
inputName = "in.the.name.of.ben.hur.2016.bdrip.x264-rusted.nzb"
guess = guessit.guessit(inputName)
if guess:
# Movie Title
title = None
if 'title' in guess:
title = guess['title']
# Movie Year
year = None
if 'year' in guess:
year = guess['year']
url = "http://www.omdbapi.com"
r = requests.get(url, params={'y': year, 't': title}, verify=False, timeout=(60, 300))
def generateOptions(self, inputfile, original=None):
# Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
# Video stream
self.log.info("Reading video stream.")
self.log.info("Video codec detected: %s." % info.video.codec)
try:
vbr = self.estimateVideoBitrate(info)
except:
vbr = info.format.bitrate / 1000
if info.video.codec.lower() in self.video_codec:
vcodec = 'copy'
else:
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate if self.video_bitrate else vbr
self.log.info("Pix Fmt: %s." % info.video.pix_fmt)
if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt:
vcodec = self.video_codec[0]
if self.video_bitrate is not None and vbr > self.video_bitrate:
self.log.debug("Overriding video bitrate. Codec cannot be copied because video bitrate is too high.")
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate
if self.video_width is not None and self.video_width < info.video.video_width:
self.log.debug("Video width is over the max width, it will be downsampled. Video stream can no longer be copied.")
vcodec = self.video_codec[0]
vwidth = self.video_width
else:
vwidth = None
if self.h264_level and info.video.video_level and (info.video.video_level / 10 > self.h264_level):
self.log.info("Video level %0.1f." % (info.video.video_level / 10))
vcodec = self.video_codec[0]
self.log.debug("Video codec: %s." % vcodec)
self.log.debug("Video bitrate: %s." % vbitrate)
# Audio streams
self.log.info("Reading audio streams.")
overrideLang = True
for a in info.audio:
try:
if a.metadata['language'].strip() == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
if (a.metadata['language'] == 'und' and self.adl) or (self.awl and a.metadata['language'].lower() in self.awl):
overrideLang = False
break
if overrideLang:
self.awl = None
self.log.info("No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present.")
audio_settings = {}
l = 0
for a in info.audio:
try:
if a.metadata['language'].strip() == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
self.log.info("Audio detected for stream #%s: %s [%s]." % (a.index, a.codec, a.metadata['language']))
# Set undefined language to default language if specified
if self.adl is not None and a.metadata['language'] == 'und':
self.log.debug("Undefined language detected, defaulting to [%s]." % self.adl)
a.metadata['language'] = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.awl is None or a.metadata['language'].lower() in self.awl:
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS:
if a.audio_channels > 2:
iOSbitrate = 256 if (self.audio_bitrate * 2) > 256 else (self.audio_bitrate * 2)
self.log.info("Creating audio stream %s from source audio stream %s [iOS-audio]." % (str(l), a.index))
self.log.debug("Audio codec: %s." % self.iOS)
self.log.debug("Channels: 2.")
self.log.debug("Bitrate: %s." % iOSbitrate)
self.log.debug("Language: %s." % a.metadata['language'])
audio_settings.update({l: {
'map': a.index,
'codec': self.iOS,
'channels': 2,
'bitrate': iOSbitrate,
'language': a.metadata['language'],
}})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
self.log.info("Creating audio stream %s from source stream %s." % (str(l), a.index))
if self.iOS and a.audio_channels <= 2:
self.log.debug("Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio].")
acodec = 'copy' if a.codec == self.iOS else self.iOS
audio_channels = a.audio_channels
abitrate = a.audio_channels * 128 if (a.audio_channels * self.audio_bitrate) > (a.audio_channels * 128) else (a.audio_channels * self.audio_bitrate)
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = 'copy' if a.codec.lower() in self.audio_codec else self.audio_codec[0]
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == 'copy':
acodec = self.audio_codec[0]
abitrate = self.maxchannels * self.audio_bitrate
else:
audio_channels = a.audio_channels
abitrate = a.audio_channels * self.audio_bitrate
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug("Attempting to set bitrate based on source stream bitrate.")
try:
abitrate = a.bitrate / 1000
except:
self.log.warning("Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel." % a.index)
abitrate = a.audio_channels * 256
self.log.debug("Audio codec: %s." % acodec)
self.log.debug("Channels: %s." % audio_channels)
self.log.debug("Bitrate: %s." % abitrate)
self.log.debug("Language: %s" % a.metadata['language'])
# If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed
if self.iOS and self.iOSFirst:
self.log.debug("Not creating any additional iOS audio streams.")
self.iOS = False
audio_settings.update({l: {
'map': a.index,
'codec': acodec,
'channels': audio_channels,
'bitrate': abitrate,
'language': a.metadata['language'],
}})
if acodec == 'copy' and a.codec == 'aac':
audio_settings[l]['bsf'] = 'aac_adtstoasc'
l = l + 1
# Subtitle streams
subtitle_settings = {}
l = 0
self.log.info("Reading subtitle streams.")
for s in info.subtitle:
try:
if s.metadata['language'].strip() == "" or s.metadata['language'] is None:
s.metadata['language'] = 'und'
except KeyError:
s.metadata['language'] = 'und'
self.log.info("Subtitle detected for stream #%s: %s [%s]." % (s.index, s.codec, s.metadata['language']))
# Set undefined language to default language if specified
if self.sdl is not None and s.metadata['language'] == 'und':
self.log.debug("Undefined language detected, defaulting to [%s]." % self.sdl)
s.metadata['language'] = self.sdl
# Make sure its not an image based codec
if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.metadata['language'].lower() in self.swl:
subtitle_settings.update({l: {
'map': s.index,
'codec': self.scodec[0],
'language': s.metadata['language'],
'encoding': self.subencoding,
# 'forced': s.sub_forced,
# 'default': s.sub_default
}})
self.log.info("Creating subtitle stream %s from source stream %s." % (l, s.index))
l = l + 1
elif s.codec.lower() not in bad_subtitle_codecs and not self.embedsubs:
if self.swl is None or s.metadata['language'].lower() in self.swl:
for codec in self.scodec:
ripsub = {0: {
'map': s.index,
'codec': codec,
'language': s.metadata['language']
}}
options = {
'format': codec,
'subtitle': ripsub,
}
try:
extension = subtitle_codec_extensions[codec]
except:
self.log.info("Wasn't able to determine subtitle file extension, defaulting to '.srt'.")
extension = 'srt'
forced = ".forced" if s.sub_forced else ""
input_dir, filename, input_extension = self.parseFile(inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + forced + "." + extension)
i = 2
while os.path.isfile(outputfile):
self.log.debug("%s exists, appending %s to filename." % (outputfile, i))
outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + forced + "." + str(i) + "." + extension)
i += 1
try:
self.log.info("Ripping %s subtitle from source stream %s into external file." % (s.metadata['language'], s.index))
conv = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).convert(inputfile, outputfile, options, timeout=None)
for timecode in conv:
pass
self.log.info("%s created." % outputfile)
except:
self.log.exception("Unabled to create external subtitle file for stream %s." % (s.index))
# Attempt to download subtitles if they are missing using subliminal
languages = set()
try:
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
self.log.error("No valid subtitle language specified, cannot download subtitles.")
except:
self.log.exception("Unable to verify subtitle languages for download.")
self.downloadsubs = False
if self.downloadsubs:
import subliminal
self.log.info("Attempting to download subtitles.")
# Attempt to set the dogpile cache
try:
subliminal.region.configure('dogpile.cache.memory')
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile), subtitles=True, embedded_subtitles=True)
subtitles = subliminal.download_best_subtitles([video], languages, hearing_impaired=False, providers=self.subproviders)
try:
subliminal.save_subtitles(video, subtitles[video])
except:
# Support for older versions of subliminal
subliminal.save_subtitles(subtitles)
self.log.info("Please update to the latest version of subliminal.")
except Exception as e:
self.log.info("Unable to download subtitles.", exc_info=True)
self.log.debug("Unable to download subtitles.", exc_info=True)
# External subtitle import
if self.embedsubs: # Don't bother if we're not embeddeding any subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
self.log.info("External %s subtitle file detected." % lang)
if self.swl is None or lang in self.swl:
self.log.info("Creating subtitle stream %s by importing %s." % (l, fname))
subtitle_settings.update({l: {
'path': os.path.join(dirName, fname),
'source': src,
'map': 0,
'codec': 'mov_text',
'language': lang}})
self.log.debug("Path: %s." % os.path.join(dirName, fname))
self.log.debug("Source: %s." % src)
self.log.debug("Codec: mov_text.")
self.log.debug("Langauge: %s." % lang)
l = l + 1
src = src + 1
self.deletesubs.add(os.path.join(dirName, fname))
else:
self.log.info("Ignoring %s external subtitle stream due to language %s." % (fname, lang))
# Collect all options
options = {
'format': self.output_format,
'video': {
'codec': vcodec,
'map': info.video.index,
'bitrate': vbitrate,
'level': self.h264_level
},
'audio': audio_settings,
'subtitle': subtitle_settings,
'preopts': ['-fix_sub_duration'],
'postopts': ['-threads', self.threads]
}
# If using h264qsv, add the codec in front of the input for decoding
if vcodec == "h264qsv" and info.video.codec.lower() == "h264" and self.qsv_decoder and (info.video.video_level / 10) < 5:
options['preopts'].extend(['-vcodec', 'h264_qsv'])
# Add width option
if vwidth:
options['video']['width'] = vwidth
# Add pix_fmt
if self.pix_fmt:
options['video']['pix_fmt'] = self.pix_fmt[0]
self.options = options
return options
class PodnapisiProvider(_PodnapisiProvider):
languages = ({
Language('por', 'BR'),
Language('srp', script='Latn'),
Language('srp', script='Cyrl')
} | {Language.fromalpha2(l)
for l in language_converters['alpha2'].codes})
server_url = 'https://podnapisi.net/subtitles/'
only_foreign = False
subtitle_class = PodnapisiSubtitle
hearing_impaired_verifiable = True
def __init__(self, only_foreign=False):
self.only_foreign = only_foreign
if only_foreign:
logger.info("Only searching for foreign/forced subtitles")
super(PodnapisiProvider, self).__init__()
def list_subtitles(self, video, languages):
if video.is_special:
logger.info("%s can't search for specials right now, skipping",
self)
return []
if isinstance(video, Episode):
return [
s for l in languages
for s in self.query(l,
video.series,
season=video.season,
episode=video.episode,
year=video.year,
only_foreign=self.only_foreign)
]
elif isinstance(video, Movie):
return [
s for l in languages
for s in self.query(l,
video.title,
year=video.year,
only_foreign=self.only_foreign)
]
def query(self,
language,
keyword,
season=None,
episode=None,
year=None,
only_foreign=False):
search_language = str(language).lower()
# sr-Cyrl specialcase
if search_language == "sr-cyrl":
search_language = "sr"
# set parameters, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164#p212652
params = {'sXML': 1, 'sL': search_language, 'sK': keyword}
is_episode = False
if season and episode:
is_episode = True
params['sTS'] = season
params['sTE'] = episode
if year:
params['sY'] = year
# loop over paginated results
logger.info('Searching subtitles %r', params)
subtitles = []
pids = set()
while True:
# query the server
content = None
try:
content = self.session.get(self.server_url + 'search/old',
params=params,
timeout=10).content
xml = etree.fromstring(content)
except XMLSyntaxError:
logger.error("Wrong data returned: %r", content)
break
# exit if no results
if not int(xml.find('pagination/results').text):
logger.debug('No subtitles found')
break
# loop over subtitles
for subtitle_xml in xml.findall('subtitle'):
# read xml elements
language = Language.fromietf(
subtitle_xml.find('language').text)
hearing_impaired = 'n' in (subtitle_xml.find('flags').text
or '')
foreign = 'f' in (subtitle_xml.find('flags').text or '')
if only_foreign and not foreign:
continue
if not only_foreign and foreign:
continue
page_link = subtitle_xml.find('url').text
pid = subtitle_xml.find('pid').text
releases = []
if subtitle_xml.find('release').text:
for release in subtitle_xml.find('release').text.split():
releases.append(re.sub(
r'\.+$', '', release)) # remove trailing dots
title = subtitle_xml.find('title').text
season = int(subtitle_xml.find('tvSeason').text)
episode = int(subtitle_xml.find('tvEpisode').text)
year = int(subtitle_xml.find('year').text)
if is_episode:
subtitle = self.subtitle_class(language,
hearing_impaired,
page_link,
pid,
releases,
title,
season=season,
episode=episode,
year=year)
else:
subtitle = self.subtitle_class(language,
hearing_impaired,
page_link,
pid,
releases,
title,
year=year)
# ignore duplicates, see http://www.podnapisi.net/forum/viewtopic.php?f=62&t=26164&start=10#p213321
if pid in pids:
continue
logger.debug('Found subtitle %r', subtitle)
subtitles.append(subtitle)
pids.add(pid)
# stop on last page
if int(xml.find('pagination/current').text) >= int(
xml.find('pagination/count').text):
break
# increment current page
params['page'] = int(xml.find('pagination/current').text) + 1
logger.debug('Getting page %d', params['page'])
xml = None
return subtitles
def build_commands(file, new_dir, movie_name, bitbucket):
if isinstance(file, string_types):
input_file = file
if 'concat:' in file:
file = file.split('|')[0].replace('concat:', '')
video_details, result = get_video_details(file)
directory, name = os.path.split(file)
name, ext = os.path.splitext(name)
check = re.match('VTS_([0-9][0-9])_[0-9]+', name)
if check and core.CONCAT:
name = movie_name
elif check:
name = ('{0}.cd{1}'.format(movie_name, check.groups()[0]))
elif core.CONCAT and re.match('(.+)[cC][dD][0-9]', name):
name = re.sub('([ ._=:-]+[cC][dD][0-9])', '', name)
if ext == core.VEXTENSION and new_dir == directory: # we need to change the name to prevent overwriting itself.
core.VEXTENSION = '-transcoded{ext}'.format(ext=core.VEXTENSION) # adds '-transcoded.ext'
else:
img, data = next(iteritems(file))
name = data['name']
video_details, result = get_video_details(data['files'][0], img, bitbucket)
input_file = '-'
file = '-'
newfile_path = os.path.normpath(os.path.join(new_dir, name) + core.VEXTENSION)
map_cmd = []
video_cmd = []
audio_cmd = []
audio_cmd2 = []
sub_cmd = []
meta_cmd = []
other_cmd = []
if not video_details or not video_details.get(
'streams'): # we couldn't read streams with ffprobe. Set defaults to try transcoding.
video_streams = []
audio_streams = []
sub_streams = []
map_cmd.extend(['-map', '0'])
if core.VCODEC:
video_cmd.extend(['-c:v', core.VCODEC])
if core.VCODEC == 'libx264' and core.VPRESET:
video_cmd.extend(['-pre', core.VPRESET])
else:
video_cmd.extend(['-c:v', 'copy'])
if core.VFRAMERATE:
video_cmd.extend(['-r', str(core.VFRAMERATE)])
if core.VBITRATE:
video_cmd.extend(['-b:v', str(core.VBITRATE)])
if core.VRESOLUTION:
video_cmd.extend(['-vf', 'scale={vres}'.format(vres=core.VRESOLUTION)])
if core.VPRESET:
video_cmd.extend(['-preset', core.VPRESET])
if core.VCRF:
video_cmd.extend(['-crf', str(core.VCRF)])
if core.VLEVEL:
video_cmd.extend(['-level', str(core.VLEVEL)])
if core.ACODEC:
audio_cmd.extend(['-c:a', core.ACODEC])
if core.ACODEC in ['aac',
'dts']: # Allow users to use the experimental AAC codec that's built into recent versions of ffmpeg
audio_cmd.extend(['-strict', '-2'])
else:
audio_cmd.extend(['-c:a', 'copy'])
if core.ACHANNELS:
audio_cmd.extend(['-ac', str(core.ACHANNELS)])
if core.ABITRATE:
audio_cmd.extend(['-b:a', str(core.ABITRATE)])
if core.OUTPUTQUALITYPERCENT:
audio_cmd.extend(['-q:a', str(core.OUTPUTQUALITYPERCENT)])
if core.SCODEC and core.ALLOWSUBS:
sub_cmd.extend(['-c:s', core.SCODEC])
elif core.ALLOWSUBS: # Not every subtitle codec can be used for every video container format!
sub_cmd.extend(['-c:s', 'copy'])
else: # http://en.wikibooks.org/wiki/FFMPEG_An_Intermediate_Guide/subtitle_options
sub_cmd.extend(['-sn']) # Don't copy the subtitles over
if core.OUTPUTFASTSTART:
other_cmd.extend(['-movflags', '+faststart'])
else:
video_streams = [item for item in video_details['streams'] if item['codec_type'] == 'video']
audio_streams = [item for item in video_details['streams'] if item['codec_type'] == 'audio']
sub_streams = [item for item in video_details['streams'] if item['codec_type'] == 'subtitle']
if core.VEXTENSION not in ['.mkv', '.mpegts']:
sub_streams = [item for item in video_details['streams'] if
item['codec_type'] == 'subtitle' and item['codec_name'] != 'hdmv_pgs_subtitle' and item[
'codec_name'] != 'pgssub']
for video in video_streams:
codec = video['codec_name']
fr = video.get('avg_frame_rate', 0)
width = video.get('width', 0)
height = video.get('height', 0)
scale = core.VRESOLUTION
if codec in core.VCODEC_ALLOW or not core.VCODEC:
video_cmd.extend(['-c:v', 'copy'])
else:
video_cmd.extend(['-c:v', core.VCODEC])
if core.VFRAMERATE and not (core.VFRAMERATE * 0.999 <= fr <= core.VFRAMERATE * 1.001):
video_cmd.extend(['-r', str(core.VFRAMERATE)])
if scale:
w_scale = width / float(scale.split(':')[0])
h_scale = height / float(scale.split(':')[1])
if w_scale > h_scale: # widescreen, Scale by width only.
scale = '{width}:{height}'.format(
width=scale.split(':')[0],
height=int((height / w_scale) / 2) * 2,
)
if w_scale > 1:
video_cmd.extend(['-vf', 'scale={width}'.format(width=scale)])
else: # lower or matching ratio, scale by height only.
scale = '{width}:{height}'.format(
width=int((width / h_scale) / 2) * 2,
height=scale.split(':')[1],
)
if h_scale > 1:
video_cmd.extend(['-vf', 'scale={height}'.format(height=scale)])
if core.VBITRATE:
video_cmd.extend(['-b:v', str(core.VBITRATE)])
if core.VPRESET:
video_cmd.extend(['-preset', core.VPRESET])
if core.VCRF:
video_cmd.extend(['-crf', str(core.VCRF)])
if core.VLEVEL:
video_cmd.extend(['-level', str(core.VLEVEL)])
no_copy = ['-vf', '-r', '-crf', '-level', '-preset', '-b:v']
if video_cmd[1] == 'copy' and any(i in video_cmd for i in no_copy):
video_cmd[1] = core.VCODEC
if core.VCODEC == 'copy': # force copy. therefore ignore all other video transcoding.
video_cmd = ['-c:v', 'copy']
map_cmd.extend(['-map', '0:{index}'.format(index=video['index'])])
break # Only one video needed
used_audio = 0
a_mapped = []
commentary = []
if audio_streams:
for i, val in reversed(list(enumerate(audio_streams))):
try:
if 'Commentary' in val.get('tags').get('title'): # Split out commentry tracks.
commentary.append(val)
del audio_streams[i]
except Exception:
continue
try:
audio1 = [item for item in audio_streams if item['tags']['language'] == core.ALANGUAGE]
except Exception: # no language tags. Assume only 1 language.
audio1 = audio_streams
try:
audio2 = [item for item in audio1 if item['codec_name'] in core.ACODEC_ALLOW]
except Exception:
audio2 = []
try:
audio3 = [item for item in audio_streams if item['tags']['language'] != core.ALANGUAGE]
except Exception:
audio3 = []
try:
audio4 = [item for item in audio3 if item['codec_name'] in core.ACODEC_ALLOW]
except Exception:
audio4 = []
if audio2: # right (or only) language and codec...
map_cmd.extend(['-map', '0:{index}'.format(index=audio2[0]['index'])])
a_mapped.extend([audio2[0]['index']])
bitrate = int(float(audio2[0].get('bit_rate', 0))) / 1000
channels = int(float(audio2[0].get('channels', 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio1: # right (or only) language, wrong codec.
map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]['index'])])
a_mapped.extend([audio1[0]['index']])
bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000
channels = int(float(audio1[0].get('channels', 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy'])
elif audio4: # wrong language, right codec.
map_cmd.extend(['-map', '0:{index}'.format(index=audio4[0]['index'])])
a_mapped.extend([audio4[0]['index']])
bitrate = int(float(audio4[0].get('bit_rate', 0))) / 1000
channels = int(float(audio4[0].get('channels', 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio3: # wrong language, wrong codec. just pick the default audio track
map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]['index'])])
a_mapped.extend([audio3[0]['index']])
bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000
channels = int(float(audio3[0].get('channels', 0)))
audio_cmd.extend(['-c:a:{0}'.format(used_audio), core.ACODEC if core.ACODEC else 'copy'])
if core.ACHANNELS and channels and channels > core.ACHANNELS:
audio_cmd.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if core.ABITRATE and not (core.ABITRATE * 0.9 < bitrate < core.ABITRATE * 1.1):
audio_cmd.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if core.OUTPUTQUALITYPERCENT:
audio_cmd.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd[1] == 'copy':
audio_cmd[1] = core.ACODEC
if audio_cmd[1] in ['aac', 'dts']:
audio_cmd[2:2] = ['-strict', '-2']
if core.ACODEC2_ALLOW:
used_audio += 1
try:
audio5 = [item for item in audio1 if item['codec_name'] in core.ACODEC2_ALLOW]
except Exception:
audio5 = []
try:
audio6 = [item for item in audio3 if item['codec_name'] in core.ACODEC2_ALLOW]
except Exception:
audio6 = []
if audio5: # right language and codec.
map_cmd.extend(['-map', '0:{index}'.format(index=audio5[0]['index'])])
a_mapped.extend([audio5[0]['index']])
bitrate = int(float(audio5[0].get('bit_rate', 0))) / 1000
channels = int(float(audio5[0].get('channels', 0)))
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio1: # right language wrong codec.
map_cmd.extend(['-map', '0:{index}'.format(index=audio1[0]['index'])])
a_mapped.extend([audio1[0]['index']])
bitrate = int(float(audio1[0].get('bit_rate', 0))) / 1000
channels = int(float(audio1[0].get('channels', 0)))
if core.ACODEC2:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2])
else:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio6: # wrong language, right codec
map_cmd.extend(['-map', '0:{index}'.format(index=audio6[0]['index'])])
a_mapped.extend([audio6[0]['index']])
bitrate = int(float(audio6[0].get('bit_rate', 0))) / 1000
channels = int(float(audio6[0].get('channels', 0)))
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
elif audio3: # wrong language, wrong codec just pick the default audio track
map_cmd.extend(['-map', '0:{index}'.format(index=audio3[0]['index'])])
a_mapped.extend([audio3[0]['index']])
bitrate = int(float(audio3[0].get('bit_rate', 0))) / 1000
channels = int(float(audio3[0].get('channels', 0)))
if core.ACODEC2:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), core.ACODEC2])
else:
audio_cmd2.extend(['-c:a:{0}'.format(used_audio), 'copy'])
if core.ACHANNELS2 and channels and channels > core.ACHANNELS2:
audio_cmd2.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS2)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if core.ABITRATE2 and not (core.ABITRATE2 * 0.9 < bitrate < core.ABITRATE2 * 1.1):
audio_cmd2.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE2)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if core.OUTPUTQUALITYPERCENT:
audio_cmd2.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd2[1] == 'copy':
audio_cmd2[1] = core.ACODEC2
if audio_cmd2[1] in ['aac', 'dts']:
audio_cmd2[2:2] = ['-strict', '-2']
if a_mapped[1] == a_mapped[0] and audio_cmd2[1:] == audio_cmd[1:]: # check for duplicate output track.
del map_cmd[-2:]
else:
audio_cmd.extend(audio_cmd2)
if core.AINCLUDE and core.ACODEC3:
audio_streams.extend(commentary) # add commentry tracks back here.
for audio in audio_streams:
if audio['index'] in a_mapped:
continue
used_audio += 1
map_cmd.extend(['-map', '0:{index}'.format(index=audio['index'])])
audio_cmd3 = []
bitrate = int(float(audio.get('bit_rate', 0))) / 1000
channels = int(float(audio.get('channels', 0)))
if audio['codec_name'] in core.ACODEC3_ALLOW:
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])
else:
if core.ACODEC3:
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), core.ACODEC3])
else:
audio_cmd3.extend(['-c:a:{0}'.format(used_audio), 'copy'])
if core.ACHANNELS3 and channels and channels > core.ACHANNELS3:
audio_cmd3.extend(['-ac:a:{0}'.format(used_audio), str(core.ACHANNELS3)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if core.ABITRATE3 and not (core.ABITRATE3 * 0.9 < bitrate < core.ABITRATE3 * 1.1):
audio_cmd3.extend(['-b:a:{0}'.format(used_audio), str(core.ABITRATE3)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if core.OUTPUTQUALITYPERCENT > 0:
audio_cmd3.extend(['-q:a:{0}'.format(used_audio), str(core.OUTPUTQUALITYPERCENT)])
if audio_cmd3[1] == 'copy':
audio_cmd3[1] = core.ACODEC3
if audio_cmd3[1] in ['aac', 'dts']:
audio_cmd3[2:2] = ['-strict', '-2']
audio_cmd.extend(audio_cmd3)
s_mapped = []
burnt = 0
n = 0
for lan in core.SLANGUAGES:
try:
subs1 = [item for item in sub_streams if item['tags']['language'] == lan]
except Exception:
subs1 = []
if core.BURN and not subs1 and not burnt and os.path.isfile(file):
for subfile in get_subs(file):
if lan in os.path.split(subfile)[1]:
video_cmd.extend(['-vf', 'subtitles={subs}'.format(subs=subfile)])
burnt = 1
for sub in subs1:
if core.BURN and not burnt and os.path.isfile(input_file):
subloc = 0
for index in range(len(sub_streams)):
if sub_streams[index]['index'] == sub['index']:
subloc = index
break
video_cmd.extend(['-vf', 'subtitles={sub}:si={loc}'.format(sub=input_file, loc=subloc)])
burnt = 1
if not core.ALLOWSUBS:
break
if sub['codec_name'] in ['dvd_subtitle', 'VobSub'] and core.SCODEC == 'mov_text': # We can't convert these.
continue
map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])])
s_mapped.extend([sub['index']])
if core.SINCLUDE:
for sub in sub_streams:
if not core.ALLOWSUBS:
break
if sub['index'] in s_mapped:
continue
if sub['codec_name'] in ['dvd_subtitle', 'VobSub'] and core.SCODEC == 'mov_text': # We can't convert these.
continue
map_cmd.extend(['-map', '0:{index}'.format(index=sub['index'])])
s_mapped.extend([sub['index']])
if core.OUTPUTFASTSTART:
other_cmd.extend(['-movflags', '+faststart'])
command = [core.FFMPEG, '-loglevel', 'warning']
if core.HWACCEL:
command.extend(['-hwaccel', 'auto'])
if core.GENERALOPTS:
command.extend(core.GENERALOPTS)
command.extend(['-i', input_file])
if core.SEMBED and os.path.isfile(file):
for subfile in get_subs(file):
sub_details, result = get_video_details(subfile)
if not sub_details or not sub_details.get('streams'):
continue
if core.SCODEC == 'mov_text':
subcode = [stream['codec_name'] for stream in sub_details['streams']]
if set(subcode).intersection(['dvd_subtitle', 'VobSub']): # We can't convert these.
continue
command.extend(['-i', subfile])
lan = os.path.splitext(os.path.splitext(subfile)[0])[1][1:].split('-')[0]
lan = text_type(lan)
metlan = None
try:
if len(lan) == 3:
metlan = Language(lan)
if len(lan) == 2:
metlan = Language.fromalpha2(lan)
except Exception:
pass
if metlan:
meta_cmd.extend(['-metadata:s:s:{x}'.format(x=len(s_mapped) + n),
'language={lang}'.format(lang=metlan.alpha3)])
n += 1
map_cmd.extend(['-map', '{x}:0'.format(x=n)])
if not core.ALLOWSUBS or (not s_mapped and not n):
sub_cmd.extend(['-sn'])
else:
if core.SCODEC:
sub_cmd.extend(['-c:s', core.SCODEC])
else:
sub_cmd.extend(['-c:s', 'copy'])
command.extend(map_cmd)
command.extend(video_cmd)
command.extend(audio_cmd)
command.extend(sub_cmd)
command.extend(meta_cmd)
command.extend(other_cmd)
command.append(newfile_path)
if platform.system() != 'Windows':
command = core.NICENESS + command
return command
def generateOptions(self, inputfile, original=None):
#Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
#Video stream
print "Video codec detected: " + info.video.codec
vcodec = 'copy' if info.video.codec in self.video_codec else self.video_codec[0]
#Audio streams
audio_settings = {}
l = 0
for a in info.audio:
print "Audio stream detected: " + a.codec + " " + a.language + " [Stream " + str(a.index) + "]"
# Set undefined language to default language if specified
if self.adl is not None and a.language == 'und':
print "Undefined language detected, defaulting to " + self.adl
a.language = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.awl is None or a.language in self.awl:
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS:
if a.audio_channels > 2:
print "Creating dual audio channels for iOS compatability for this stream"
audio_settings.update({l: {
'map': a.index,
'codec': self.iOS,
'channels': 2,
'bitrate': 256,
'language': a.language,
}})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
if self.iOS and a.audio_channels == 2:
acodec = 'copy' if a.codec == 'aac' else self.iOS
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = 'copy' if a.codec in self.audio_codec else self.audio_codec[0]
# Bitrate calculations/overrides
if self.audio_bitrate is None or self.audio_bitrate > (a.audio_channels * 256):
abitrate = 256 * a.audio_channels
else:
abitrate = self.audio_bitrate
audio_settings.update({l: {
'map': a.index,
'codec': acodec,
'channels': a.audio_channels,
'bitrate': abitrate,
'language': a.language,
}})
l = l + 1
# Subtitle streams
subtitle_settings = {}
l = 0
for s in info.subtitle:
print "Subtitle stream detected: " + s.codec + " " + s.language + " [Stream " + str(s.index) + "]"
# Set undefined language to default language if specified
if self.sdl is not None and s.language == 'und':
s.language = self.sdl
# Make sure its not an image based codec
if s.codec not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.language in self.swl:
subtitle_settings.update({l: {
'map': s.index,
'codec': 'mov_text',
'language': s.language
#'forced': s.sub_forced,
#'default': s.sub_default
}})
l = l + 1
else:
if self.swl is None or s.language in self.swl:
ripsub = {1: {
'map': s.index,
'codec': 'srt',
'language': s.language
}}
options = {
'format': 'srt',
'subtitle': ripsub,
}
input_dir, filename, input_extension = self.parseFile(inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(output_dir, filename + "." + s.language + ".srt")
i = 2
while os.path.isfile(outputfile):
outputfile = os.path.join(output_dir, filename + "." + s.language + "." + str(i) + ".srt")
i += i
print "Ripping " + s.language + " subtitle from file"
conv = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).convert(inputfile, outputfile, options, timeout=None)
for timecode in conv:
pass
try:
print outputfile + " created"
except:
print "File created"
# External subtitle import
# Attempt to download subtitles if they are missing using subliminal
languages = set()
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
if self.downloadsubs:
import subliminal
print "Attempting to download subtitles, please wait"
try:
subliminal.cache_region.configure('dogpile.cache.memory')
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile), subtitles=True, embedded_subtitles=True, original=original)
subtitles = subliminal.download_best_subtitles([video], languages, hearing_impaired=False, providers=self.subproviders)
subliminal.save_subtitles(subtitles)
except Exception as e:
print e
print "Unable to download subtitle"
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename and self.embedsubs:
print "External subtitle file detected, language " + lang
if self.swl is None or lang in self.swl:
print "Importing %s subtitle stream" % (fname)
subtitle_settings.update({l: {
'path': os.path.join(dirName, fname),
'source': src,
'map': 0,
'codec': 'mov_text',
'language': lang,
}})
l = l + 1
src = src + 1
self.deletesubs.add(os.path.join(dirName, fname))
else:
print "Ignoring %s external subtitle stream due to language: %s" % (fname, lang)
# Collect all options
options = {
'format': self.output_format,
'video': {
'codec': vcodec,
'map': info.video.index,
'bitrate': info.format.bitrate
},
'audio': audio_settings,
'subtitle': subtitle_settings,
}
self.options = options
return options
def generateOptions(self, inputfile, original=None):
# Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
# Video stream
self.log.info("Reading video stream.")
self.log.info("Video codec detected: %s." % info.video.codec)
try:
vbr = self.estimateVideoBitrate(info)
except Exception:
raise
vbr = info.format.bitrate / 1000
if info.video.codec.lower() in self.video_codec:
vcodec = "copy"
else:
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate if self.video_bitrate else vbr
self.log.info("Pix Fmt: %s." % info.video.pix_fmt)
if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt:
self.log.debug(
"Overriding video pix_fmt. Codec cannot be copied because pix_fmt is not approved."
)
vcodec = self.video_codec[0]
pix_fmt = self.pix_fmt[0]
if self.video_profile:
vprofile = self.video_profile[0]
else:
pix_fmt = None
if self.video_bitrate is not None and vbr > self.video_bitrate:
self.log.debug(
"Overriding video bitrate. Codec cannot be copied because video bitrate is too high."
)
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate
if self.video_width is not None and self.video_width < info.video.video_width:
self.log.debug(
"Video width is over the max width, it will be downsampled. Video stream can no longer be copied."
)
vcodec = self.video_codec[0]
vwidth = self.video_width
else:
vwidth = None
if (
"264" in info.video.codec.lower()
and self.h264_level
and info.video.video_level
and (info.video.video_level / 10 > self.h264_level)
):
self.log.info("Video level %0.1f." % (info.video.video_level / 10))
vcodec = self.video_codec[0]
self.log.debug("Video codec: %s." % vcodec)
self.log.debug("Video bitrate: %s." % vbitrate)
self.log.info("Profile: %s." % info.video.profile)
if (
self.video_profile
and info.video.profile.lower().replace(" ", "") not in self.video_profile
):
self.log.debug(
"Video profile is not supported. Video stream can no longer be copied."
)
vcodec = self.video_codec[0]
vprofile = self.video_profile[0]
if self.pix_fmt:
pix_fmt = self.pix_fmt[0]
else:
vprofile = None
# Audio streams
self.log.info("Reading audio streams.")
overrideLang = True
for a in info.audio:
try:
if (
a.metadata["language"].strip() == ""
or a.metadata["language"] is None
):
a.metadata["language"] = "und"
except KeyError:
a.metadata["language"] = "und"
if (a.metadata["language"] == "und" and self.adl) or (
self.awl and a.metadata["language"].lower() in self.awl
):
overrideLang = False
break
if overrideLang:
self.awl = None
self.log.info(
"No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present."
)
audio_settings = {}
blocked_audio_languages = []
l = 0
for a in info.audio:
try:
if (
a.metadata["language"].strip() == ""
or a.metadata["language"] is None
):
a.metadata["language"] = "und"
except KeyError:
a.metadata["language"] = "und"
self.log.info(
"Audio detected for stream #%s: %s [%s]."
% (a.index, a.codec, a.metadata["language"])
)
if (
self.output_extension in valid_tagging_extensions
and a.codec.lower() == "truehd"
): # Need to skip it early so that it flags the next track as default.
self.log.info(
"MP4 containers do not support truehd audio, and converting it is inconsistent due to video/audio sync issues. Skipping stream %s as typically the 2nd audio track is the AC3 core of the truehd stream."
% a.index
)
continue
# Set undefined language to default language if specified
if self.adl is not None and a.metadata["language"] == "und":
self.log.debug(
"Undefined language detected, defaulting to [%s]." % self.adl
)
a.metadata["language"] = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
iosdata = None
if self.awl is None or (
a.metadata["language"].lower() in self.awl
and a.metadata["language"].lower() not in blocked_audio_languages
):
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS and a.audio_channels > 2:
iOSbitrate = (
256
if (self.audio_bitrate * 2) > 256
else (self.audio_bitrate * 2)
)
self.log.info(
"Creating audio stream %s from source audio stream %s [iOS-audio]."
% (str(l), a.index)
)
self.log.debug("Audio codec: %s." % self.iOS[0])
self.log.debug("Channels: 2.")
self.log.debug("Filter: %s." % self.iOS_filter)
self.log.debug("Bitrate: %s." % iOSbitrate)
self.log.debug("Language: %s." % a.metadata["language"])
if l == 0:
disposition = "default"
self.log.info(
"Audio track is number %s setting disposition to %s"
% (str(l), disposition)
)
else:
disposition = "none"
self.log.info(
"Audio track is number %s setting disposition to %s"
% (str(l), disposition)
)
iosdata = {
"map": a.index,
"codec": self.iOS[0],
"channels": 2,
"bitrate": iOSbitrate,
"filter": self.iOS_filter,
"language": a.metadata["language"],
"disposition": disposition,
}
if not self.iOSLast:
audio_settings.update({l: iosdata})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
self.log.info(
"Creating audio stream %s from source stream %s."
% (str(l), a.index)
)
if self.iOS and a.audio_channels <= 2:
self.log.debug(
"Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio]."
)
acodec = "copy" if a.codec in self.iOS else self.iOS[0]
audio_channels = a.audio_channels
afilter = self.iOS_filter
abitrate = (
a.audio_channels * 128
if (a.audio_channels * self.audio_bitrate)
> (a.audio_channels * 128)
else (a.audio_channels * self.audio_bitrate)
)
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = (
"copy"
if a.codec.lower() in self.audio_codec
else self.audio_codec[0]
)
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == "copy":
acodec = self.audio_codec[0]
abitrate = self.maxchannels * self.audio_bitrate
else:
audio_channels = a.audio_channels
abitrate = a.audio_channels * self.audio_bitrate
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug(
"Attempting to set bitrate based on source stream bitrate."
)
try:
abitrate = a.bitrate / 1000
except Exception:
self.log.warning(
"Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel."
% a.index
)
abitrate = a.audio_channels * 256
afilter = self.audio_filter
self.log.debug("Audio codec: %s." % acodec)
self.log.debug("Channels: %s." % audio_channels)
self.log.debug("Bitrate: %s." % abitrate)
self.log.debug("Language: %s" % a.metadata["language"])
self.log.debug("Filter: %s" % afilter)
# If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed
if self.iOS and self.iOSFirst:
self.log.debug("Not creating any additional iOS audio streams.")
self.iOS = False
# Set first track as default disposition
if l == 0:
disposition = "default"
self.log.info(
"Audio Track is number %s setting disposition to %s"
% (a.index, disposition)
)
else:
disposition = "none"
self.log.info(
"Audio Track is number %s setting disposition to %s"
% (a.index, disposition)
)
audio_settings.update(
{
l: {
"map": a.index,
"codec": acodec,
"channels": audio_channels,
"bitrate": abitrate,
"filter": afilter,
"language": a.metadata["language"],
"disposition": disposition,
}
}
)
if acodec == "copy" and a.codec == "aac" and self.aac_adtstoasc:
audio_settings[l]["bsf"] = "aac_adtstoasc"
l += 1
# Add the iOS track last instead
if self.iOSLast and iosdata:
iosdata["disposition"] = "none"
audio_settings.update({l: iosdata})
l += 1
if self.audio_copyoriginal and acodec != "copy":
self.log.info(
"Adding copy of original audio track in format %s" % a.codec
)
audio_settings.update(
{
l: {
"map": a.index,
"codec": "copy",
"language": a.metadata["language"],
"disposition": "none",
}
}
)
# Remove the language if we only want the first track from a given language
if self.audio_first_language_track and self.awl:
try:
blocked_audio_languages.append(a.metadata["language"].lower())
self.log.debug(
"Removing language from whitelist to prevent multiple tracks of the same: %s."
% a.metadata["language"]
)
except Exception:
self.log.error(
"Unable to remove language %s from whitelist."
% a.metadata["language"]
)
# Subtitle streams
subtitle_settings = {}
l = 0
self.log.info("Reading subtitle streams.")
for s in info.subtitle:
try:
if (
s.metadata["language"].strip() == ""
or s.metadata["language"] is None
):
s.metadata["language"] = "und"
except KeyError:
s.metadata["language"] = "und"
self.log.info(
"Subtitle detected for stream #%s: %s [%s]."
% (s.index, s.codec, s.metadata["language"])
)
# Set undefined language to default language if specified
if self.sdl is not None and s.metadata["language"] == "und":
self.log.debug(
"Undefined language detected, defaulting to [%s]." % self.sdl
)
s.metadata["language"] = self.sdl
# Make sure its not an image based codec
if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.metadata["language"].lower() in self.swl:
subtitle_settings.update(
{
l: {
"map": s.index,
"codec": self.scodec[0],
"language": s.metadata["language"],
"encoding": self.subencoding,
# 'forced': s.sub_forced,
# 'default': s.sub_default
}
}
)
self.log.info(
"Creating subtitle stream %s from source stream %s."
% (l, s.index)
)
l = l + 1
elif s.codec.lower() not in bad_subtitle_codecs and not self.embedsubs:
if self.swl is None or s.metadata["language"].lower() in self.swl:
for codec in self.scodec:
ripsub = {
0: {
"map": s.index,
"codec": codec,
"language": s.metadata["language"],
}
}
options = {"format": codec, "subtitle": ripsub}
try:
extension = subtitle_codec_extensions[codec]
except Exception:
self.log.info(
"Wasn't able to determine subtitle file extension, defaulting to '.srt'."
)
extension = "srt"
forced = ".forced" if s.sub_forced else ""
input_dir, filename, input_extension = self.parseFile(inputfile)
output_dir = (
input_dir if self.output_dir is None else self.output_dir
)
outputfile = os.path.join(
output_dir,
filename
+ "."
+ s.metadata["language"]
+ forced
+ "."
+ extension,
)
i = 2
while os.path.isfile(outputfile):
self.log.debug(
"%s exists, appending %s to filename." % (outputfile, i)
)
outputfile = os.path.join(
output_dir,
filename
+ "."
+ s.metadata["language"]
+ forced
+ "."
+ str(i)
+ "."
+ extension,
)
i += 1
try:
self.log.info(
"Ripping %s subtitle from source stream %s into external file."
% (s.metadata["language"], s.index)
)
conv = Converter(
self.FFMPEG_PATH, self.FFPROBE_PATH
).convert(inputfile, outputfile, options, timeout=None)
for timecode in conv:
pass
self.log.info("%s created." % outputfile)
except Exception:
self.log.exception(
"Unabled to create external subtitle file for stream %s."
% (s.index)
)
try:
os.chmod(
outputfile, self.permissions
) # Set permissions of newly created file
except Exception:
self.log.exception("Unable to set new file permissions.")
# Attempt to download subtitles if they are missing using subliminal
languages = set()
try:
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
self.log.error(
"No valid subtitle language specified, cannot download subtitles."
)
except Exception:
self.log.exception("Unable to verify subtitle languages for download.")
self.downloadsubs = False
if self.downloadsubs:
import subliminal
self.log.info("Attempting to download subtitles.")
# Attempt to set the dogpile cache
try:
subliminal.region.configure("dogpile.cache.memory")
except Exception:
pass
try:
video = subliminal.scan_video(
os.path.abspath(inputfile), subtitles=True, embedded_subtitles=True
)
subtitles = subliminal.download_best_subtitles(
[video],
languages,
hearing_impaired=False,
providers=self.subproviders,
)
try:
subliminal.save_subtitles(video, subtitles[video])
except Exception:
# Support for older versions of subliminal
subliminal.save_subtitles(subtitles)
self.log.info("Please update to the latest version of subliminal.")
except Exception:
self.log.info("Unable to download subtitles.", exc_info=True)
self.log.debug("Unable to download subtitles.", exc_info=True)
# External subtitle import
if (
self.embedsubs and not self.embedonlyinternalsubs
): # Don't bother if we're not embeddeding subtitles and external subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except Exception:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
self.log.info("External %s subtitle file detected." % lang)
if self.swl is None or lang in self.swl:
self.log.info(
"Creating subtitle stream %s by importing %s."
% (l, fname)
)
subtitle_settings.update(
{
l: {
"path": os.path.join(dirName, fname),
"source": src,
"map": 0,
"codec": "mov_text",
"language": lang,
}
}
)
self.log.debug(
"Path: %s." % os.path.join(dirName, fname)
)
self.log.debug("Source: %s." % src)
self.log.debug("Codec: mov_text.")
self.log.debug("Langauge: %s." % lang)
l = l + 1
src = src + 1
self.deletesubs.add(os.path.join(dirName, fname))
else:
self.log.info(
"Ignoring %s external subtitle stream due to language %s."
% (fname, lang)
)
# Collect all options
options = {
"format": self.output_format,
"video": {
"codec": vcodec,
"map": info.video.index,
"bitrate": vbitrate,
"level": self.h264_level,
"profile": vprofile,
"pix_fmt": pix_fmt,
},
"audio": audio_settings,
"subtitle": subtitle_settings,
"preopts": [],
"postopts": ["-threads", self.threads],
}
# If a CRF option is set, override the determine bitrate
if self.vcrf:
del options["video"]["bitrate"]
options["video"]["crf"] = self.vcrf
if len(options["subtitle"]) > 0:
options["preopts"].append("-fix_sub_duration")
if self.preopts:
options["preopts"].extend(self.preopts)
if self.postopts:
options["postopts"].extend(self.postopts)
if (
self.dxva2_decoder
): # DXVA2 will fallback to CPU decoding when it hits a file that it cannot handle, so we don't need to check if the file is supported.
options["preopts"].extend(["-hwaccel", "dxva2"])
elif info.video.codec.lower() == "hevc" and self.hevc_qsv_decoder:
options["preopts"].extend(["-vcodec", "hevc_qsv"])
elif (
vcodec == "h264qsv"
and info.video.codec.lower() == "h264"
and self.qsv_decoder
and (info.video.video_level / 10) < 5
):
options["preopts"].extend(["-vcodec", "h264_qsv"])
# Add width option
if vwidth:
options["video"]["width"] = vwidth
# HEVC Tagging for copied streams
if info.video.codec.lower() in ["x265", "h265", "hevc"] and vcodec == "copy":
options["postopts"].extend(["-tag:v", "hvc1"])
self.log.info("Tagging copied video stream as hvc1")
self.options = options
return options
def generateOptions(self, inputfile, original=None):
# Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
# Video stream
self.log.info("Reading video stream.")
self.log.info("Video codec detected: %s." % info.video.codec)
try:
vbr = self.estimateVideoBitrate(info)
except:
vbr = info.format.bitrate / 1000
if info.video.codec.lower() in self.video_codec:
vcodec = "copy"
else:
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate if self.video_bitrate else vbr
self.log.info("Pix Fmt: %s." % info.video.pix_fmt)
if self.pix_fmt and info.video.pix_fmt.lower() not in self.pix_fmt:
vcodec = self.video_codec[0]
if self.video_bitrate is not None and vbr > self.video_bitrate:
self.log.debug("Overriding video bitrate. Codec cannot be copied because video bitrate is too high.")
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate
if self.video_width is not None and self.video_width < info.video.video_width:
self.log.debug(
"Video width is over the max width, it will be downsampled. Video stream can no longer be copied."
)
vcodec = self.video_codec[0]
vwidth = self.video_width
else:
vwidth = None
if self.h264_level and info.video.video_level and (info.video.video_level / 10 > self.h264_level):
self.log.info("Video level %0.1f." % (info.video.video_level / 10))
vcodec = self.video_codec[0]
self.log.debug("Video codec: %s." % vcodec)
self.log.debug("Video bitrate: %s." % vbitrate)
# Audio streams
self.log.info("Reading audio streams.")
overrideLang = True
for a in info.audio:
try:
if a.metadata["language"].strip() == "" or a.metadata["language"] is None:
a.metadata["language"] = "und"
except KeyError:
a.metadata["language"] = "und"
if (a.metadata["language"] == "und" and self.adl) or (
self.awl and a.metadata["language"].lower() in self.awl
):
overrideLang = False
break
if overrideLang:
self.awl = None
self.log.info(
"No audio streams detected in any appropriate language, relaxing restrictions so there will be some audio stream present."
)
audio_settings = {}
l = 0
for a in info.audio:
try:
if a.metadata["language"].strip() == "" or a.metadata["language"] is None:
a.metadata["language"] = "und"
except KeyError:
a.metadata["language"] = "und"
self.log.info("Audio detected for stream #%s: %s [%s]." % (a.index, a.codec, a.metadata["language"]))
# Set undefined language to default language if specified
if self.adl is not None and a.metadata["language"] == "und":
self.log.debug("Undefined language detected, defaulting to [%s]." % self.adl)
a.metadata["language"] = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.awl is None or a.metadata["language"].lower() in self.awl:
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS:
if a.audio_channels > 2:
iOSbitrate = 256 if (self.audio_bitrate * 2) > 256 else (self.audio_bitrate * 2)
self.log.info(
"Creating audio stream %s from source audio stream %s [iOS-audio]." % (str(l), a.index)
)
self.log.debug("Audio codec: %s." % self.iOS)
self.log.debug("Channels: 2.")
self.log.debug("Bitrate: %s." % iOSbitrate)
self.log.debug("Language: %s." % a.metadata["language"])
audio_settings.update(
{
l: {
"map": a.index,
"codec": self.iOS,
"channels": 2,
"bitrate": iOSbitrate,
"language": a.metadata["language"],
}
}
)
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
self.log.info("Creating audio stream %s from source stream %s." % (str(l), a.index))
if self.iOS and a.audio_channels <= 2:
self.log.debug(
"Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio]."
)
acodec = "copy" if a.codec == self.iOS else self.iOS
audio_channels = a.audio_channels
abitrate = (
a.audio_channels * 128
if (a.audio_channels * self.audio_bitrate) > (a.audio_channels * 128)
else (a.audio_channels * self.audio_bitrate)
)
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = "copy" if a.codec.lower() in self.audio_codec else self.audio_codec[0]
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == "copy":
acodec = self.audio_codec[0]
abitrate = self.maxchannels * self.audio_bitrate
else:
audio_channels = a.audio_channels
abitrate = a.audio_channels * self.audio_bitrate
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug("Attempting to set bitrate based on source stream bitrate.")
try:
abitrate = a.bitrate / 1000
except:
self.log.warning(
"Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel."
% a.index
)
abitrate = a.audio_channels * 256
self.log.debug("Audio codec: %s." % acodec)
self.log.debug("Channels: %s." % audio_channels)
self.log.debug("Bitrate: %s." % abitrate)
self.log.debug("Language: %s" % a.metadata["language"])
# If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed
if self.iOS and self.iOSFirst:
self.log.debug("Not creating any additional iOS audio streams.")
self.iOS = False
audio_settings.update(
{
l: {
"map": a.index,
"codec": acodec,
"channels": audio_channels,
"bitrate": abitrate,
"language": a.metadata["language"],
}
}
)
if acodec == "copy" and a.codec == "aac":
audio_settings[l]["bsf"] = "aac_adtstoasc"
l = l + 1
# Subtitle streams
subtitle_settings = {}
l = 0
self.log.info("Reading subtitle streams.")
for s in info.subtitle:
try:
if s.metadata["language"].strip() == "" or s.metadata["language"] is None:
s.metadata["language"] = "und"
except KeyError:
s.metadata["language"] = "und"
self.log.info("Subtitle detected for stream #%s: %s [%s]." % (s.index, s.codec, s.metadata["language"]))
# Set undefined language to default language if specified
if self.sdl is not None and s.metadata["language"] == "und":
self.log.debug("Undefined language detected, defaulting to [%s]." % self.sdl)
s.metadata["language"] = self.sdl
# Make sure its not an image based codec
if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.metadata["language"].lower() in self.swl:
subtitle_settings.update(
{
l: {
"map": s.index,
"codec": self.scodec[0],
"language": s.metadata["language"],
"encoding": self.subencoding,
# 'forced': s.sub_forced,
# 'default': s.sub_default
}
}
)
self.log.info("Creating subtitle stream %s from source stream %s." % (l, s.index))
l = l + 1
elif s.codec.lower() not in bad_subtitle_codecs and not self.embedsubs:
if self.swl is None or s.metadata["language"].lower() in self.swl:
for codec in self.scodec:
ripsub = {0: {"map": s.index, "codec": codec, "language": s.metadata["language"]}}
options = {"format": codec, "subtitle": ripsub}
try:
extension = subtitle_codec_extensions[codec]
except:
self.log.info("Wasn't able to determine subtitle file extension, defaulting to '.srt'.")
extension = "srt"
forced = ".forced" if s.sub_forced else ""
input_dir, filename, input_extension = self.parseFile(inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(
output_dir, filename + "." + s.metadata["language"] + forced + "." + extension
)
i = 2
while os.path.isfile(outputfile):
self.log.debug("%s exists, appending %s to filename." % (outputfile, i))
outputfile = os.path.join(
output_dir,
filename + "." + s.metadata["language"] + forced + "." + str(i) + "." + extension,
)
i += 1
try:
self.log.info(
"Ripping %s subtitle from source stream %s into external file."
% (s.metadata["language"], s.index)
)
conv = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).convert(
inputfile, outputfile, options, timeout=None
)
for timecode in conv:
pass
self.log.info("%s created." % outputfile)
except:
self.log.exception("Unabled to create external subtitle file for stream %s." % (s.index))
# Attempt to download subtitles if they are missing using subliminal
languages = set()
try:
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
self.log.error("No valid subtitle language specified, cannot download subtitles.")
except:
self.log.exception("Unable to verify subtitle languages for download.")
self.downloadsubs = False
if self.downloadsubs:
import subliminal
self.log.info("Attempting to download subtitles.")
# Attempt to set the dogpile cache
try:
subliminal.region.configure("dogpile.cache.memory")
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile), subtitles=True, embedded_subtitles=True)
subtitles = subliminal.download_best_subtitles(
[video], languages, hearing_impaired=False, providers=self.subproviders
)
try:
subliminal.save_subtitles(video, subtitles[video])
except:
# Support for older versions of subliminal
subliminal.save_subtitles(subtitles)
self.log.info("Please update to the latest version of subliminal.")
except Exception as e:
self.log.info("Unable to download subtitles.", exc_info=True)
self.log.debug("Unable to download subtitles.", exc_info=True)
# External subtitle import
if self.embedsubs: # Don't bother if we're not embeddeding any subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
self.log.info("External %s subtitle file detected." % lang)
if self.swl is None or lang in self.swl:
self.log.info("Creating subtitle stream %s by importing %s." % (l, fname))
subtitle_settings.update(
{
l: {
"path": os.path.join(dirName, fname),
"source": src,
"map": 0,
"codec": "mov_text",
"language": lang,
}
}
)
self.log.debug("Path: %s." % os.path.join(dirName, fname))
self.log.debug("Source: %s." % src)
self.log.debug("Codec: mov_text.")
self.log.debug("Langauge: %s." % lang)
l = l + 1
src = src + 1
self.deletesubs.add(os.path.join(dirName, fname))
else:
self.log.info(
"Ignoring %s external subtitle stream due to language %s." % (fname, lang)
)
# Collect all options
options = {
"format": self.output_format,
"video": {"codec": vcodec, "map": info.video.index, "bitrate": vbitrate, "level": self.h264_level},
"audio": audio_settings,
"subtitle": subtitle_settings,
"preopts": ["-fix_sub_duration"],
"postopts": ["-threads", self.threads],
}
# If using h264qsv, add the codec in front of the input for decoding
if (
vcodec == "h264qsv"
and info.video.codec.lower() == "h264"
and self.qsv_decoder
and (info.video.video_level / 10) < 5
):
options["preopts"].extend(["-vcodec", "h264_qsv"])
# Add width option
if vwidth:
options["video"]["width"] = vwidth
# Add pix_fmt
if self.pix_fmt:
options["video"]["pix_fmt"] = self.pix_fmt[0]
self.options = options
return options
def generateOptions(self, inputfile, original=None):
#Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
#Video stream
self.log.info("Reading video stream.")
self.log.info("Video codec detected: %s" % info.video.codec)
try:
vbr = self.estimateVideoBitrate(info)
except:
vbr = info.format.bitrate/1000
vcodec = 'copy' if info.video.codec.lower() in self.video_codec else self.video_codec[0]
vbitrate = vbr
self.log.info("Pix Fmt: %s" % info.video.pix_fmt)
if self.pix_fmt and self.pix_fmt.lower() != info.video.pix_fmt.lower():
vcodec = self.video_codec[0]
if self.video_bitrate is not None and vbr > self.video_bitrate:
self.log.debug("Overriding video bitrate. Codec cannot be copied because video bitrate is too high.")
vcodec = self.video_codec[0]
vbitrate = self.video_bitrate
if self.video_width is not None and self.video_width < info.video.video_width:
self.log.debug("Video width is over the max width, it will be downsampled. Video stream can no longer be copied.")
vcodec = self.video_codec[0]
vwidth = self.video_width
else:
vwidth = None
if self.h264_level and info.video.video_level and info.video.video_level/10 > self.h264_level:
self.log.info("Video level %0.1f." % info.video.video_level)
vcodec = self.video_codec[0]
self.log.debug("Video codec: %s" % vcodec)
self.log.debug("Video bitrate: %s" % vbitrate)
#Audio streams
self.log.info("Reading audio streams.")
audio_settings = {}
l = 0
for a in info.audio:
try:
if a.metadata['language'].strip() == "" or a.metadata['language'] is None:
a.metadata['language'] = 'und'
except KeyError:
a.metadata['language'] = 'und'
self.log.info("Audio detected for stream #%s: %s [%s]." % (a.index, a.codec, a.metadata['language']))
# Set undefined language to default language if specified
if self.adl is not None and a.metadata['language'] == 'und':
self.log.debug("Undefined language detected, defaulting to [%s]." % self.adl)
a.metadata['language'] = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.awl is None or a.metadata['language'].lower() in self.awl:
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS:
if a.audio_channels > 2:
self.log.info("Creating audio stream %s from source audio stream %s [iOS-audio]." % (str(l), a.index))
self.log.debug("Audio codec: %s." % self.iOS)
self.log.debug("Channels: 2.")
self.log.debug("Bitrate: 256.")
self.log.debug("Language: %s" % a.metadata['language'])
audio_settings.update({l: {
'map': a.index,
'codec': self.iOS,
'channels': 2,
'bitrate': 256,
'language': a.metadata['language'],
}})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
self.log.info("Creating audio stream %s from source stream %s." % (str(l), a.index))
if self.iOS and a.audio_channels <= 2:
self.log.debug("Overriding default channel settings because iOS audio is enabled but the source is stereo [iOS-audio].")
acodec = 'copy' if a.codec == self.iOS else self.iOS
audio_channels = a.audio_channels
abitrate = a.audio_channels * 128
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = 'copy' if a.codec.lower() in self.audio_codec else self.audio_codec[0]
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == 'copy':
acodec = self.audio_codec[0]
abitrate = self.maxchannels * self.audio_bitrate
else:
audio_channels = a.audio_channels
abitrate = a.audio_channels * self.audio_bitrate
# Bitrate calculations/overrides
if self.audio_bitrate is 0:
self.log.debug("Attempting to set bitrate based on source stream bitrate.")
try:
abitrate = a.bitrate/1000
except:
self.log.warning("Unable to determine audio bitrate from source stream %s, defaulting to 256 per channel." % a.index)
abitrate = a.audio_channels * 256
self.log.debug("Audio codec: %s." % acodec)
self.log.debug("Channels: %s." % audio_channels)
self.log.debug("Bitrate: %s." % abitrate)
self.log.debug("Language: %s" % a.metadata['language'])
# If the iOSFirst option is enabled, disable the iOS option after the first audio stream is processed
if self.iOS and self.iOSFirst:
self.log.debug("Not creating any additional iOS audio streams.")
self.iOS = False
audio_settings.update({l: {
'map': a.index,
'codec': acodec,
'channels': audio_channels,
'bitrate': abitrate,
'language': a.metadata['language'],
}})
l = l + 1
# Subtitle streams
subtitle_settings = {}
l = 0
self.log.info("Reading subtitle streams.")
for s in info.subtitle:
try:
if s.metadata['language'].strip() == "" or s.metadata['language'] is None:
s.metadata['language'] = 'und'
except KeyError:
s.metadata['language'] = 'und'
self.log.info("Subtitle detected for stream #%s: %s [%s]." % (s.index, s.codec, s.metadata['language']))
# Set undefined language to default language if specified
if self.sdl is not None and s.metadata['language'] == 'und':
self.log.debug("Undefined language detected, defaulting to [%s]." % self.sdl)
s.metadata['language'] = self.sdl
# Make sure its not an image based codec
if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.metadata['language'].lower() in self.swl:
subtitle_settings.update({l: {
'map': s.index,
'codec': self.scodec,
'language': s.metadata['language']
#'forced': s.sub_forced,
#'default': s.sub_default
}})
self.log.info("Creating subtitle stream %s from source stream %s." % (l, s.index))
l = l + 1
elif s.codec.lower() not in bad_subtitle_codecs and not self.embedsubs:
if self.swl is None or s.metadata['language'].lower() in self.swl:
ripsub = {0: {
'map': s.index,
'codec': self.scodec,
'language': s.metadata['language']
}}
options = {
'format': self.scodec,
'subtitle': ripsub,
}
try:
extension = subtitle_codec_extensions[self.scodec]
except:
self.log.info("Wasn't able to determine subtitle file extension, defaulting to '.srt'.")
extension = 'srt'
input_dir, filename, input_extension = self.parseFile(inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + "." + extension)
i = 2
while os.path.isfile(outputfile):
self.log.debug("%s exists, appending %s to filename." % (outputfile, i))
outputfile = os.path.join(output_dir, filename + "." + s.metadata['language'] + "." + str(i) + "." + extension)
i += 1
try:
self.log.info("Ripping %s subtitle from source stream %s into external file." % (s.metadata['language'], s.index))
conv = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).convert(inputfile, outputfile, options, timeout=None)
for timecode in conv:
pass
self.log.info("%s created." % outputfile)
except:
self.log.exception("Unabled to create external subtitle file for stream %s." % (s.index))
# Attempt to download subtitles if they are missing using subliminal
languages = set()
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
if self.downloadsubs:
import subliminal
self.log.info("Attempting to download subtitles.")
try:
subliminal.cache_region.configure('dogpile.cache.memory')
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile.decode(sys.getfilesystemencoding())), subtitles=True, embedded_subtitles=True, original=original)
subtitles = subliminal.download_best_subtitles([video], languages, hearing_impaired=False, providers=self.subproviders)
subliminal.save_subtitles(subtitles)
except Exception as e:
self.log.debug("Unable to download subtitles.", exc_info=True)
# External subtitle import
if self.embedsubs: #Don't bother if we're not embeddeding any subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
self.log.info("External %s subtitle file detected." % lang)
if self.swl is None or lang in self.swl:
self.log.info("Creating subtitle stream %s by importing %s." % (l, fname))
subtitle_settings.update({l: {
'path': os.path.join(dirName, fname),
'source': src,
'map': 0,
'codec': 'mov_text',
'language': lang,
}})
self.log.debug("Path: %s." % os.path.join(dirName, fname))
self.log.debug("Source: %s." % src)
self.log.debug("Codec: mov_text.")
self.log.debug("Langauge: %s." % lang)
l = l + 1
src = src + 1
self.deletesubs.add(os.path.join(dirName, fname))
else:
self.log.info("Ignoring %s external subtitle stream due to language %s." % (fname, lang))
# Collect all options
options = {
'format': self.output_format,
'video': {
'codec': vcodec,
'map': info.video.index,
'bitrate': vbitrate,
'level': self.h264_level
},
'audio': audio_settings,
'subtitle': subtitle_settings,
}
# Add width option
if vwidth: options['video']['width'] = vwidth
# Add pix_fmt
if self.pix_fmt: options['video']['pix_fmt'] = self.pix_fmt
self.options = options
return options
class SubsCenterProvider(Provider):
"""SubsCenter Provider."""
languages = {Language.fromalpha2(l) for l in ['he']}
server_url = 'http://www.subscenter.info/he/'
def __init__(self, username=None, password=None):
if username is not None and password is None or username is None and password is not None:
raise ConfigurationError('Username and password must be specified')
self.session = None
self.username = username
self.password = password
self.logged_in = False
def initialize(self):
self.session = Session()
self.session.headers['User-Agent'] = 'Subliminal/{}'.format(
__short_version__)
# login
if self.username is not None and self.password is not None:
logger.debug('Logging in')
url = self.server_url + 'subscenter/accounts/login/'
# retrieve CSRF token
self.session.get(url)
csrf_token = self.session.cookies['csrftoken']
# actual login
data = {
'username': self.username,
'password': self.password,
'csrfmiddlewaretoken': csrf_token
}
r = self.session.post(url, data, allow_redirects=False, timeout=10)
if r.status_code != 302:
raise AuthenticationError(self.username)
logger.info('Logged in')
self.logged_in = True
def terminate(self):
# logout
if self.logged_in:
logger.info('Logging out')
r = self.session.get(self.server_url +
'subscenter/accounts/logout/',
timeout=10)
r.raise_for_status()
logger.info('Logged out')
self.logged_in = False
self.session.close()
@region.cache_on_arguments(expiration_time=SHOW_EXPIRATION_TIME)
def _search_url_titles(self, title):
"""Search the URL titles by kind for the given `title`.
:param str title: title to search for.
:return: the URL titles by kind.
:rtype: collections.defaultdict
"""
# make the search
logger.info('Searching title name for %r', title)
r = self.session.get(self.server_url + 'subtitle/search/',
params={'q': title},
timeout=10)
r.raise_for_status()
# check for redirections
if r.history and all([h.status_code == 302 for h in r.history]):
logger.debug('Redirected to the subtitles page')
links = [r.url]
else:
# get the suggestions (if needed)
soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser'])
links = [
link.attrs['href']
for link in soup.select('#processes div.generalWindowTop a')
]
logger.debug('Found %d suggestions', len(links))
url_titles = defaultdict(list)
for link in links:
parts = link.split('/')
url_titles[parts[-3]].append(parts[-2])
return url_titles
def query(self, title, season=None, episode=None):
# search for the url title
url_titles = self._search_url_titles(title)
# episode
if season and episode:
if 'series' not in url_titles:
logger.error('No URL title found for series %r', title)
return []
url_title = url_titles['series'][0]
logger.debug('Using series title %r', url_title)
url = self.server_url + 'cst/data/series/sb/{}/{}/{}/'.format(
url_title, season, episode)
page_link = self.server_url + 'subtitle/series/{}/{}/{}/'.format(
url_title, season, episode)
else:
if 'movie' not in url_titles:
logger.error('No URL title found for movie %r', title)
return []
url_title = url_titles['movie'][0]
logger.debug('Using movie title %r', url_title)
url = self.server_url + 'cst/data/movie/sb/{}/'.format(url_title)
page_link = self.server_url + 'subtitle/movie/{}/'.format(
url_title)
# get the list of subtitles
logger.debug('Getting the list of subtitles')
r = self.session.get(url)
r.raise_for_status()
results = json.loads(r.text)
# loop over results
subtitles = {}
for language_code, language_data in results.items():
for quality_data in language_data.values():
for quality, subtitles_data in quality_data.items():
for subtitle_item in subtitles_data.values():
# read the item
language = Language.fromalpha2(language_code)
hearing_impaired = bool(
subtitle_item['hearing_impaired'])
subtitle_id = subtitle_item['id']
subtitle_key = subtitle_item['key']
subtitle_version = subtitle_item['h_version']
downloaded = subtitle_item['downloaded']
release = subtitle_item['subtitle_version']
# add the release and increment downloaded count if we already have the subtitle
if subtitle_id in subtitles:
logger.debug(
'Found additional release %r for subtitle %d',
release, subtitle_id)
bisect.insort_left(subtitles[subtitle_id].releases,
release) # deterministic order
subtitles[subtitle_id].downloaded += downloaded
continue
# otherwise create it
subtitle = SubsCenterSubtitle(
language, hearing_impaired, page_link, title,
season, episode, title, subtitle_id, subtitle_key,
subtitle_version, downloaded, [release])
logger.debug('Found subtitle %r', subtitle)
subtitles[subtitle_id] = subtitle
return subtitles.values()
def list_subtitles(self, video, languages):
season = episode = None
title = video.title
if isinstance(video, Episode):
title = video.series
season = video.season
episode = video.episode
return [
s for s in self.query(title, season, episode)
if s.language in languages
]
def download_subtitle(self, subtitle):
# download
url = self.server_url + 'subtitle/download/{}/{}/'.format(
subtitle.language.alpha2, subtitle.subtitle_id)
params = {'v': subtitle.subtitle_version, 'key': subtitle.subtitle_key}
r = self.session.get(url,
params=params,
headers={'Referer': subtitle.page_link},
timeout=10)
r.raise_for_status()
# open the zip
try:
with zipfile.ZipFile(io.BytesIO(r.content)) as zf:
# remove some filenames from the namelist
namelist = [n for n in zf.namelist() if not n.endswith('.txt')]
if len(namelist) > 1:
raise ProviderError('More than one file to unzip')
subtitle.content = fix_line_ending(zf.read(namelist[0]))
except zipfile.BadZipfile:
# if no zip file was retrieved, daily downloads limit has exceeded
raise ProviderError('Daily limit exceeded')
class ArgenteamProvider(Provider):
provider_name = 'argenteam'
language = Language.fromalpha2('es')
languages = {language}
video_types = (Episode, )
server_url = "http://argenteam.net/api/v1/"
subtitle_class = ArgenteamSubtitle
def __init__(self):
self.session = None
def initialize(self):
self.session = Session()
self.session.headers['User-Agent'] = self.user_agent
def terminate(self):
self.session.close()
@region.cache_on_arguments(expiration_time=EPISODE_EXPIRATION_TIME,
should_cache_fn=lambda value: value)
def search_episode_id(self, series, season, episode):
"""Search the episode id from the `series`, `season` and `episode`.
:param str series: series of the episode.
:param int season: season of the episode.
:param int episode: episode number.
:return: the episode id, if any.
:rtype: int or None
"""
# make the search
query = '%s S%#02dE%#02d' % (series, season, episode)
logger.info('Searching episode id for %r', query)
r = self.session.get(self.server_url + 'search',
params={'q': query},
timeout=10)
r.raise_for_status()
results = json.loads(r.text)
if results['total'] == 1:
return results['results'][0]['id']
logger.error('No episode id found for %r', series)
def query(self, series, season, episode):
episode_id = self.search_episode_id(series, season, episode)
if episode_id is None:
return []
response = self.session.get(self.server_url + 'episode',
params={'id': episode_id},
timeout=10)
response.raise_for_status()
content = json.loads(response.text)
subtitles = []
for r in content['releases']:
for s in r['subtitles']:
subtitle = self.subtitle_class(self.language, s['uri'], series,
season, episode, r['team'],
r['tags'])
logger.debug('Found subtitle %r', subtitle)
subtitles.append(subtitle)
return subtitles
def list_subtitles(self, video, languages):
titles = [video.series] + video.alternative_series
for title in titles:
subs = self.query(title, video.season, video.episode)
if subs:
return subs
return []
def download_subtitle(self, subtitle):
# download as a zip
logger.info('Downloading subtitle %r', subtitle)
r = self.session.get(subtitle.download_link, timeout=10)
r.raise_for_status()
# open the zip
with ZipFile(io.BytesIO(r.content)) as zf:
if len(zf.namelist()) > 1:
raise ProviderError('More than one file to unzip')
subtitle.content = fix_line_ending(zf.read(zf.namelist()[0]))
def generateOptions(self, inputfile, original=None):
#Get path information from the input file
input_dir, filename, input_extension = self.parseFile(inputfile)
info = Converter(self.FFMPEG_PATH, self.FFPROBE_PATH).probe(inputfile)
#Video stream
print "Video codec detected: " + info.video.codec
vcodec = 'copy' if info.video.codec in self.video_codec else self.video_codec[
0]
if self.video_bitrate is not None:
vbitrate = self.video_bitrate
else:
vbitrate = info.format.bitrate
#Audio streams
audio_settings = {}
l = 0
for a in info.audio:
print "Audio stream detected: " + a.codec + " " + a.language + " [Stream " + str(
a.index) + "]"
# Set undefined language to default language if specified
if self.adl is not None and a.language == 'und':
print "Undefined language detected, defaulting to " + self.adl
a.language = self.adl
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.awl is None or a.language in self.awl:
# Create iOS friendly audio stream if the default audio stream has too many channels (iOS only likes AAC stereo)
if self.iOS:
if a.audio_channels > 2:
print "Creating dual audio channels for iOS compatability for this stream"
audio_settings.update({
l: {
'map': a.index,
'codec': 'aac',
'channels': 2,
'bitrate': 256,
'language': a.language,
}
})
l += 1
# If the iOS audio option is enabled and the source audio channel is only stereo, the additional iOS channel will be skipped and a single AAC 2.0 channel will be made regardless of codec preference to avoid multiple stereo channels
if self.iOS and a.audio_channels <= 2:
acodec = 'copy' if a.codec == 'aac' else self.iOS
else:
# If desired codec is the same as the source codec, copy to avoid quality loss
acodec = 'copy' if a.codec in self.audio_codec else self.audio_codec[
0]
# Audio channel adjustments
if self.maxchannels and a.audio_channels > self.maxchannels:
audio_channels = self.maxchannels
if acodec == 'copy':
acodec = self.audio_codec[0]
else:
audio_channels = a.audio_channels
# Bitrate calculations/overrides
if self.audio_bitrate is None or self.audio_bitrate > (
a.audio_channels * 256):
abitrate = 256 * audio_channels
else:
abitrate = self.audio_bitrate
audio_settings.update({
l: {
'map': a.index,
'codec': acodec,
'channels': audio_channels,
'bitrate': abitrate,
'language': a.language,
}
})
l = l + 1
# Subtitle streams
subtitle_settings = {}
l = 0
for s in info.subtitle:
print "Subtitle stream detected: " + s.codec + " " + s.language + " [Stream " + str(
s.index) + "]"
# Set undefined language to default language if specified
if self.sdl is not None and s.language == 'und':
s.language = self.sdl
# Make sure its not an image based codec
if s.codec.lower() not in bad_subtitle_codecs and self.embedsubs:
# Proceed if no whitelist is set, or if the language is in the whitelist
if self.swl is None or s.language in self.swl:
subtitle_settings.update({
l: {
'map': s.index,
'codec': 'mov_text',
'language': s.language
#'forced': s.sub_forced,
#'default': s.sub_default
}
})
l = l + 1
elif s.codec.lower(
) not in bad_subtitle_codecs and not self.embedsubs:
if self.swl is None or s.language in self.swl:
ripsub = {
1: {
'map': s.index,
'codec': 'srt',
'language': s.language
}
}
options = {
'format': 'srt',
'subtitle': ripsub,
}
input_dir, filename, input_extension = self.parseFile(
inputfile)
output_dir = input_dir if self.output_dir is None else self.output_dir
outputfile = os.path.join(
output_dir, filename + "." + s.language + ".srt")
i = 2
while os.path.isfile(outputfile):
outputfile = os.path.join(
output_dir, filename + "." + s.language + "." +
str(i) + ".srt")
i += i
print "Ripping " + s.language + " subtitle from file"
conv = Converter(self.FFMPEG_PATH,
self.FFPROBE_PATH).convert(inputfile,
outputfile,
options,
timeout=None)
for timecode in conv:
pass
try:
print outputfile + " created"
except:
print "File created"
# Attempt to download subtitles if they are missing using subliminal
languages = set()
if self.swl:
for alpha3 in self.swl:
languages.add(Language(alpha3))
elif self.sdl:
languages.add(Language(self.sdl))
else:
self.downloadsubs = False
if self.downloadsubs:
import subliminal
print "Attempting to download subtitles, please wait"
try:
subliminal.cache_region.configure('dogpile.cache.memory')
except:
pass
try:
video = subliminal.scan_video(os.path.abspath(inputfile),
subtitles=True,
embedded_subtitles=True,
original=original)
subtitles = subliminal.download_best_subtitles(
[video],
languages,
hearing_impaired=False,
providers=self.subproviders)
subliminal.save_subtitles(subtitles)
except Exception as e:
print e
print "Unable to download subtitle"
# External subtitle import
if self.embedsubs: #Don't bother if we're not embeddeding any subtitles
src = 1 # FFMPEG input source number
for dirName, subdirList, fileList in os.walk(input_dir):
for fname in fileList:
subname, subextension = os.path.splitext(fname)
# Watch for appropriate file extension
if subextension[1:] in valid_subtitle_extensions:
x, lang = os.path.splitext(subname)
lang = lang[1:]
# Using bablefish to convert a 2 language code to a 3 language code
if len(lang) is 2:
try:
babel = Language.fromalpha2(lang)
lang = babel.alpha3
except:
pass
# If subtitle file name and input video name are the same, proceed
if x == filename:
print "External subtitle file detected, language " + lang
if self.swl is None or lang in self.swl:
print "Importing %s subtitle stream" % (fname)
subtitle_settings.update({
l: {
'path': os.path.join(dirName, fname),
'source': src,
'map': 0,
'codec': 'mov_text',
'language': lang,
}
})
l = l + 1
src = src + 1
self.deletesubs.add(
os.path.join(dirName, fname))
else:
print "Ignoring %s external subtitle stream due to language: %s" % (
fname, lang)
# Collect all options
options = {
'format': self.output_format,
'video': {
'codec': vcodec,
'map': info.video.index,
'bitrate': vbitrate
},
'audio': audio_settings,
'subtitle': subtitle_settings,
}
self.options = options
return options
def test_get_matches_no_match(episodes):
subtitle = ArgenteamSubtitle(Language.fromalpha2('es'), None,
'Marvels Agents Of S.H.I.E.L.D.', 2, 6,
'KILLERS', '1080p')
matches = subtitle.get_matches(episodes['house_of_cards_us_s06e01'])
assert matches == set()
