class OpenSubtitles(SubtitleProvider):
URL = 'http://api.opensubtitles.org/xml-rpc'
def __init__(self):
SubtitleProvider.__init__(self)
self._xmlrpc = None
self._token = None
self._settings = OpenSubtitlesSettings()
def get_settings(self):
return self._settings
def set_settings(self, settings):
self._settings = settings
def connect(self):
log.debug('connect()')
if self.connected():
return
self._xmlrpc = ServerProxy(self.URL, allow_none=False)
def disconnect(self):
log.debug('disconnect()')
if self.logged_in():
self.logout()
if self.connected():
self._xmlrpc = None
def connected(self):
return self._xmlrpc is not None
def login(self):
log.debug('login()')
if self.logged_in():
return
if not self.connected():
raise ProviderNotConnectedError()
def login_query():
# FIXME: 'en' language ok??? or '' as in the original
return self._xmlrpc.LogIn(str(self._settings.username),
str(self._settings.password), 'en',
str(self._settings.get_user_agent()))
result = self._safe_exec(login_query, None)
self.check_result(result)
self._token = result['token']
def logout(self):
log.debug('logout()')
if self.logged_in():
def logout_query():
return self._xmlrpc.LogOut(self._token)
result = self._safe_exec(logout_query, None)
self.check_result(result)
self._token = None
def logged_in(self):
return self._token is not None
SEARCH_LIMIT = 500
def search_videos(self, videos, callback, languages=None):
log.debug('search_videos(#videos={})'.format(len(videos)))
if not self.logged_in():
raise ProviderNotConnectedError()
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(
window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc.SearchSubtitles(
self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(
remote_language_raw, xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(
rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitlesSubtitleFile(
filename=remote_filename,
file_size=remote_file_size,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
age=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
imdb_identity = ImdbIdentity(imdb_id=imdb_id,
imdb_rating=None)
identity = ProviderIdentities(imdb_identity=imdb_identity,
provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception(
'Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(
queries=queries))
log.error('Offending result is: {remote_sub}'.format(
remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def query_text(self, query):
return OpenSubtitlesTextQuery(query=query)
def download_subtitles(self, os_rsubs):
log.debug('download_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(
window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({
item['idsubtitlefile']: item['data']
for item in result['data']
})
subtitles = [
unzip_bytes(base64.b64decode(
map_id_data[os_rsub.get_id_online()])).read()
for os_rsub in os_rsubs
]
return subtitles
def ping(self):
log.debug('ping()')
if not self.logged_in():
raise ProviderNotConnectedError()
def run_query():
return self._xmlrpc.NoOperation(self._token)
result = self._safe_exec(run_query, None)
self.check_result(result)
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
@classmethod
def get_name(cls):
return 'opensubtitles'
@classmethod
def get_short_name(cls):
return 'os'
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
try:
result = query()
return result
except (ProtocolError, CannotSendRequest, SocketError):
self._signal_connection_failed()
log.warning('Query failed', exc_info=sys.exc_info())
return default
STATUS_CODE_RE = re.compile('(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise ProviderConnectionError(_('No message'))
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(
status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(
_('Server returned status="{status}". Expected "200 OK".').
format(status=data['status']), data['status'])
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(message, code)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')
class OpenSubtitles(SubtitleProvider):
URL = 'http://api.opensubtitles.org/xml-rpc'
def __init__(self, settings=None):
SubtitleProvider.__init__(self)
self._xmlrpc = None
self._token = None
self._last_time = None
if settings is None:
settings = OpenSubtitlesSettings()
self._settings = settings
def get_settings(self):
return self._settings
def set_settings(self, settings):
if self.connected():
raise RuntimeError(
'Cannot set settings while connected') # FIXME: change error
self._settings = settings
def connect(self):
log.debug('connect()')
if self.connected():
return
self._xmlrpc = ServerProxy(self.URL, allow_none=False)
self._last_time = time.time()
def disconnect(self):
log.debug('disconnect()')
if self.logged_in():
self.logout()
if self.connected():
self._xmlrpc = None
def connected(self):
return self._xmlrpc is not None
def login(self):
log.debug('login()')
if self.logged_in():
return
if not self.connected():
self.connect()
def login_query():
# FIXME: 'en' language ok??? or '' as in the original
return self._xmlrpc.LogIn(str(self._settings.username),
str(self._settings.password), 'en',
str(self._settings.get_user_agent()))
result = self._safe_exec(login_query, None)
self.check_result(result)
self._token = result['token']
def logout(self):
log.debug('logout()')
if self.logged_in():
def logout_query():
return self._xmlrpc.LogOut(self._token)
# Do no check result of this call. Assume connection closed.
self._safe_exec(logout_query, None)
self._token = None
def logged_in(self):
return self._token is not None
def reestablish(self):
log.debug('reestablish()')
connected = self.connected()
logged_in = self.logged_in()
self.disconnect()
if connected:
self.connect()
if logged_in:
self.login()
_TIMEOUT_MS = 60000
def _ensure_connection(self):
now = time.time()
if now - time.time() > self._TIMEOUT_MS:
self.reestablish()
self._last_time = now
SEARCH_LIMIT = 500
def search_videos(self, videos, callback, languages=None):
log.debug('search_videos(#videos={})'.format(len(videos)))
if not self.logged_in():
raise ProviderNotConnectedError()
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(
window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
if video.get_osdb_hash() is None:
log.debug('osdb hash of "{}" is empty -> skip'.format(
video.get_filepath()))
self._signal_connection_failed(
) # FIXME: other name + general signaling
continue
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc.SearchSubtitles(
self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(
remote_language_raw, xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(
rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitlesSubtitleFile(
filename=remote_filename,
file_size=remote_file_size,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
date=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
try:
imdb_rating = float(rsub_raw['MovieImdbRating'])
except (ValueError, KeyError):
imdb_rating = None
imdb_identity = ImdbIdentity(imdb_id=imdb_id,
imdb_rating=imdb_rating)
video_name = rsub_raw['MovieName']
try:
video_year = int(rsub_raw['MovieYear'])
except (ValueError, KeyError):
video_year = None
video_identity = VideoIdentity(name=video_name,
year=video_year)
try:
series_season = int(rsub_raw['SeriesSeason'])
except (KeyError, ValueError):
series_season = None
try:
series_episode = int(rsub_raw['SeriesEpisode'])
except (KeyError, ValueError):
series_episode = None
series_identity = SeriesIdentity(season=series_season,
episode=series_episode)
identity = ProviderIdentities(
video_identity=video_identity,
imdb_identity=imdb_identity,
episode_identity=series_identity,
provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception(
'Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(
queries=queries))
log.error('Offending result is: {remote_sub}'.format(
remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def query_text(self, query):
return OpenSubtitlesTextQuery(query=query)
def download_subtitles(self, os_rsubs):
log.debug('download_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(
window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({
item['idsubtitlefile']: item['data']
for item in result['data']
})
subtitles = [
unzip_bytes(base64.b64decode(
map_id_data[os_rsub.get_id_online()])).read()
for os_rsub in os_rsubs
]
return subtitles
def upload_subtitles(self, local_movie):
log.debug('upload_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
video_subtitles = list(local_movie.iter_video_subtitles())
if not video_subtitles:
return UploadResult(
type=UploadResult.Type.MISSINGDATA,
reason=_('Need at least one subtitle to upload'))
query_try = dict()
for sub_i, (video, subtitle) in enumerate(video_subtitles):
if not video:
return UploadResult(
type=UploadResult.Type.MISSINGDATA,
reason=_('Each subtitle needs an accompanying video'))
query_try['cd{}'.format(sub_i + 1)] = {
'subhash':
subtitle.get_md5_hash(),
'subfilename':
subtitle.get_filename(),
'moviehash':
video.get_osdb_hash(),
'moviebytesize':
str(video.get_size()),
'moviefps':
str(video.get_fps()) if video.get_fps() else None,
'movieframes':
str(video.get_framecount())
if video.get_framecount() else None,
'moviefilename':
video.get_filename(),
}
def run_query_try_upload():
return self._xmlrpc.TryUploadSubtitles(self._token, query_try)
try_result = self._safe_exec(run_query_try_upload, None)
self.check_result(try_result)
if int(try_result['alreadyindb']):
return UploadResult(type=UploadResult.Type.DUPLICATE,
reason=_('Subtitle is already in database'))
if local_movie.get_imdb_id() is None:
return UploadResult(type=UploadResult.Type.MISSINGDATA,
reason=_('Need IMDb id'))
upload_base_info = {
'idmovieimdb': local_movie.get_imdb_id(),
}
if local_movie.get_comments() is not None:
upload_base_info['subauthorcomment'] = local_movie.get_comments()
if not local_movie.get_language().is_generic():
upload_base_info['sublanguageid'] = local_movie.get_language().xxx(
)
if local_movie.get_release_name() is not None:
upload_base_info[
'moviereleasename'] = local_movie.get_release_name()
if local_movie.get_movie_name() is not None:
upload_base_info['movieaka'] = local_movie.get_movie_name()
if local_movie.is_hearing_impaired() is not None:
upload_base_info[
'hearingimpaired'] = local_movie.is_hearing_impaired()
if local_movie.is_high_definition() is not None:
upload_base_info[
'highdefinition'] = local_movie.is_high_definition()
if local_movie.is_automatic_translation() is not None:
upload_base_info[
'automatictranslation'] = local_movie.is_automatic_translation(
)
if local_movie.get_author() is not None:
upload_base_info['subtranslator'] = local_movie.get_author()
if local_movie.is_foreign_only() is not None:
upload_base_info['foreignpartsonly'] = local_movie.is_foreign_only(
)
query_upload = {
'baseinfo': upload_base_info,
}
for sub_i, (video, subtitle) in enumerate(video_subtitles):
sub_bytes = subtitle.get_filepath().open(mode='rb').read()
sub_tx_data = base64.b64encode(zlib.compress(sub_bytes)).decode()
query_upload['cd{}'.format(sub_i + 1)] = {
'subhash':
subtitle.get_md5_hash(),
'subfilename':
subtitle.get_filename(),
'moviehash':
video.get_osdb_hash(),
'moviebytesize':
str(video.get_size()),
'movietimems':
str(video.get_time_ms()) if video.get_time_ms() else None,
'moviefps':
str(video.get_fps()) if video.get_fps() else None,
'movieframes':
str(video.get_framecount())
if video.get_framecount() else None,
'moviefilename':
video.get_filename(),
'subcontent':
sub_tx_data,
}
def run_query_upload():
return self._xmlrpc.UploadSubtitles(self._token, query_upload)
result = self._safe_exec(run_query_upload, None)
self.check_result(result)
rsubs = []
for sub_data in result['data']:
filename = sub_data['SubFileName']
file_size = sub_data['SubSize']
md5_hash = sub_data['SubHash']
id_online = sub_data['IDSubMOvieFile']
download_link = sub_data['SubDownloadLink']
link = None
uploader = sub_data['UserNickName']
language = Language.from_xxx(sub_data['SubLanguageID'])
rating = float(sub_data['SubRating'])
add_date = datetime.datetime.strptime(sub_data['SubAddDate'],
'%Y-%m-%d %H:%M:%S')
sub = OpenSubtitlesSubtitleFile(filename=filename,
file_size=file_size,
md5_hash=md5_hash,
id_online=id_online,
download_link=download_link,
link=link,
uploader=uploader,
language=language,
rating=rating,
date=add_date)
rsubs.append(sub)
return UploadResult(type=UploadResult.Type.OK, rsubs=rsubs)
def imdb_search_title(self, title):
self._ensure_connection()
def run_query():
return self._xmlrpc.SearchMoviesOnIMDB(self._token, title.strip())
result = self._safe_exec(run_query, default=None)
self.check_result(result)
imdbs = []
re_title = re.compile(r'(?P<title>.*) \((?P<year>[0-9]+)\)')
for imdb_data in result['data']:
imdb_id = imdb_data['id']
if all(c in string.digits for c in imdb_id):
imdb_id = 'tt{}'.format(imdb_id)
m = re_title.match(imdb_data['title'])
if m:
imdb_title = m['title']
imdb_year = int(m['year'])
else:
imdb_title = imdb_data['title']
imdb_year = None
imdbs.append(
ImdbMovieMatch(imdb_id=imdb_id,
title=imdb_title,
year=imdb_year))
return imdbs
def ping(self):
log.debug('ping()')
if not self.logged_in():
raise ProviderNotConnectedError()
def run_query():
return self._xmlrpc.NoOperation(self._token)
result = self._safe_exec(run_query, None)
self.check_result(result)
def provider_info(self):
if self.connected():
def run_query():
return self._xmlrpc.ServerInfo()
result = self._safe_exec(run_query, None)
data = [
(_('XML-RPC version'), result['xmlrpc_version']),
(_('XML-RPC url'), result['xmlrpc_url']),
(_('Application'), result['application']),
(_('Contact'), result['contact']),
(_('Website url'), result['website_url']),
(_('Users online'), result['users_online_total']),
(_('Programs online'), result['users_online_program']),
(_('Users logged in'), result['users_loggedin']),
(_('Max users online'), result['users_max_alltime']),
(_('Users registered'), result['users_registered']),
(_('Subtitles downloaded'), result['subs_downloads']),
(_('Subtitles available'), result['subs_subtitle_files']),
(_('Number movies'), result['movies_total']),
(_('Number languages'), result['total_subtitles_languages']),
(_('Client IP'), result['download_limits']['client_ip']),
(_('24h global download limit'),
result['download_limits']['global_24h_download_limit']),
(_('24h client download limit'),
result['download_limits']['client_24h_download_limit']),
(_('24h client download count'),
result['download_limits']['client_24h_download_count']),
(_('Client download quota'),
result['download_limits']['client_download_quota']),
]
else:
data = []
return data
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
@classmethod
def get_name(cls):
return 'opensubtitles'
@classmethod
def get_short_name(cls):
return 'os'
@classmethod
def get_icon(cls):
return ':/images/sites/opensubtitles.png'
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
self._ensure_connection()
try:
result = query()
return result
except (ProtocolError, CannotSendRequest, SocketError,
ExpatError) as e:
self._signal_connection_failed()
log.debug('Query failed: {} {}'.format(type(e), e.args))
return default
STATUS_CODE_RE = re.compile(r'(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise OpenSubtitlesProviderConnectionError(None, _('No message'))
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(
status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise OpenSubtitlesProviderConnectionError(
None,
_('Server returned status="{status}". Expected "200 OK".').
format(status=data['status']), data['status'])
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise OpenSubtitlesProviderConnectionError(code, message)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')