class OpenSubtitles(SubtitleProvider): URL = 'http://api.opensubtitles.org/xml-rpc' def __init__(self): SubtitleProvider.__init__(self) self._xmlrpc = None self._token = None self._settings = OpenSubtitlesSettings() def get_settings(self): return self._settings def set_settings(self, settings): self._settings = settings def connect(self): log.debug('connect()') if self.connected(): return self._xmlrpc = ServerProxy(self.URL, allow_none=False) def disconnect(self): log.debug('disconnect()') if self.logged_in(): self.logout() if self.connected(): self._xmlrpc = None def connected(self): return self._xmlrpc is not None def login(self): log.debug('login()') if self.logged_in(): return if not self.connected(): raise ProviderNotConnectedError() def login_query(): # FIXME: 'en' language ok??? or '' as in the original return self._xmlrpc.LogIn(str(self._settings.username), str(self._settings.password), 'en', str(self._settings.get_user_agent())) result = self._safe_exec(login_query, None) self.check_result(result) self._token = result['token'] def logout(self): log.debug('logout()') if self.logged_in(): def logout_query(): return self._xmlrpc.LogOut(self._token) result = self._safe_exec(logout_query, None) self.check_result(result) self._token = None def logged_in(self): return self._token is not None SEARCH_LIMIT = 500 def search_videos(self, videos, callback, languages=None): log.debug('search_videos(#videos={})'.format(len(videos))) if not self.logged_in(): raise ProviderNotConnectedError() lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate( window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc.SearchSubtitles( self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown( remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime( rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitlesSubtitleFile( filename=remote_filename, file_size=remote_file_size, md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, age=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception( 'Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format( queries=queries)) log.error('Offending result is: {remote_sub}'.format( remote_sub=rsub_raw)) callback.finish() return remote_subtitles def query_text(self, query): return OpenSubtitlesTextQuery(query=query) def download_subtitles(self, os_rsubs): log.debug('download_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate( window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({ item['idsubtitlefile']: item['data'] for item in result['data'] }) subtitles = [ unzip_bytes(base64.b64decode( map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs ] return subtitles def ping(self): log.debug('ping()') if not self.logged_in(): raise ProviderNotConnectedError() def run_query(): return self._xmlrpc.NoOperation(self._token) result = self._safe_exec(run_query, None) self.check_result(result) @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str @classmethod def get_name(cls): return 'opensubtitles' @classmethod def get_short_name(cls): return 'os' def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): try: result = query() return result except (ProtocolError, CannotSendRequest, SocketError): self._signal_connection_failed() log.warning('Query failed', exc_info=sys.exc_info()) return default STATUS_CODE_RE = re.compile('(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise ProviderConnectionError(_('No message')) log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format( status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError( _('Server returned status="{status}". Expected "200 OK".'). format(status=data['status']), data['status']) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(message, code) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)')
class OpenSubtitles(SubtitleProvider): URL = 'http://api.opensubtitles.org/xml-rpc' def __init__(self, settings=None): SubtitleProvider.__init__(self) self._xmlrpc = None self._token = None self._last_time = None if settings is None: settings = OpenSubtitlesSettings() self._settings = settings def get_settings(self): return self._settings def set_settings(self, settings): if self.connected(): raise RuntimeError( 'Cannot set settings while connected') # FIXME: change error self._settings = settings def connect(self): log.debug('connect()') if self.connected(): return self._xmlrpc = ServerProxy(self.URL, allow_none=False) self._last_time = time.time() def disconnect(self): log.debug('disconnect()') if self.logged_in(): self.logout() if self.connected(): self._xmlrpc = None def connected(self): return self._xmlrpc is not None def login(self): log.debug('login()') if self.logged_in(): return if not self.connected(): self.connect() def login_query(): # FIXME: 'en' language ok??? or '' as in the original return self._xmlrpc.LogIn(str(self._settings.username), str(self._settings.password), 'en', str(self._settings.get_user_agent())) result = self._safe_exec(login_query, None) self.check_result(result) self._token = result['token'] def logout(self): log.debug('logout()') if self.logged_in(): def logout_query(): return self._xmlrpc.LogOut(self._token) # Do no check result of this call. Assume connection closed. self._safe_exec(logout_query, None) self._token = None def logged_in(self): return self._token is not None def reestablish(self): log.debug('reestablish()') connected = self.connected() logged_in = self.logged_in() self.disconnect() if connected: self.connect() if logged_in: self.login() _TIMEOUT_MS = 60000 def _ensure_connection(self): now = time.time() if now - time.time() > self._TIMEOUT_MS: self.reestablish() self._last_time = now SEARCH_LIMIT = 500 def search_videos(self, videos, callback, languages=None): log.debug('search_videos(#videos={})'.format(len(videos))) if not self.logged_in(): raise ProviderNotConnectedError() lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate( window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } if video.get_osdb_hash() is None: log.debug('osdb hash of "{}" is empty -> skip'.format( video.get_filepath())) self._signal_connection_failed( ) # FIXME: other name + general signaling continue queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc.SearchSubtitles( self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown( remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime( rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitlesSubtitleFile( filename=remote_filename, file_size=remote_file_size, md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, date=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] try: imdb_rating = float(rsub_raw['MovieImdbRating']) except (ValueError, KeyError): imdb_rating = None imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=imdb_rating) video_name = rsub_raw['MovieName'] try: video_year = int(rsub_raw['MovieYear']) except (ValueError, KeyError): video_year = None video_identity = VideoIdentity(name=video_name, year=video_year) try: series_season = int(rsub_raw['SeriesSeason']) except (KeyError, ValueError): series_season = None try: series_episode = int(rsub_raw['SeriesEpisode']) except (KeyError, ValueError): series_episode = None series_identity = SeriesIdentity(season=series_season, episode=series_episode) identity = ProviderIdentities( video_identity=video_identity, imdb_identity=imdb_identity, episode_identity=series_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception( 'Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format( queries=queries)) log.error('Offending result is: {remote_sub}'.format( remote_sub=rsub_raw)) callback.finish() return remote_subtitles def query_text(self, query): return OpenSubtitlesTextQuery(query=query) def download_subtitles(self, os_rsubs): log.debug('download_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate( window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({ item['idsubtitlefile']: item['data'] for item in result['data'] }) subtitles = [ unzip_bytes(base64.b64decode( map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs ] return subtitles def upload_subtitles(self, local_movie): log.debug('upload_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() video_subtitles = list(local_movie.iter_video_subtitles()) if not video_subtitles: return UploadResult( type=UploadResult.Type.MISSINGDATA, reason=_('Need at least one subtitle to upload')) query_try = dict() for sub_i, (video, subtitle) in enumerate(video_subtitles): if not video: return UploadResult( type=UploadResult.Type.MISSINGDATA, reason=_('Each subtitle needs an accompanying video')) query_try['cd{}'.format(sub_i + 1)] = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), } def run_query_try_upload(): return self._xmlrpc.TryUploadSubtitles(self._token, query_try) try_result = self._safe_exec(run_query_try_upload, None) self.check_result(try_result) if int(try_result['alreadyindb']): return UploadResult(type=UploadResult.Type.DUPLICATE, reason=_('Subtitle is already in database')) if local_movie.get_imdb_id() is None: return UploadResult(type=UploadResult.Type.MISSINGDATA, reason=_('Need IMDb id')) upload_base_info = { 'idmovieimdb': local_movie.get_imdb_id(), } if local_movie.get_comments() is not None: upload_base_info['subauthorcomment'] = local_movie.get_comments() if not local_movie.get_language().is_generic(): upload_base_info['sublanguageid'] = local_movie.get_language().xxx( ) if local_movie.get_release_name() is not None: upload_base_info[ 'moviereleasename'] = local_movie.get_release_name() if local_movie.get_movie_name() is not None: upload_base_info['movieaka'] = local_movie.get_movie_name() if local_movie.is_hearing_impaired() is not None: upload_base_info[ 'hearingimpaired'] = local_movie.is_hearing_impaired() if local_movie.is_high_definition() is not None: upload_base_info[ 'highdefinition'] = local_movie.is_high_definition() if local_movie.is_automatic_translation() is not None: upload_base_info[ 'automatictranslation'] = local_movie.is_automatic_translation( ) if local_movie.get_author() is not None: upload_base_info['subtranslator'] = local_movie.get_author() if local_movie.is_foreign_only() is not None: upload_base_info['foreignpartsonly'] = local_movie.is_foreign_only( ) query_upload = { 'baseinfo': upload_base_info, } for sub_i, (video, subtitle) in enumerate(video_subtitles): sub_bytes = subtitle.get_filepath().open(mode='rb').read() sub_tx_data = base64.b64encode(zlib.compress(sub_bytes)).decode() query_upload['cd{}'.format(sub_i + 1)] = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None, 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), 'subcontent': sub_tx_data, } def run_query_upload(): return self._xmlrpc.UploadSubtitles(self._token, query_upload) result = self._safe_exec(run_query_upload, None) self.check_result(result) rsubs = [] for sub_data in result['data']: filename = sub_data['SubFileName'] file_size = sub_data['SubSize'] md5_hash = sub_data['SubHash'] id_online = sub_data['IDSubMOvieFile'] download_link = sub_data['SubDownloadLink'] link = None uploader = sub_data['UserNickName'] language = Language.from_xxx(sub_data['SubLanguageID']) rating = float(sub_data['SubRating']) add_date = datetime.datetime.strptime(sub_data['SubAddDate'], '%Y-%m-%d %H:%M:%S') sub = OpenSubtitlesSubtitleFile(filename=filename, file_size=file_size, md5_hash=md5_hash, id_online=id_online, download_link=download_link, link=link, uploader=uploader, language=language, rating=rating, date=add_date) rsubs.append(sub) return UploadResult(type=UploadResult.Type.OK, rsubs=rsubs) def imdb_search_title(self, title): self._ensure_connection() def run_query(): return self._xmlrpc.SearchMoviesOnIMDB(self._token, title.strip()) result = self._safe_exec(run_query, default=None) self.check_result(result) imdbs = [] re_title = re.compile(r'(?P<title>.*) \((?P<year>[0-9]+)\)') for imdb_data in result['data']: imdb_id = imdb_data['id'] if all(c in string.digits for c in imdb_id): imdb_id = 'tt{}'.format(imdb_id) m = re_title.match(imdb_data['title']) if m: imdb_title = m['title'] imdb_year = int(m['year']) else: imdb_title = imdb_data['title'] imdb_year = None imdbs.append( ImdbMovieMatch(imdb_id=imdb_id, title=imdb_title, year=imdb_year)) return imdbs def ping(self): log.debug('ping()') if not self.logged_in(): raise ProviderNotConnectedError() def run_query(): return self._xmlrpc.NoOperation(self._token) result = self._safe_exec(run_query, None) self.check_result(result) def provider_info(self): if self.connected(): def run_query(): return self._xmlrpc.ServerInfo() result = self._safe_exec(run_query, None) data = [ (_('XML-RPC version'), result['xmlrpc_version']), (_('XML-RPC url'), result['xmlrpc_url']), (_('Application'), result['application']), (_('Contact'), result['contact']), (_('Website url'), result['website_url']), (_('Users online'), result['users_online_total']), (_('Programs online'), result['users_online_program']), (_('Users logged in'), result['users_loggedin']), (_('Max users online'), result['users_max_alltime']), (_('Users registered'), result['users_registered']), (_('Subtitles downloaded'), result['subs_downloads']), (_('Subtitles available'), result['subs_subtitle_files']), (_('Number movies'), result['movies_total']), (_('Number languages'), result['total_subtitles_languages']), (_('Client IP'), result['download_limits']['client_ip']), (_('24h global download limit'), result['download_limits']['global_24h_download_limit']), (_('24h client download limit'), result['download_limits']['client_24h_download_limit']), (_('24h client download count'), result['download_limits']['client_24h_download_count']), (_('Client download quota'), result['download_limits']['client_download_quota']), ] else: data = [] return data @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str @classmethod def get_name(cls): return 'opensubtitles' @classmethod def get_short_name(cls): return 'os' @classmethod def get_icon(cls): return ':/images/sites/opensubtitles.png' def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): self._ensure_connection() try: result = query() return result except (ProtocolError, CannotSendRequest, SocketError, ExpatError) as e: self._signal_connection_failed() log.debug('Query failed: {} {}'.format(type(e), e.args)) return default STATUS_CODE_RE = re.compile(r'(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise OpenSubtitlesProviderConnectionError(None, _('No message')) log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format( status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise OpenSubtitlesProviderConnectionError( None, _('Server returned status="{status}". Expected "200 OK".'). format(status=data['status']), data['status']) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise OpenSubtitlesProviderConnectionError(code, message) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)')