class SDService(object): """ Contains the class that represents the OSDB RPC Server. Encapsules all the XMLRPC methods. Consult the OSDB API methods at http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC If it fails to connect directly to the XMLRPC server, it will try to do so through a default proxy. Default proxy uses a form to set which URL to open. We will try to change this in later stage. """ def __init__(self, server=None, proxy=None): self.log = logging.getLogger("subdownloader.SDService.SDService") self.log.debug( "Creating Server with server = %s and proxy = %r" % (server, proxy)) self.timeout = 30 self.user_agent = USER_AGENT self.language = '' if server: self.server = server else: self.server = DEFAULT_OSDB_SERVER self.proxy = proxy self.logged_as = None self._xmlrpc_server = None self._token = None def connected(self): return self._token is not None def connect(self): server = self.server proxy = self.proxy self.log.debug("connect()... to server %s with proxy %s" % (server, proxy)) connect_res = False try: self.log.debug( "Connecting with parameters (%r, %r)" % (server, proxy)) connect = TimeoutFunction(self._connect) connect_res = connect(server, proxy) except TimeoutFunctionException as e: self.log.error("Connection timed out. Maybe you need a proxy.") raise except: self.log.exception("connect: Unexpected error") raise finally: self.log.debug("connection connected %s" % connect_res) return connect_res def _connect(self, server, proxy): try: if proxy: self.log.debug("Trying proxied connection... ({})".format(proxy)) self.proxied_transport = ProxiedTransport() self.proxied_transport.set_proxy(proxy) self._xmlrpc_server = ServerProxy( server, transport=self.proxied_transport, allow_none=True) # self.ServerInfo() self.log.debug("...connected") return True elif test_connection(TEST_URL): self.log.debug("Trying direct connection...") self._xmlrpc_server = ServerProxy( server, allow_none=True) # self.ServerInfo() self.log.debug("...connected") return True else: self.log.debug("...failed") self.log.error("Unable to connect. Try setting a proxy.") return False except ProtocolError as e: self._connection_failed() self.log.debug("error in HTTP/HTTPS transport layer") raise except Fault as e: self.log.debug("error in xml-rpc server") raise except: self.log.exception("Connection to the server failed/other error") raise def logged_in(self): return self._token is not None def login(self, username="", password=""): try: login = TimeoutFunction(self._login) return login(username, password) except TimeoutFunctionException: self.log.error("login timed out") except: self.log.exception("login: other issue") raise def _login(self, username="", password=""): """Login to the Server using username/password, empty parameters means an anonymously login Returns True if login sucessful, and False if not. """ self.log.debug("----------------") self.log.debug("Logging in (username: %s)..." % username) def run_query(): return self._xmlrpc_server.LogIn( username, password, self.language, self.user_agent) info = self._safe_exec(run_query, None) if info is None: self._token = None return False self.log.debug("Login ended in %s with status: %s" % (info['seconds'], info['status'])) if info['status'] == "200 OK": self.log.debug("Session ID: %s" % info['token']) self.log.debug("----------------") self._token = info['token'] return True else: # force token reset self.log.debug("----------------") self._token = None return False def logout(self): try: logout = TimeoutFunction(self._logout) result = logout() self._token = None return result except TimeoutFunctionException: self.log.error("logout timed out") def _logout(self): """Logout from current session(token) This functions doesn't return any boolean value, since it can 'fail' for anonymous logins """ self.log.debug("Logging out from session ID: %s" % self._token) try: info = self._xmlrpc_server.LogOut(self._token) self.log.debug("Logout ended in %s with status: %s" % (info['seconds'], info['status'])) except ProtocolError as e: self.log.debug("error in HTTP/HTTPS transport layer") raise except Fault as e: self.log.debug("error in xml-rpc server") raise except: self.log.exception("Connection to the server failed/other error") raise finally: # force token reset self._token = None STATUS_CODE_RE = re.compile('(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise ProviderConnectionError(None, 'No message') log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format(status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(None, 'Server returned status="{status}". Expected "200 OK".'.format( status=data['status'])) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(code, message) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)') @classmethod def name(cls): return "opensubtitles" def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): try: result = query() return result except (ProtocolError, xml.parsers.expat.ExpatError): self._signal_connection_failed() log.debug("Query failed", exc_info=sys.exc_info()) return default @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str def imdb_query(self, query): if not self.connected(): return None def run_query(): return self._xmlrpc_server.SearchMoviesOnIMDB(self._token, query) result = self._safe_exec(run_query, None) if result is None: return None provider_identities = [] for imdb_data in result['data']: if not imdb_data: continue imdb_identity = ImdbIdentity(imdb_id=imdb_data['id'], imdb_rating=None) video_identity = VideoIdentity(name=imdb_data['title'], year=None) provider_identities.append(ProviderIdentities( video_identity=video_identity, imdb_identity=imdb_identity, provider=self)) return provider_identities SEARCH_LIMIT = 500 def search_text(self, text, languages=None): lang_str = self._languages_to_str(languages) query = { 'sublanguageid': lang_str, 'query': str(text), } queries = [query] def run_query(): return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT}) self._safe_exec(run_query, None) def search_videos(self, videos, callback, languages=None): if not self.connected(): return None lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate(window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown(remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime(rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitles_SubtitleFile( filename=remote_filename, file_size=remote_file_size , md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, age=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception('Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format(queries=queries)) log.error('Offending result is: {remote_sub}'.format(remote_sub=rsub_raw)) callback.finish() return remote_subtitles def _video_info_to_identification(self, video_info): name = video_info['MovieName'] year = int(video_info['MovieYear']) imdb_id = video_info['MovieImdbID'] video_identity = VideoIdentity(name=name, year=year) imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) episode_identity = None movie_kind = video_info['MovieKind'] if movie_kind == 'episode': season = int(video_info['SeriesSeason']) episode = int(video_info['SeriesEpisode']) episode_identity = EpisodeIdentity(season=season, episode=episode) elif movie_kind == 'movie': pass else: log.warning('Unknown MoviesKind="{}"'.format(video_info['MovieKind'])) return ProviderIdentities(video_identity=video_identity, episode_identity=episode_identity, imdb_identity=imdb_identity, provider=self) def identify_videos(self, videos): if not self.connected(): return for part_videos in window_iterator(videos, 200): hashes = [video.get_osdb_hash() for video in part_videos] hash_video = {hash: video for hash, video in zip(hashes, part_videos)} def run_query(): return self._xmlrpc_server.CheckMovieHash2(self._token, hashes) result = self._safe_exec(run_query, None) self.check_result(result) for video_hash, video_info in result['data'].items(): identification = self._video_info_to_identification(video_info[0]) video = hash_video[video_hash] video.add_identity(identification) def download_subtitles(self, os_rsubs): if not self.connected(): return None window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate(window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc_server.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({item['idsubtitlefile']: item['data'] for item in result['data']}) subtitles = [unzip_bytes(base64.b64decode(map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs] return subtitles def can_upload_subtitles(self, local_movie): if not self.connected(): return False query = {} for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()): # sub_bytes = open(subtitle.get_filepath(), mode='rb').read() # sub_tx_data = b64encode(zlib.compress(sub_bytes)) cd = "cd{i}".format(i=i+1) cd_data = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()), 'moviefps': video.get_fps(), 'movieframes': str(video.get_framecount()), 'moviefilename': video.get_filename(), } query[cd] = cd_data def run_query(): return self._xmlrpc_server.TryUploadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) movie_already_in_db = int(result['alreadyindb']) != 0 if movie_already_in_db: return False return True def upload_subtitles(self, local_movie): query = { 'baseinfo': { 'idmovieimdb': local_movie.get_imdb_id(), 'moviereleasename': local_movie.get_release_name(), 'movieaka': local_movie.get_movie_name(), 'sublanguageid': local_movie.get_language().xxx(), 'subauthorcomment': local_movie.get_comments(), }, } if local_movie.is_hearing_impaired() is not None: query['hearingimpaired'] = local_movie.is_hearing_impaired() if local_movie.is_high_definition() is not None: query['highdefinition'] = local_movie.is_high_definition() if local_movie.is_automatic_translation() is not None: query['automatictranslation'] = local_movie.is_automatic_translation() if local_movie.get_subtitle_author() is not None: query['subtranslator'] = local_movie.get_subtitle_author() if local_movie.is_foreign_only() is not None: query['foreignpartsonly'] = local_movie.is_foreign_only() for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()): sub_bytes = subtitle.get_filepath().open(mode='rb').read() sub_tx_data = b64encode(zlib.compress(sub_bytes)) cd = "cd{i}".format(i=i+1) cd_data = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None, 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), 'subcontent': sub_tx_data, } query[cd] = cd_data def run_query(): return self._xmlrpc_server.UploadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result)