class OpenSubtitlesClient: def __init__(self, username, password, language): self.client = ServerProxy('http://api.opensubtitles.org/xml-rpc', allow_none=True) self.token = self.client.LogIn(username, password, language, "TemporaryUserAgent")['token'] def get_list_subtitle_response(self, query, language): response = self.client.SearchSubtitles(self.token, [{ 'query': query, 'sublanguageid': language }], [{ 'limit': 3 }]) return response['data'] def download_encrypted_subtitles(self, subtitle_file_id): response = self.client.DownloadSubtitles(self.token, [subtitle_file_id]) decompresses_subtitle_file = zlib.decompress( base64.b64decode(response['data'][0]['data']), 16 + zlib.MAX_WBITS) return decompresses_subtitle_file def get_encryption_method(self, encrypted_subtitles): return chardet.detect(encrypted_subtitles)['encoding'] def decode(self, encryption_method, data): return data.decode(encryption_method)
class SubtitlesConncector: RETURNED_OK_STATUS = '200 OK' def __init__(self, user_name='', password='', language='', agent='OS Test User Agent'): self.user_name = user_name self.password = password self.language = language self.agent = agent self._api_adres = 'http://api.opensubtitles.org/xml-rpc' self.proxy_server = ServerProxy(self._api_adres) self.token = self._get_token() def _is_status_ok(self, status): if status == SubtitlesConncector.RETURNED_OK_STATUS: return True return False def _get_token(self): data = self.proxy_server.LogIn(self.user_name, self.password, self.language, self.agent) status = StatusParsser(data=data).parse() if self._is_status_ok(status): return data['token'] else: raise StatusError(status) def _save_zip(self, data_to_zip, path, file_name): b64img = base64.b64decode(data_to_zip.encode('ascii')) b64imgfile = open(str(path + "/" + file_name + ".zip"), 'w') b64imgfile.write(str(b64img)) def get_subtitles(self, subtitle, subtitles_dir=None): if not subtitles_dir: subtitles_dir = os.path.expanduser("~") + "/subtitles" if not os.path.exists(subtitles_dir): os.makedirs(subtitles_dir) data = self.proxy_server.DownloadSubtitles(self.token, [subtitle.idsubtitlefile]) try: data = data['data'][0]['data'] except Exception: raise GetSubtitlesError("Error: cant get data") self._save_zip(data, subtitles_dir, subtitle.subfilename) def search_subtitles(self, subtitles): parsed_data_to_send = ParseSubtitlesToSend(subtitles).parse() data = self.proxy_server.SearchSubtitles(self.token, [parsed_data_to_send]) data = ParseIncomingData(data).parse() collect = SubtitlesFindedCollection() for d in data: collect.append( SubtitlesFindedProperties(sublangid=d['SubLanguageID'], subfilename=d['SubFileName'], idsubtitlefile=d['IDSubtitleFile'])) return collect
def get_subtitles(db, l_post_id, l_imdbid): try: server = ServerProxy('http://api.opensubtitles.org/xml-rpc') token = server.LogIn('dh_moviebarcodes', 'dh_ws2015', 'eng', 'Moviebarcode Analyzer')['token'] imdb_id = int(l_imdbid[2:]) search_request = [] search_request.append({'imdbid': imdb_id, 'sublanguageid': 'eng'}) resp = server.SearchSubtitles(token, search_request) subtitle_id = [] try: sub = get_best_subtitle(resp['data']) subtitle_id.append(sub['IDSubtitleFile']) subtitle_data = server.DownloadSubtitles(token, subtitle_id) except IndexError: print("No subtitle available") return "" if subtitle_data['status'] == '200 OK': compressed_data = subtitle_data['data'][0]['data'] decoded_subtitle = base64.b64decode(compressed_data) # subtitle in byte format decoded_subtitle = gzip.GzipFile(fileobj=io.BytesIO(decoded_subtitle)).read() clean_subtitle = remove_invalid_characters(decoded_subtitle) allLemmasAndMostFrequentLemmas = lemmatize(clean_subtitle) print(allLemmasAndMostFrequentLemmas[0]) print(allLemmasAndMostFrequentLemmas[1]) update_value_in_db(db, l_post_id, "subtitlesLemmatisation", allLemmasAndMostFrequentLemmas[0]) update_value_in_db(db, l_post_id, "subtitlesMostFrequentWords", allLemmasAndMostFrequentLemmas[1]) except ValueError: print("No subtitle available")
class Downloader: """ Connects to the OpenSubtitles server and allows for file downloading. """ def __init__(self, user_agent, server='http://api.opensubtitles.org/xml-rpc', language='eng'): self.server = server self.language = language trans = Transport() trans.user_agent = user_agent self._rpc = ServerProxy(self.server, allow_none=True, transport=trans) login_response = self._rpc.LogIn('', '', language, user_agent) assert_status(login_response) self._token = login_response.get('token') def download(self, *ids): """ Downloads the subtitles with the given ids. :param ids: The subtitles to download :return: Result instances :raises NotOKException """ bundles = sublists_of(ids, 20) # 20 files at once is an API restriction for bundle in bundles: download_response = self._rpc.DownloadSubtitles( self._token, bundle) assert_status(download_response) download_data = download_response.get('data') for item in download_data: subtitle_id = item['idsubtitlefile'] subtitle_data = item['data'] decompressed = decompress(subtitle_data) yield Result(subtitle_id, decompressed) def close(self): """ Closes the connection. """ self._rpc.LogOut(self._token) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close()
class OpenSubtitles: def __init__(self): # noinspection SpellCheckingInspection opensubtitles_ua = b'VkxzdWIgMC4xMC4y' self.ua = base64.b64decode(opensubtitles_ua).decode() self.opensubtitles_api_url = 'http://api.opensubtitles.org/xml-rpc' self.opensubtitles_lang = 'en' self.token = None self.xmlrpc_transport = Transport() self.xmlrpc_transport.user_agent = self.ua self.xmlrpc = ServerProxy(self.opensubtitles_api_url, allow_none=True, transport=self.xmlrpc_transport) def retrieve_subtitles(self, ids: List[str]) -> Dict[str, str]: response = self.xmlrpc.DownloadSubtitles(self.token, ids) status = response.get('status', '').split()[0] if status == '200': subtitles_content = {} encoded_data = response.get('data') for item in encoded_data: subfile_id = item['idsubtitlefile'] try: decoded_data = decompress(item['data'], 'utf_8_sig') or decompress( item['data'], 'utf-8') except UnicodeDecodeError: decoded_data = decompress(item['data'], 'latin1') if not decoded_data: logger.error('An error occurred while decoding subtitle' 'file ID {}.'.format(subfile_id)) else: subtitles_content[subfile_id] = decoded_data return subtitles_content else: raise RuntimeWarning( f'Unable to get subtitles, returned status is {status}') def search_subtitles(self, param: List[dict]) -> List[Dict[str, str]]: response = self.xmlrpc.SearchSubtitles(self.token, param) status = response.get('status', '').split()[0] if status == '200': return response.get('data') else: raise RuntimeWarning( f'Unable to search subtitles, returned status is {status}') def login(self, username, password): response = self.xmlrpc.LogIn(username, password, self.opensubtitles_lang, self.ua) status = response.get('status', '').split()[0] if status == '200': self.token = response.get('token') else: raise RuntimeWarning( f'Unable to authenticate, returned status is {status}')
class OpenSubtitles(SubtitleProvider): URL = 'http://api.opensubtitles.org/xml-rpc' def __init__(self): SubtitleProvider.__init__(self) self._xmlrpc = None self._token = None self._settings = OpenSubtitlesSettings() def get_settings(self): return self._settings def set_settings(self, settings): self._settings = settings def connect(self): log.debug('connect()') if self.connected(): return self._xmlrpc = ServerProxy(self.URL, allow_none=False) def disconnect(self): log.debug('disconnect()') if self.logged_in(): self.logout() if self.connected(): self._xmlrpc = None def connected(self): return self._xmlrpc is not None def login(self): log.debug('login()') if self.logged_in(): return if not self.connected(): raise ProviderNotConnectedError() def login_query(): # FIXME: 'en' language ok??? or '' as in the original return self._xmlrpc.LogIn(str(self._settings.username), str(self._settings.password), 'en', str(self._settings.get_user_agent())) result = self._safe_exec(login_query, None) self.check_result(result) self._token = result['token'] def logout(self): log.debug('logout()') if self.logged_in(): def logout_query(): return self._xmlrpc.LogOut(self._token) result = self._safe_exec(logout_query, None) self.check_result(result) self._token = None def logged_in(self): return self._token is not None SEARCH_LIMIT = 500 def search_videos(self, videos, callback, languages=None): log.debug('search_videos(#videos={})'.format(len(videos))) if not self.logged_in(): raise ProviderNotConnectedError() lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate( window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc.SearchSubtitles( self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown( remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime( rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitlesSubtitleFile( filename=remote_filename, file_size=remote_file_size, md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, age=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception( 'Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format( queries=queries)) log.error('Offending result is: {remote_sub}'.format( remote_sub=rsub_raw)) callback.finish() return remote_subtitles def query_text(self, query): return OpenSubtitlesTextQuery(query=query) def download_subtitles(self, os_rsubs): log.debug('download_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate( window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({ item['idsubtitlefile']: item['data'] for item in result['data'] }) subtitles = [ unzip_bytes(base64.b64decode( map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs ] return subtitles def ping(self): log.debug('ping()') if not self.logged_in(): raise ProviderNotConnectedError() def run_query(): return self._xmlrpc.NoOperation(self._token) result = self._safe_exec(run_query, None) self.check_result(result) @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str @classmethod def get_name(cls): return 'opensubtitles' @classmethod def get_short_name(cls): return 'os' def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): try: result = query() return result except (ProtocolError, CannotSendRequest, SocketError): self._signal_connection_failed() log.warning('Query failed', exc_info=sys.exc_info()) return default STATUS_CODE_RE = re.compile('(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise ProviderConnectionError(_('No message')) log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format( status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError( _('Server returned status="{status}". Expected "200 OK".'). format(status=data['status']), data['status']) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(message, code) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)')
class SubtitleDownload: '''Traverses a directory and all subdirectories and downloads subtitles. Relies on an undocumented OpenSubtitles feature where subtitles seems to be sorted by filehash and popularity. ''' api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None moviefiles = [] movie_exts = (".avi", ".mkv", ".mp4") subb_exts = (".srt", ".sub", ".mpl") def __init__(self, movie_path, lang = "en"): print("OpenSubtitles Subtitle Downloader".center(78)) print("===================================".center(78)) self.server = ServerProxy(self.api_url, verbose=False) self.lang_id = lang # Traverse the directory tree and select all movie files for root, _, files in os.walk(movie_path): for file in files: if self.is_movie(file): file_path = os.path.join(root, file) if not self.subtitles_already_present(file_path): print("Found: " + file) filehash = self.hashFile(file_path) filesize = os.path.getsize(file_path) self.moviefiles.append({'dir': root, 'file': file, 'hash': filehash, 'size': filesize, 'subtitleid': None}) try: print("Login...") self.login() print("Searching for subtitles...") self.search_subtitles() print("Logout...") self.logout() except Error as e: print("XML-RPC error:", e) except UserWarning as uw: print(uw) def login(self): '''Log in to OpenSubtitles''' resp = self.server.LogIn('', '', 'en', 'OS Test User Agent') self.check_status(resp) self.login_token = resp['token'] def logout(self): '''Log out from OpenSubtitles''' resp = self.server.LogOut(self.login_token) self.check_status(resp) def search_subtitles(self): '''Search OpenSubtitles for matching subtitles''' search = [] for movie in self.moviefiles: search.append({'sublanguageid': self.lang_id, 'moviehash': movie['hash'], 'moviebytesize': str(movie['size'])}) # fixes weird problem where subs aren't found when only searching # for one movie. if len(search) == 1: search.append(search[0]) resp = self.server.SearchSubtitles(self.login_token, search) self.check_status(resp) if resp['data'] == False: print("No subtitles found") return subtitles = [] for result in resp['data']: if int(result['SubBad']) != 1: subtitles.append({'subid': result['IDSubtitleFile'], 'hash': result['MovieHash']}) downloadable_subs = subtitles[:] hash = None for s in subtitles: if hash == s['hash']: downloadable_subs.remove(s) hash = s['hash'] for ds in downloadable_subs: sub = self.download_subtitles(ds) for movie in self.moviefiles: if movie['hash'] == ds['hash']: print("Saving subtitle for: " + movie['file']) filename = os.path.join(movie['dir'], os.path.splitext(movie['file'])[0] + ".srt") file = open(filename, "wb") file.write(sub) file.close() def download_subtitles(self, subtitle): resp = self.server.DownloadSubtitles(self.login_token, [subtitle['subid']]) self.check_status(resp) decoded = base64.standard_b64decode(resp['data'][0]['data'].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed def is_movie(self, file): return os.path.splitext(file.lower())[1] in self.movie_exts def subtitles_already_present(self, file): file_base = os.path.splitext(file.lower())[0] for ext in self.subb_exts: if os.path.exists(file_base + ext): return True return False def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise UserWarning("Response error from " + self.api_url + ". Response status was: " + resp['status']) def hashFile(self, name): '''Calculates the hash value of a movie. Copied from OpenSubtitles own examples: http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes ''' try: longlongformat = 'q' # long long bytesize = struct.calcsize(longlongformat) f = open(name, "rb") filesize = os.path.getsize(name) hash = filesize if filesize < 65536 * 2: return "SizeError" for x in range(65536//bytesize): buffer = f.read(bytesize) (l_value,)= struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number f.seek(max(0,filesize-65536),0) for x in range(65536//bytesize): buffer = f.read(bytesize) (l_value,)= struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF f.close() returnedhash = "%016x" % hash return returnedhash except(IOError): return "IOError"
class OpenSubtitles(SubtitleProvider): URL = 'http://api.opensubtitles.org/xml-rpc' def __init__(self, settings=None): SubtitleProvider.__init__(self) self._xmlrpc = None self._token = None self._last_time = None if settings is None: settings = OpenSubtitlesSettings() self._settings = settings def get_settings(self): return self._settings def set_settings(self, settings): if self.connected(): raise RuntimeError( 'Cannot set settings while connected') # FIXME: change error self._settings = settings def connect(self): log.debug('connect()') if self.connected(): return self._xmlrpc = ServerProxy(self.URL, allow_none=False) self._last_time = time.time() def disconnect(self): log.debug('disconnect()') if self.logged_in(): self.logout() if self.connected(): self._xmlrpc = None def connected(self): return self._xmlrpc is not None def login(self): log.debug('login()') if self.logged_in(): return if not self.connected(): self.connect() def login_query(): # FIXME: 'en' language ok??? or '' as in the original return self._xmlrpc.LogIn(str(self._settings.username), str(self._settings.password), 'en', str(self._settings.get_user_agent())) result = self._safe_exec(login_query, None) self.check_result(result) self._token = result['token'] def logout(self): log.debug('logout()') if self.logged_in(): def logout_query(): return self._xmlrpc.LogOut(self._token) # Do no check result of this call. Assume connection closed. self._safe_exec(logout_query, None) self._token = None def logged_in(self): return self._token is not None def reestablish(self): log.debug('reestablish()') connected = self.connected() logged_in = self.logged_in() self.disconnect() if connected: self.connect() if logged_in: self.login() _TIMEOUT_MS = 60000 def _ensure_connection(self): now = time.time() if now - time.time() > self._TIMEOUT_MS: self.reestablish() self._last_time = now SEARCH_LIMIT = 500 def search_videos(self, videos, callback, languages=None): log.debug('search_videos(#videos={})'.format(len(videos))) if not self.logged_in(): raise ProviderNotConnectedError() lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate( window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } if video.get_osdb_hash() is None: log.debug('osdb hash of "{}" is empty -> skip'.format( video.get_filepath())) self._signal_connection_failed( ) # FIXME: other name + general signaling continue queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc.SearchSubtitles( self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown( remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime( rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitlesSubtitleFile( filename=remote_filename, file_size=remote_file_size, md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, date=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] try: imdb_rating = float(rsub_raw['MovieImdbRating']) except (ValueError, KeyError): imdb_rating = None imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=imdb_rating) video_name = rsub_raw['MovieName'] try: video_year = int(rsub_raw['MovieYear']) except (ValueError, KeyError): video_year = None video_identity = VideoIdentity(name=video_name, year=video_year) try: series_season = int(rsub_raw['SeriesSeason']) except (KeyError, ValueError): series_season = None try: series_episode = int(rsub_raw['SeriesEpisode']) except (KeyError, ValueError): series_episode = None series_identity = SeriesIdentity(season=series_season, episode=series_episode) identity = ProviderIdentities( video_identity=video_identity, imdb_identity=imdb_identity, episode_identity=series_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception( 'Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format( queries=queries)) log.error('Offending result is: {remote_sub}'.format( remote_sub=rsub_raw)) callback.finish() return remote_subtitles def query_text(self, query): return OpenSubtitlesTextQuery(query=query) def download_subtitles(self, os_rsubs): log.debug('download_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate( window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({ item['idsubtitlefile']: item['data'] for item in result['data'] }) subtitles = [ unzip_bytes(base64.b64decode( map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs ] return subtitles def upload_subtitles(self, local_movie): log.debug('upload_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() video_subtitles = list(local_movie.iter_video_subtitles()) if not video_subtitles: return UploadResult( type=UploadResult.Type.MISSINGDATA, reason=_('Need at least one subtitle to upload')) query_try = dict() for sub_i, (video, subtitle) in enumerate(video_subtitles): if not video: return UploadResult( type=UploadResult.Type.MISSINGDATA, reason=_('Each subtitle needs an accompanying video')) query_try['cd{}'.format(sub_i + 1)] = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), } def run_query_try_upload(): return self._xmlrpc.TryUploadSubtitles(self._token, query_try) try_result = self._safe_exec(run_query_try_upload, None) self.check_result(try_result) if int(try_result['alreadyindb']): return UploadResult(type=UploadResult.Type.DUPLICATE, reason=_('Subtitle is already in database')) if local_movie.get_imdb_id() is None: return UploadResult(type=UploadResult.Type.MISSINGDATA, reason=_('Need IMDb id')) upload_base_info = { 'idmovieimdb': local_movie.get_imdb_id(), } if local_movie.get_comments() is not None: upload_base_info['subauthorcomment'] = local_movie.get_comments() if not local_movie.get_language().is_generic(): upload_base_info['sublanguageid'] = local_movie.get_language().xxx( ) if local_movie.get_release_name() is not None: upload_base_info[ 'moviereleasename'] = local_movie.get_release_name() if local_movie.get_movie_name() is not None: upload_base_info['movieaka'] = local_movie.get_movie_name() if local_movie.is_hearing_impaired() is not None: upload_base_info[ 'hearingimpaired'] = local_movie.is_hearing_impaired() if local_movie.is_high_definition() is not None: upload_base_info[ 'highdefinition'] = local_movie.is_high_definition() if local_movie.is_automatic_translation() is not None: upload_base_info[ 'automatictranslation'] = local_movie.is_automatic_translation( ) if local_movie.get_author() is not None: upload_base_info['subtranslator'] = local_movie.get_author() if local_movie.is_foreign_only() is not None: upload_base_info['foreignpartsonly'] = local_movie.is_foreign_only( ) query_upload = { 'baseinfo': upload_base_info, } for sub_i, (video, subtitle) in enumerate(video_subtitles): sub_bytes = subtitle.get_filepath().open(mode='rb').read() sub_tx_data = base64.b64encode(zlib.compress(sub_bytes)).decode() query_upload['cd{}'.format(sub_i + 1)] = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None, 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), 'subcontent': sub_tx_data, } def run_query_upload(): return self._xmlrpc.UploadSubtitles(self._token, query_upload) result = self._safe_exec(run_query_upload, None) self.check_result(result) rsubs = [] for sub_data in result['data']: filename = sub_data['SubFileName'] file_size = sub_data['SubSize'] md5_hash = sub_data['SubHash'] id_online = sub_data['IDSubMOvieFile'] download_link = sub_data['SubDownloadLink'] link = None uploader = sub_data['UserNickName'] language = Language.from_xxx(sub_data['SubLanguageID']) rating = float(sub_data['SubRating']) add_date = datetime.datetime.strptime(sub_data['SubAddDate'], '%Y-%m-%d %H:%M:%S') sub = OpenSubtitlesSubtitleFile(filename=filename, file_size=file_size, md5_hash=md5_hash, id_online=id_online, download_link=download_link, link=link, uploader=uploader, language=language, rating=rating, date=add_date) rsubs.append(sub) return UploadResult(type=UploadResult.Type.OK, rsubs=rsubs) def imdb_search_title(self, title): self._ensure_connection() def run_query(): return self._xmlrpc.SearchMoviesOnIMDB(self._token, title.strip()) result = self._safe_exec(run_query, default=None) self.check_result(result) imdbs = [] re_title = re.compile(r'(?P<title>.*) \((?P<year>[0-9]+)\)') for imdb_data in result['data']: imdb_id = imdb_data['id'] if all(c in string.digits for c in imdb_id): imdb_id = 'tt{}'.format(imdb_id) m = re_title.match(imdb_data['title']) if m: imdb_title = m['title'] imdb_year = int(m['year']) else: imdb_title = imdb_data['title'] imdb_year = None imdbs.append( ImdbMovieMatch(imdb_id=imdb_id, title=imdb_title, year=imdb_year)) return imdbs def ping(self): log.debug('ping()') if not self.logged_in(): raise ProviderNotConnectedError() def run_query(): return self._xmlrpc.NoOperation(self._token) result = self._safe_exec(run_query, None) self.check_result(result) def provider_info(self): if self.connected(): def run_query(): return self._xmlrpc.ServerInfo() result = self._safe_exec(run_query, None) data = [ (_('XML-RPC version'), result['xmlrpc_version']), (_('XML-RPC url'), result['xmlrpc_url']), (_('Application'), result['application']), (_('Contact'), result['contact']), (_('Website url'), result['website_url']), (_('Users online'), result['users_online_total']), (_('Programs online'), result['users_online_program']), (_('Users logged in'), result['users_loggedin']), (_('Max users online'), result['users_max_alltime']), (_('Users registered'), result['users_registered']), (_('Subtitles downloaded'), result['subs_downloads']), (_('Subtitles available'), result['subs_subtitle_files']), (_('Number movies'), result['movies_total']), (_('Number languages'), result['total_subtitles_languages']), (_('Client IP'), result['download_limits']['client_ip']), (_('24h global download limit'), result['download_limits']['global_24h_download_limit']), (_('24h client download limit'), result['download_limits']['client_24h_download_limit']), (_('24h client download count'), result['download_limits']['client_24h_download_count']), (_('Client download quota'), result['download_limits']['client_download_quota']), ] else: data = [] return data @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str @classmethod def get_name(cls): return 'opensubtitles' @classmethod def get_short_name(cls): return 'os' @classmethod def get_icon(cls): return ':/images/sites/opensubtitles.png' def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): self._ensure_connection() try: result = query() return result except (ProtocolError, CannotSendRequest, SocketError, ExpatError) as e: self._signal_connection_failed() log.debug('Query failed: {} {}'.format(type(e), e.args)) return default STATUS_CODE_RE = re.compile(r'(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise OpenSubtitlesProviderConnectionError(None, _('No message')) log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format( status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise OpenSubtitlesProviderConnectionError( None, _('Server returned status="{status}". Expected "200 OK".'). format(status=data['status']), data['status']) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise OpenSubtitlesProviderConnectionError(code, message) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)')
class SubtitleDownload(QtCore.QThread): '''Traverses a directory and all subdirectories and downloads subtitles. Relies on an undocumented OpenSubtitles feature where subtitles seems to be sorted by filehash and popularity. ''' api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None def __init__(self, parent=None): QtCore.QThread.__init__(self, parent) def init(self, movie_paths): self._movie_paths = movie_paths self.start() def __del__(self): if self.login_token: self.logout() def stopTask(self): self.stopping = True def run(self): self.stopping = False self.moviefiles = {} self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Searching for video files...") """Check if the movie_path is a file or directory. Calculate the file stats if its a file, else traverse the directory tree and calculate filestats for all movie files""" for path in self._movie_paths: if os.path.isfile(path): self.calcfilestats(path, os.path.dirname(path)) else: for root, _, files in os.walk(path): for file in files: if not self.stopping: self.calcfilestats(file, root) else: return try: if self.moviefiles: if not self.login_token: try: self.login() except (noInternetConnection): self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, No active Internet connection found. Re-Check and try again." ) my_logger.debug( "Sorry, No active Internet connection found. Re-Check and try again." ) self.emit( QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"), self._movie_paths) return self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Searching for subtitles...") self.search_subtitles() self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Done...") my_logger.debug("Done...") else: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, no video files were found!") self.emit(QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"), self._movie_paths) except Error as e: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), ("XML-RPC error:", e)) except UserWarning as uw: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), uw) def calcfilestats(self, file, parentdir): ''' Calculates and adds the hash of a movie file to the moviefiles dictionary. Also emits a signal to update the LCD counter showing the found files.''' video_extns = [ '.avi', '.divx', '.mkv', '.mp4', '.ogg', '.rm', '.rmvb', '.vob', '.x264', '.xvid' ] if os.path.splitext(file)[1].lower() in video_extns: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Found: " + file) my_logger.debug("Found: " + file) self.emit(QtCore.SIGNAL("updateFound()")) filehash = self.hashFile(os.path.join(parentdir, file)) filesize = os.path.getsize(os.path.join(parentdir, file)) self.moviefiles[filehash] = { 'dir': parentdir, 'file': file, 'size': filesize } def login(self): '''Log in to OpenSubtitles''' self.server = ServerProxy(self.api_url, verbose=False) self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging in...") my_logger.debug("Logging in...") try: resp = self.server.LogIn('', '', 'en', 'PySubD v1.0') self.check_status(resp) self.login_token = resp['token'] except Error as e: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), e) my_logger.debug(str(e)) def logout(self): '''Log out from OpenSubtitles''' self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging out...") resp = self.server.LogOut(self.login_token) self.check_status(resp) def search_subtitles(self): '''Search OpenSubtitles for matching subtitles using moviehash and filebytesize''' search = [] for hash in self.moviefiles.keys(): search.append({ 'sublanguageid': 'eng', 'moviehash': hash, 'moviebytesize': str(self.moviefiles[hash]['size']) }) my_logger.debug("Length of search string: " + str(len(search))) resp = {} resp['data'] = [] while search[:500]: if not self.stopping: tempresp = self.server.SearchSubtitles(self.login_token, search[:500]) if tempresp['data'] != False: resp['data'].extend(tempresp['data']) self.check_status(tempresp) search = search[500:] else: return #Check if we actually got some matching subtitles to download, else return if not resp['data']: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, no subtitles were found!") my_logger.debug("Sorry, no subtitles were found!") return #User Ranks on OpenSubtitles.org.These are decided based upon the number of subtitles uploaded by the member. #A better rank is often an indication of a better subtitle quality. user_ranks = { 'admin': 1, 'platinum member': 2, 'vip member': 3, 'gold member': 4, 'trusted': 5, 'silver member': 6, 'bronze member': 7, 'sub leecher': 8, '': 9 } # A dictionary to store the subtitle id's found corresponding to every file hash subtitles = {} for result in resp['data']: #The subtitle must not be rated bad if int(result['SubBad']) != 1: hash = result['MovieHash'] subid = result['IDSubtitleFile'] downcount = result['SubDownloadsCnt'] rating = result['SubRating'] user_rank = user_ranks[result['UserRank']] #First good matching subtitle found if not subtitles.get(hash): subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } self.emit(QtCore.SIGNAL("updateAvailable()")) #Another good quality subtitle found uploaded by a more reputed user elif subtitles[hash]['user_rank'] > user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } #Another good quality subtitle found with a perfect rating, uploaded by a more or equally reputed user elif float(rating) == 10.0 and subtitles[hash][ 'user_rank'] >= user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } #Another good quality subtitle found with the better rating, higher download count and uploaded by a more or equally trusted user elif float(subtitles[hash]['rating']) >= float(rating) \ and int(subtitles[hash]['downcount']) < int(downcount) and subtitles[hash]['user_rank'] >= user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } my_logger.debug("Total number of subtitles found: " + str(len(subtitles))) notfound = [] for hash, filedetails in self.moviefiles.iteritems(): if not self.stopping: if subtitles.get(hash): try: subtitle = self.download_subtitles( [subtitles[hash]['subid']]) self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Saving subtitle for: " + filedetails['file']) my_logger.debug("Saving subtitle for: " + filedetails['file'] + " Hash : " + hash + " Rating: " + subtitles[hash]['rating'] + " DwnCnt: " + subtitles[hash]['downcount'] + " UpldrRnk:" + \ str(subtitles[hash]['user_rank'])) self.emit(QtCore.SIGNAL("updateDownloaded()")) filename = os.path.join( filedetails['dir'], os.path.splitext(filedetails['file'])[0] + ".srt") file = open(filename, "wb") file.write(subtitle) file.close() except IOError: self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "IO Error in saving subs for " + filedetails['file']) else: notfound.append(filedetails['file']) else: return #Report all the files for which no subtitles were found in an alphabetically sorted order. notfound.sort(key=str.lower) for file in notfound: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "No subtitles found for: " + file) my_logger.debug("No subtitles were found for: " + file) def download_subtitles(self, subparam): resp = self.server.DownloadSubtitles(self.login_token, subparam) self.check_status(resp) decoded = base64.standard_b64decode( resp['data'][0]['data'].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise UserWarning("Response error from " + self.api_url + ". Response status was: " + resp['status']) def hashFile(self, name): ''' Calculates the hash value of a movie. Copied from OpenSubtitles own examples: http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes ''' try: longlongformat = 'q' # long long bytesize = struct.calcsize(longlongformat) f = open(name, "rb") filesize = os.path.getsize(name) hash = filesize if filesize < 65536 * 2: return "SizeError: Minimum file size must be 120Kb" for x in range(65536 // bytesize): buffer = f.read(bytesize) (l_value, ) = struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number f.seek(max(0, filesize - 65536), 0) for x in range(65536 // bytesize): buffer = f.read(bytesize) (l_value, ) = struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF f.close() returnedhash = "%016x" % hash return returnedhash except (IOError): return "IOError"
class OpenSubtitles(QtCore.QThread): api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None def __init__(self, parent=None): QtCore.QThread.__init__(self, parent) self.stopping = False def process(self, files_list, lang='English'): self.moviefiles = files_list self.imdbid_to_hash = {} self.lang = LANGUAGES[lang][1] if not self.stopping: self.start() def __del__(self): if self.login_token: self.logout() def stopTask(self): self.stopping = True def run(self): utils.communicator.updategui.emit('Querying OpenSubtitles.org...', 'info' ) try: if not self.login_token: self.login() self.search_subtitles() except utils.NoInternetConnectionFound: utils.communicator.updategui.emit('No active Internet connection found. Kindly check and try again.', 'error') def login(self): '''Log in to OpenSubtitles.org''' self.server = ServerProxy(self.api_url, verbose=False) utils.communicator.updategui.emit('Logging in to OpenSubtitles.org...', 'info' ) try: resp = self.server.LogIn('', '', 'en', 'PySubD v2.0') self.check_status(resp) self.login_token = resp['token'] except Exception as e: if e.args[0] == 11004: utils.communicator.updategui.emit("No Internet Connection Found", 'error') else: utils.communicator.updategui.emit(str(e), 'error') def logout(self): '''Log out from OpenSubtitles''' utils.communicator.updategui.emit('Logging out...', 'info') resp = self.server.LogOut(self.login_token) self.check_status(resp) def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise utils.IncorrectResponseRecieved('Response error from %s. Response status: %s' % (self.api_url, resp['status'])) def _query_opensubs(self, search): results = [] while search[:500]: if not self.stopping: try: tempresp = self.server.SearchSubtitles(self.login_token, search[:500]) except Exception as e: if e.args[0] == 11004: utils.communicator.updategui.emit("No Internet Connection Found", 'error') else: utils.communicator.updategui.emit(str(e), 'error') return results if tempresp['data']: results.extend(tempresp['data']) self.check_status(tempresp) search = search[500:] else: return [] logger.debug('Results: %s' % results) return results def clean_results(self, results, imdb=False): subtitles = {} user_ranks = { 'administrator': 1, 'platinum member': 2, 'vip member': 3, 'gold member': 4, 'trusted': 5, 'silver member': 6, 'bronze member': 7, 'sub leecher': 8, '': 9, } for result in results: if result['SubBad'] != '1': movie_hash = result.get('MovieHash') if not movie_hash: movie_hash = self.imdbid_to_hash[int(result['IDMovieImdb'])] subid = result['IDSubtitleFile'] downcount = int(result['SubDownloadsCnt']) rating = float(result['SubRating']) if rating and rating < 8: # Ignore poorly rated subtitles, while not # penalizing the ones that haven't yet been rated continue user_rank = user_ranks[result['UserRank']] if imdb: cleaned_release_name = utils.clean_name(result['MovieReleaseName']) file_name = self.moviefiles[movie_hash]['file_name'] cleaned_file_name = utils.clean_name(file_name) overlap = len(set.intersection(set(cleaned_release_name), set(cleaned_file_name))) else: overlap = 0 subtitles.setdefault(movie_hash, []).append({ 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank, 'overlap' : overlap }) return subtitles def search_subtitles(self): search = [] for video_file_details in self.moviefiles.itervalues(): video_file_details['sublanguageid'] = self.lang search.append(video_file_details) results = self._query_opensubs(search) subtitles = self.clean_results(results) for (hash, found_matches) in subtitles.iteritems(): subtitles[hash] = utils.multikeysort(found_matches, ['overlap', 'user_rank', '-rating', '-downcount'])[0] for (hash, filedetails) in self.moviefiles.iteritems(): if not self.stopping: if subtitles.get(hash): utils.communicator.updategui.emit('Saving subtitles for %s'%filedetails['file_name'], 'success') subtitle = \ self.download_subtitles([subtitles[hash]['subid' ]]) utils.communicator.downloaded_sub.emit() utils.save_subs(subtitle, filedetails['save_subs_to'], subtitles[hash]) else: utils.communicator.no_sub_found.emit(filedetails['file_name']) else: return def download_subtitles(self, subparam): resp = self.server.DownloadSubtitles(self.login_token, subparam) self.check_status(resp) decoded = base64.standard_b64decode(resp['data'][0]['data' ].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed
class OpenSubtitles(object): def __init__(self): self.xmlrpc = ServerProxy(Settings.OPENSUBTITLES_SERVER, allow_none=True) self.language = Settings.LANGUAGE self.token = None def _get_from_data_or_none(self, key): """ Проверка на аутентификацию Args: key (str): строка с параметром который нужно вернуть Returns: (str) : возвращаемое значение """ status = self.data.get('status').split()[0] return self.data.get(key) if '200' == status else None def login(self): """ Авторизация Returns: (str) : токен """ self.data = self.xmlrpc.LogIn(Settings.USER_NAME, Settings.USER_PASSWORD, self.language, Settings.USER_AGENT) token = self._get_from_data_or_none('token') if token: self.token = token return token def logout(self): """ Логаут Returns: (str) : статус """ data = self.xmlrpc.LogOut(self.token) return '200' in data.get('status') def search_subtitles(self, params): """ Поиск субтитров Args: params(dict): + язык субтитров + id фильма на сайте *imdb.com* Returns: (dic) : + id субтитров + формат субтитров + id фильма """ self.data = self.xmlrpc.SearchSubtitles(self.token, params) return self.data def search_movies_on_imdb(self, params): """ Поиск фильмов Args: params(dict): + название фильма Returns: (dic) : + id фильма + название фильма """ self.data = self.xmlrpc.SearchMoviesOnIMDB(self.token, params) return self.data def download_subtitles(self, params): """ Загрузка субтитров Args: params(int): id субтитров Returns: (dic) : + id субтитров + строка с субтитрами """ self.data = self.xmlrpc.DownloadSubtitles(self.token, params) return self.data
class SDService(object): """ Contains the class that represents the OSDB RPC Server. Encapsules all the XMLRPC methods. Consult the OSDB API methods at http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC If it fails to connect directly to the XMLRPC server, it will try to do so through a default proxy. Default proxy uses a form to set which URL to open. We will try to change this in later stage. """ def __init__(self, server=None, proxy=None): self.log = logging.getLogger("subdownloader.SDService.SDService") self.log.debug( "Creating Server with server = %s and proxy = %r" % (server, proxy)) self.timeout = 30 self.user_agent = USER_AGENT self.language = '' if server: self.server = server else: self.server = DEFAULT_OSDB_SERVER self.proxy = proxy self.logged_as = None self._xmlrpc_server = None self._token = None def connected(self): return self._token is not None def connect(self): server = self.server proxy = self.proxy self.log.debug("connect()... to server %s with proxy %s" % (server, proxy)) connect_res = False try: self.log.debug( "Connecting with parameters (%r, %r)" % (server, proxy)) connect = TimeoutFunction(self._connect) connect_res = connect(server, proxy) except TimeoutFunctionException as e: self.log.error("Connection timed out. Maybe you need a proxy.") raise except: self.log.exception("connect: Unexpected error") raise finally: self.log.debug("connection connected %s" % connect_res) return connect_res def _connect(self, server, proxy): try: if proxy: self.log.debug("Trying proxied connection... ({})".format(proxy)) self.proxied_transport = ProxiedTransport() self.proxied_transport.set_proxy(proxy) self._xmlrpc_server = ServerProxy( server, transport=self.proxied_transport, allow_none=True) # self.ServerInfo() self.log.debug("...connected") return True elif test_connection(TEST_URL): self.log.debug("Trying direct connection...") self._xmlrpc_server = ServerProxy( server, allow_none=True) # self.ServerInfo() self.log.debug("...connected") return True else: self.log.debug("...failed") self.log.error("Unable to connect. Try setting a proxy.") return False except ProtocolError as e: self._connection_failed() self.log.debug("error in HTTP/HTTPS transport layer") raise except Fault as e: self.log.debug("error in xml-rpc server") raise except: self.log.exception("Connection to the server failed/other error") raise def logged_in(self): return self._token is not None def login(self, username="", password=""): try: login = TimeoutFunction(self._login) return login(username, password) except TimeoutFunctionException: self.log.error("login timed out") except: self.log.exception("login: other issue") raise def _login(self, username="", password=""): """Login to the Server using username/password, empty parameters means an anonymously login Returns True if login sucessful, and False if not. """ self.log.debug("----------------") self.log.debug("Logging in (username: %s)..." % username) def run_query(): return self._xmlrpc_server.LogIn( username, password, self.language, self.user_agent) info = self._safe_exec(run_query, None) if info is None: self._token = None return False self.log.debug("Login ended in %s with status: %s" % (info['seconds'], info['status'])) if info['status'] == "200 OK": self.log.debug("Session ID: %s" % info['token']) self.log.debug("----------------") self._token = info['token'] return True else: # force token reset self.log.debug("----------------") self._token = None return False def logout(self): try: logout = TimeoutFunction(self._logout) result = logout() self._token = None return result except TimeoutFunctionException: self.log.error("logout timed out") def _logout(self): """Logout from current session(token) This functions doesn't return any boolean value, since it can 'fail' for anonymous logins """ self.log.debug("Logging out from session ID: %s" % self._token) try: info = self._xmlrpc_server.LogOut(self._token) self.log.debug("Logout ended in %s with status: %s" % (info['seconds'], info['status'])) except ProtocolError as e: self.log.debug("error in HTTP/HTTPS transport layer") raise except Fault as e: self.log.debug("error in xml-rpc server") raise except: self.log.exception("Connection to the server failed/other error") raise finally: # force token reset self._token = None STATUS_CODE_RE = re.compile('(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise ProviderConnectionError(None, 'No message') log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format(status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(None, 'Server returned status="{status}". Expected "200 OK".'.format( status=data['status'])) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(code, message) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)') @classmethod def name(cls): return "opensubtitles" def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): try: result = query() return result except (ProtocolError, xml.parsers.expat.ExpatError): self._signal_connection_failed() log.debug("Query failed", exc_info=sys.exc_info()) return default @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str def imdb_query(self, query): if not self.connected(): return None def run_query(): return self._xmlrpc_server.SearchMoviesOnIMDB(self._token, query) result = self._safe_exec(run_query, None) if result is None: return None provider_identities = [] for imdb_data in result['data']: if not imdb_data: continue imdb_identity = ImdbIdentity(imdb_id=imdb_data['id'], imdb_rating=None) video_identity = VideoIdentity(name=imdb_data['title'], year=None) provider_identities.append(ProviderIdentities( video_identity=video_identity, imdb_identity=imdb_identity, provider=self)) return provider_identities SEARCH_LIMIT = 500 def search_text(self, text, languages=None): lang_str = self._languages_to_str(languages) query = { 'sublanguageid': lang_str, 'query': str(text), } queries = [query] def run_query(): return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT}) self._safe_exec(run_query, None) def search_videos(self, videos, callback, languages=None): if not self.connected(): return None lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate(window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown(remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime(rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitles_SubtitleFile( filename=remote_filename, file_size=remote_file_size , md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, age=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception('Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format(queries=queries)) log.error('Offending result is: {remote_sub}'.format(remote_sub=rsub_raw)) callback.finish() return remote_subtitles def _video_info_to_identification(self, video_info): name = video_info['MovieName'] year = int(video_info['MovieYear']) imdb_id = video_info['MovieImdbID'] video_identity = VideoIdentity(name=name, year=year) imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) episode_identity = None movie_kind = video_info['MovieKind'] if movie_kind == 'episode': season = int(video_info['SeriesSeason']) episode = int(video_info['SeriesEpisode']) episode_identity = EpisodeIdentity(season=season, episode=episode) elif movie_kind == 'movie': pass else: log.warning('Unknown MoviesKind="{}"'.format(video_info['MovieKind'])) return ProviderIdentities(video_identity=video_identity, episode_identity=episode_identity, imdb_identity=imdb_identity, provider=self) def identify_videos(self, videos): if not self.connected(): return for part_videos in window_iterator(videos, 200): hashes = [video.get_osdb_hash() for video in part_videos] hash_video = {hash: video for hash, video in zip(hashes, part_videos)} def run_query(): return self._xmlrpc_server.CheckMovieHash2(self._token, hashes) result = self._safe_exec(run_query, None) self.check_result(result) for video_hash, video_info in result['data'].items(): identification = self._video_info_to_identification(video_info[0]) video = hash_video[video_hash] video.add_identity(identification) def download_subtitles(self, os_rsubs): if not self.connected(): return None window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate(window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc_server.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({item['idsubtitlefile']: item['data'] for item in result['data']}) subtitles = [unzip_bytes(base64.b64decode(map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs] return subtitles def can_upload_subtitles(self, local_movie): if not self.connected(): return False query = {} for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()): # sub_bytes = open(subtitle.get_filepath(), mode='rb').read() # sub_tx_data = b64encode(zlib.compress(sub_bytes)) cd = "cd{i}".format(i=i+1) cd_data = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()), 'moviefps': video.get_fps(), 'movieframes': str(video.get_framecount()), 'moviefilename': video.get_filename(), } query[cd] = cd_data def run_query(): return self._xmlrpc_server.TryUploadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) movie_already_in_db = int(result['alreadyindb']) != 0 if movie_already_in_db: return False return True def upload_subtitles(self, local_movie): query = { 'baseinfo': { 'idmovieimdb': local_movie.get_imdb_id(), 'moviereleasename': local_movie.get_release_name(), 'movieaka': local_movie.get_movie_name(), 'sublanguageid': local_movie.get_language().xxx(), 'subauthorcomment': local_movie.get_comments(), }, } if local_movie.is_hearing_impaired() is not None: query['hearingimpaired'] = local_movie.is_hearing_impaired() if local_movie.is_high_definition() is not None: query['highdefinition'] = local_movie.is_high_definition() if local_movie.is_automatic_translation() is not None: query['automatictranslation'] = local_movie.is_automatic_translation() if local_movie.get_subtitle_author() is not None: query['subtranslator'] = local_movie.get_subtitle_author() if local_movie.is_foreign_only() is not None: query['foreignpartsonly'] = local_movie.is_foreign_only() for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()): sub_bytes = subtitle.get_filepath().open(mode='rb').read() sub_tx_data = b64encode(zlib.compress(sub_bytes)) cd = "cd{i}".format(i=i+1) cd_data = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None, 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), 'subcontent': sub_tx_data, } query[cd] = cd_data def run_query(): return self._xmlrpc_server.UploadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result)
subtitlesResultList['data'][subIndex]['LanguageName'] + '</b> subtitles for <b>' + videoTitle + '</b>...")', shell=True) elif opt_gui == 'kde': process_subtitlesDownload = subprocess.call( "(wget -q -O - " + subURL + " | gunzip > " + subPath + ") 2>&1", shell=True) else: # CLI print( ">> Downloading '" + subtitlesResultList['data'][subIndex]['LanguageName'] + "' subtitles for '" + videoTitle + "'") process_subtitlesDownload = 1 downloadResult = osd_server.DownloadSubtitles( session['token'], [subID]) if ('data' in downloadResult) \ and (downloadResult['data']) \ and (len(downloadResult['data']) > 0) \ and ('data' in downloadResult['data'][0]) \ and (downloadResult['data'][0]['data']): decodedBytes = base64.b64decode( downloadResult['data'][0]['data']) decompressed = gzip.decompress(decodedBytes) if len(decompressed) > 0: decodedStr = str(decompressed, subEncoding, 'replace') byteswritten = open( subPath, 'w', encoding=subEncoding,