class OpenSubtitlesClient: def __init__(self, username, password, language): self.client = ServerProxy('http://api.opensubtitles.org/xml-rpc', allow_none=True) self.token = self.client.LogIn(username, password, language, "TemporaryUserAgent")['token'] def get_list_subtitle_response(self, query, language): response = self.client.SearchSubtitles(self.token, [{ 'query': query, 'sublanguageid': language }], [{ 'limit': 3 }]) return response['data'] def download_encrypted_subtitles(self, subtitle_file_id): response = self.client.DownloadSubtitles(self.token, [subtitle_file_id]) decompresses_subtitle_file = zlib.decompress( base64.b64decode(response['data'][0]['data']), 16 + zlib.MAX_WBITS) return decompresses_subtitle_file def get_encryption_method(self, encrypted_subtitles): return chardet.detect(encrypted_subtitles)['encoding'] def decode(self, encryption_method, data): return data.decode(encryption_method)
def _perform_query_and_store(self, payload_for_sub_search: dict, proxy: ServerProxy): """ Searches for the desired subtitles through the OpenSubtitles API and writes the download URL information to a table ("search_subs"). :param payload_for_sub_search: (dictionary) contains the information about the movie for which the subtitle will download :param proxy: ServerProxy.LogIn(username, password, language, useragent) :return download_data: (dictionary) contains crucial information for file downloading """ try: query_result = proxy.SearchSubtitles(self.opensubs_token, [payload_for_sub_search], {"limit": 10}) except Fault: self.prompt_label.setText("A fault has occurred") except ProtocolError: self.prompt_label.setText("A ProtocolError has occurred.") else: if query_result["status"] == "200 OK": if query_result["data"]: payload_for_download = self._create_download_data( query_result["data"], payload_for_sub_search) self.interactor.add_subtitle_search_data_to_db( payload_for_download) else: self.prompt_label.setText( "There is no subtitles in this language for {}".format( payload_for_sub_search["query"])) else: self.prompt_label.setText("Wrong status code: {}".format( query_result["status"]))
def query_open_subtitles(movie_filename, language): uri = 'http://api.opensubtitles.org/xml-rpc' server = ServerProxy(uri, verbose=0, allow_none=True, use_datetime=True) login_info = server.LogIn('', '', 'en', 'ss v' + __version__) token = login_info['token'] try: guessit_query = obtain_guessit_query(movie_filename, language) search_queries = [ guessit_query, obtain_movie_hash_query(movie_filename, language), ] response = server.SearchSubtitles(token, search_queries) try: search_results = response['data'] except KeyError: # noqa raise KeyError('"data" key not found in response: %r' % response) if search_results: search_results = filter_bad_results(search_results, guessit_query) return search_results finally: server.LogOut(token)
class SubtitlesConncector: RETURNED_OK_STATUS = '200 OK' def __init__(self, user_name='', password='', language='', agent='OS Test User Agent'): self.user_name = user_name self.password = password self.language = language self.agent = agent self._api_adres = 'http://api.opensubtitles.org/xml-rpc' self.proxy_server = ServerProxy(self._api_adres) self.token = self._get_token() def _is_status_ok(self, status): if status == SubtitlesConncector.RETURNED_OK_STATUS: return True return False def _get_token(self): data = self.proxy_server.LogIn(self.user_name, self.password, self.language, self.agent) status = StatusParsser(data=data).parse() if self._is_status_ok(status): return data['token'] else: raise StatusError(status) def _save_zip(self, data_to_zip, path, file_name): b64img = base64.b64decode(data_to_zip.encode('ascii')) b64imgfile = open(str(path + "/" + file_name + ".zip"), 'w') b64imgfile.write(str(b64img)) def get_subtitles(self, subtitle, subtitles_dir=None): if not subtitles_dir: subtitles_dir = os.path.expanduser("~") + "/subtitles" if not os.path.exists(subtitles_dir): os.makedirs(subtitles_dir) data = self.proxy_server.DownloadSubtitles(self.token, [subtitle.idsubtitlefile]) try: data = data['data'][0]['data'] except Exception: raise GetSubtitlesError("Error: cant get data") self._save_zip(data, subtitles_dir, subtitle.subfilename) def search_subtitles(self, subtitles): parsed_data_to_send = ParseSubtitlesToSend(subtitles).parse() data = self.proxy_server.SearchSubtitles(self.token, [parsed_data_to_send]) data = ParseIncomingData(data).parse() collect = SubtitlesFindedCollection() for d in data: collect.append( SubtitlesFindedProperties(sublangid=d['SubLanguageID'], subfilename=d['SubFileName'], idsubtitlefile=d['IDSubtitleFile'])) return collect
def get_subtitles(db, l_post_id, l_imdbid): try: server = ServerProxy('http://api.opensubtitles.org/xml-rpc') token = server.LogIn('dh_moviebarcodes', 'dh_ws2015', 'eng', 'Moviebarcode Analyzer')['token'] imdb_id = int(l_imdbid[2:]) search_request = [] search_request.append({'imdbid': imdb_id, 'sublanguageid': 'eng'}) resp = server.SearchSubtitles(token, search_request) subtitle_id = [] try: sub = get_best_subtitle(resp['data']) subtitle_id.append(sub['IDSubtitleFile']) subtitle_data = server.DownloadSubtitles(token, subtitle_id) except IndexError: print("No subtitle available") return "" if subtitle_data['status'] == '200 OK': compressed_data = subtitle_data['data'][0]['data'] decoded_subtitle = base64.b64decode(compressed_data) # subtitle in byte format decoded_subtitle = gzip.GzipFile(fileobj=io.BytesIO(decoded_subtitle)).read() clean_subtitle = remove_invalid_characters(decoded_subtitle) allLemmasAndMostFrequentLemmas = lemmatize(clean_subtitle) print(allLemmasAndMostFrequentLemmas[0]) print(allLemmasAndMostFrequentLemmas[1]) update_value_in_db(db, l_post_id, "subtitlesLemmatisation", allLemmasAndMostFrequentLemmas[0]) update_value_in_db(db, l_post_id, "subtitlesMostFrequentWords", allLemmasAndMostFrequentLemmas[1]) except ValueError: print("No subtitle available")
class OpenSubtitlesApiWrapper(): def __init__(self): try: self.auth_token = "-1" ### state to check if initialized properly self.server_connector = ServerProxy(SUBTITLE_SERVER, allow_none=True) auth_response = self.server_connector.LogIn( USER_NAME, PASSWORD, "en", USER_AGENT_OPEN_SUTBTITLE) if auth_response['status'] != HTTP_CODE_200: raise Exception("Auth failed!!!") self.auth_token = auth_response["token"] except Exception as e: LOGGER.exception(str(e)) def pull_subtitles(self, media_source): movie_vs_id = {} movie_ids = [] ### to preserve ordering with relevancy media_file_name = os.path.basename(media_source) media_dir = media_source.replace(media_file_name, "") media_name = os.path.splitext(media_file_name)[0] response = self.server_connector.SearchMoviesOnIMDB( self.auth_token, media_name) if len(response['data']) > 0: for movie in response['data']: movie_vs_id[movie['id']] = movie['title'] movie_ids.append(movie['id']) #os.system("autosub " + media_source + " -o " + PROJECT_HOME + "/tmp/tmp.srt") print(movie_vs_id[movie_ids[0]]) if input( "Is this your media file for which you are searching subs y/n? " ) == "y": media_id = movie_ids[0] else: count = 0 for movie_id in movie_ids: print(str(count + 1) + ") " + movie_vs_id[movie_id]) count += 1 choice = int( input("Please choose between the options [1-" + str(count) + "] ")) media_id = movie_ids[choice - 1] print("Pulling subtitles from opensubtitle for " + movie_vs_id[media_id]) subtitles = self.server_connector.SearchSubtitles( self.auth_token, [{ "sublanguageid": "eng", "imdbid": media_id }])['data'] print(subtitles) counter = 1 for subtitle in subtitles: download_link = subtitle['SubDownloadLink'] print("Downloading sub(s) from " + download_link) downloaded_file = os.path.basename(download_link) os.system("cd " + media_dir + "; wget " + download_link + "; gunzip -c " + downloaded_file + " > " + media_name + "." + str(counter) + ".srt; rm -rf '*.gz'") counter += 1
class OpenSubtitles: def __init__(self): # noinspection SpellCheckingInspection opensubtitles_ua = b'VkxzdWIgMC4xMC4y' self.ua = base64.b64decode(opensubtitles_ua).decode() self.opensubtitles_api_url = 'http://api.opensubtitles.org/xml-rpc' self.opensubtitles_lang = 'en' self.token = None self.xmlrpc_transport = Transport() self.xmlrpc_transport.user_agent = self.ua self.xmlrpc = ServerProxy(self.opensubtitles_api_url, allow_none=True, transport=self.xmlrpc_transport) def retrieve_subtitles(self, ids: List[str]) -> Dict[str, str]: response = self.xmlrpc.DownloadSubtitles(self.token, ids) status = response.get('status', '').split()[0] if status == '200': subtitles_content = {} encoded_data = response.get('data') for item in encoded_data: subfile_id = item['idsubtitlefile'] try: decoded_data = decompress(item['data'], 'utf_8_sig') or decompress( item['data'], 'utf-8') except UnicodeDecodeError: decoded_data = decompress(item['data'], 'latin1') if not decoded_data: logger.error('An error occurred while decoding subtitle' 'file ID {}.'.format(subfile_id)) else: subtitles_content[subfile_id] = decoded_data return subtitles_content else: raise RuntimeWarning( f'Unable to get subtitles, returned status is {status}') def search_subtitles(self, param: List[dict]) -> List[Dict[str, str]]: response = self.xmlrpc.SearchSubtitles(self.token, param) status = response.get('status', '').split()[0] if status == '200': return response.get('data') else: raise RuntimeWarning( f'Unable to search subtitles, returned status is {status}') def login(self, username, password): response = self.xmlrpc.LogIn(username, password, self.opensubtitles_lang, self.ua) status = response.get('status', '').split()[0] if status == '200': self.token = response.get('token') else: raise RuntimeWarning( f'Unable to authenticate, returned status is {status}')
def query_open_subtitles(movie_filename, language): uri = 'http://api.opensubtitles.org/xml-rpc' server = ServerProxy(uri, verbose=0, allow_none=True, use_datetime=True) login_info = server.LogIn('', '', 'en', 'OS Test User Agent') token = login_info['token'] try: guessit_query = obtain_guessit_query(movie_filename, language) search_queries = [ guessit_query, obtain_movie_hash_query(movie_filename, language), ] response = server.SearchSubtitles(token, search_queries) search_results = response['data'] if search_results: search_results = filter_bad_results(search_results, guessit_query) return search_results finally: server.LogOut(token)
class SubtitleDownload: '''Traverses a directory and all subdirectories and downloads subtitles. Relies on an undocumented OpenSubtitles feature where subtitles seems to be sorted by filehash and popularity. ''' api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None moviefiles = [] movie_exts = (".avi", ".mkv", ".mp4") subb_exts = (".srt", ".sub", ".mpl") def __init__(self, movie_path, lang = "en"): print("OpenSubtitles Subtitle Downloader".center(78)) print("===================================".center(78)) self.server = ServerProxy(self.api_url, verbose=False) self.lang_id = lang # Traverse the directory tree and select all movie files for root, _, files in os.walk(movie_path): for file in files: if self.is_movie(file): file_path = os.path.join(root, file) if not self.subtitles_already_present(file_path): print("Found: " + file) filehash = self.hashFile(file_path) filesize = os.path.getsize(file_path) self.moviefiles.append({'dir': root, 'file': file, 'hash': filehash, 'size': filesize, 'subtitleid': None}) try: print("Login...") self.login() print("Searching for subtitles...") self.search_subtitles() print("Logout...") self.logout() except Error as e: print("XML-RPC error:", e) except UserWarning as uw: print(uw) def login(self): '''Log in to OpenSubtitles''' resp = self.server.LogIn('', '', 'en', 'OS Test User Agent') self.check_status(resp) self.login_token = resp['token'] def logout(self): '''Log out from OpenSubtitles''' resp = self.server.LogOut(self.login_token) self.check_status(resp) def search_subtitles(self): '''Search OpenSubtitles for matching subtitles''' search = [] for movie in self.moviefiles: search.append({'sublanguageid': self.lang_id, 'moviehash': movie['hash'], 'moviebytesize': str(movie['size'])}) # fixes weird problem where subs aren't found when only searching # for one movie. if len(search) == 1: search.append(search[0]) resp = self.server.SearchSubtitles(self.login_token, search) self.check_status(resp) if resp['data'] == False: print("No subtitles found") return subtitles = [] for result in resp['data']: if int(result['SubBad']) != 1: subtitles.append({'subid': result['IDSubtitleFile'], 'hash': result['MovieHash']}) downloadable_subs = subtitles[:] hash = None for s in subtitles: if hash == s['hash']: downloadable_subs.remove(s) hash = s['hash'] for ds in downloadable_subs: sub = self.download_subtitles(ds) for movie in self.moviefiles: if movie['hash'] == ds['hash']: print("Saving subtitle for: " + movie['file']) filename = os.path.join(movie['dir'], os.path.splitext(movie['file'])[0] + ".srt") file = open(filename, "wb") file.write(sub) file.close() def download_subtitles(self, subtitle): resp = self.server.DownloadSubtitles(self.login_token, [subtitle['subid']]) self.check_status(resp) decoded = base64.standard_b64decode(resp['data'][0]['data'].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed def is_movie(self, file): return os.path.splitext(file.lower())[1] in self.movie_exts def subtitles_already_present(self, file): file_base = os.path.splitext(file.lower())[0] for ext in self.subb_exts: if os.path.exists(file_base + ext): return True return False def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise UserWarning("Response error from " + self.api_url + ". Response status was: " + resp['status']) def hashFile(self, name): '''Calculates the hash value of a movie. Copied from OpenSubtitles own examples: http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes ''' try: longlongformat = 'q' # long long bytesize = struct.calcsize(longlongformat) f = open(name, "rb") filesize = os.path.getsize(name) hash = filesize if filesize < 65536 * 2: return "SizeError" for x in range(65536//bytesize): buffer = f.read(bytesize) (l_value,)= struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number f.seek(max(0,filesize-65536),0) for x in range(65536//bytesize): buffer = f.read(bytesize) (l_value,)= struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF f.close() returnedhash = "%016x" % hash return returnedhash except(IOError): return "IOError"
class OpenSubtitles(SubtitleProvider): URL = 'http://api.opensubtitles.org/xml-rpc' def __init__(self, settings=None): SubtitleProvider.__init__(self) self._xmlrpc = None self._token = None self._last_time = None if settings is None: settings = OpenSubtitlesSettings() self._settings = settings def get_settings(self): return self._settings def set_settings(self, settings): if self.connected(): raise RuntimeError( 'Cannot set settings while connected') # FIXME: change error self._settings = settings def connect(self): log.debug('connect()') if self.connected(): return self._xmlrpc = ServerProxy(self.URL, allow_none=False) self._last_time = time.time() def disconnect(self): log.debug('disconnect()') if self.logged_in(): self.logout() if self.connected(): self._xmlrpc = None def connected(self): return self._xmlrpc is not None def login(self): log.debug('login()') if self.logged_in(): return if not self.connected(): self.connect() def login_query(): # FIXME: 'en' language ok??? or '' as in the original return self._xmlrpc.LogIn(str(self._settings.username), str(self._settings.password), 'en', str(self._settings.get_user_agent())) result = self._safe_exec(login_query, None) self.check_result(result) self._token = result['token'] def logout(self): log.debug('logout()') if self.logged_in(): def logout_query(): return self._xmlrpc.LogOut(self._token) # Do no check result of this call. Assume connection closed. self._safe_exec(logout_query, None) self._token = None def logged_in(self): return self._token is not None def reestablish(self): log.debug('reestablish()') connected = self.connected() logged_in = self.logged_in() self.disconnect() if connected: self.connect() if logged_in: self.login() _TIMEOUT_MS = 60000 def _ensure_connection(self): now = time.time() if now - time.time() > self._TIMEOUT_MS: self.reestablish() self._last_time = now SEARCH_LIMIT = 500 def search_videos(self, videos, callback, languages=None): log.debug('search_videos(#videos={})'.format(len(videos))) if not self.logged_in(): raise ProviderNotConnectedError() lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate( window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } if video.get_osdb_hash() is None: log.debug('osdb hash of "{}" is empty -> skip'.format( video.get_filepath())) self._signal_connection_failed( ) # FIXME: other name + general signaling continue queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc.SearchSubtitles( self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown( remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime( rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitlesSubtitleFile( filename=remote_filename, file_size=remote_file_size, md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, date=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] try: imdb_rating = float(rsub_raw['MovieImdbRating']) except (ValueError, KeyError): imdb_rating = None imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=imdb_rating) video_name = rsub_raw['MovieName'] try: video_year = int(rsub_raw['MovieYear']) except (ValueError, KeyError): video_year = None video_identity = VideoIdentity(name=video_name, year=video_year) try: series_season = int(rsub_raw['SeriesSeason']) except (KeyError, ValueError): series_season = None try: series_episode = int(rsub_raw['SeriesEpisode']) except (KeyError, ValueError): series_episode = None series_identity = SeriesIdentity(season=series_season, episode=series_episode) identity = ProviderIdentities( video_identity=video_identity, imdb_identity=imdb_identity, episode_identity=series_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception( 'Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format( queries=queries)) log.error('Offending result is: {remote_sub}'.format( remote_sub=rsub_raw)) callback.finish() return remote_subtitles def query_text(self, query): return OpenSubtitlesTextQuery(query=query) def download_subtitles(self, os_rsubs): log.debug('download_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate( window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({ item['idsubtitlefile']: item['data'] for item in result['data'] }) subtitles = [ unzip_bytes(base64.b64decode( map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs ] return subtitles def upload_subtitles(self, local_movie): log.debug('upload_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() video_subtitles = list(local_movie.iter_video_subtitles()) if not video_subtitles: return UploadResult( type=UploadResult.Type.MISSINGDATA, reason=_('Need at least one subtitle to upload')) query_try = dict() for sub_i, (video, subtitle) in enumerate(video_subtitles): if not video: return UploadResult( type=UploadResult.Type.MISSINGDATA, reason=_('Each subtitle needs an accompanying video')) query_try['cd{}'.format(sub_i + 1)] = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), } def run_query_try_upload(): return self._xmlrpc.TryUploadSubtitles(self._token, query_try) try_result = self._safe_exec(run_query_try_upload, None) self.check_result(try_result) if int(try_result['alreadyindb']): return UploadResult(type=UploadResult.Type.DUPLICATE, reason=_('Subtitle is already in database')) if local_movie.get_imdb_id() is None: return UploadResult(type=UploadResult.Type.MISSINGDATA, reason=_('Need IMDb id')) upload_base_info = { 'idmovieimdb': local_movie.get_imdb_id(), } if local_movie.get_comments() is not None: upload_base_info['subauthorcomment'] = local_movie.get_comments() if not local_movie.get_language().is_generic(): upload_base_info['sublanguageid'] = local_movie.get_language().xxx( ) if local_movie.get_release_name() is not None: upload_base_info[ 'moviereleasename'] = local_movie.get_release_name() if local_movie.get_movie_name() is not None: upload_base_info['movieaka'] = local_movie.get_movie_name() if local_movie.is_hearing_impaired() is not None: upload_base_info[ 'hearingimpaired'] = local_movie.is_hearing_impaired() if local_movie.is_high_definition() is not None: upload_base_info[ 'highdefinition'] = local_movie.is_high_definition() if local_movie.is_automatic_translation() is not None: upload_base_info[ 'automatictranslation'] = local_movie.is_automatic_translation( ) if local_movie.get_author() is not None: upload_base_info['subtranslator'] = local_movie.get_author() if local_movie.is_foreign_only() is not None: upload_base_info['foreignpartsonly'] = local_movie.is_foreign_only( ) query_upload = { 'baseinfo': upload_base_info, } for sub_i, (video, subtitle) in enumerate(video_subtitles): sub_bytes = subtitle.get_filepath().open(mode='rb').read() sub_tx_data = base64.b64encode(zlib.compress(sub_bytes)).decode() query_upload['cd{}'.format(sub_i + 1)] = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None, 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), 'subcontent': sub_tx_data, } def run_query_upload(): return self._xmlrpc.UploadSubtitles(self._token, query_upload) result = self._safe_exec(run_query_upload, None) self.check_result(result) rsubs = [] for sub_data in result['data']: filename = sub_data['SubFileName'] file_size = sub_data['SubSize'] md5_hash = sub_data['SubHash'] id_online = sub_data['IDSubMOvieFile'] download_link = sub_data['SubDownloadLink'] link = None uploader = sub_data['UserNickName'] language = Language.from_xxx(sub_data['SubLanguageID']) rating = float(sub_data['SubRating']) add_date = datetime.datetime.strptime(sub_data['SubAddDate'], '%Y-%m-%d %H:%M:%S') sub = OpenSubtitlesSubtitleFile(filename=filename, file_size=file_size, md5_hash=md5_hash, id_online=id_online, download_link=download_link, link=link, uploader=uploader, language=language, rating=rating, date=add_date) rsubs.append(sub) return UploadResult(type=UploadResult.Type.OK, rsubs=rsubs) def imdb_search_title(self, title): self._ensure_connection() def run_query(): return self._xmlrpc.SearchMoviesOnIMDB(self._token, title.strip()) result = self._safe_exec(run_query, default=None) self.check_result(result) imdbs = [] re_title = re.compile(r'(?P<title>.*) \((?P<year>[0-9]+)\)') for imdb_data in result['data']: imdb_id = imdb_data['id'] if all(c in string.digits for c in imdb_id): imdb_id = 'tt{}'.format(imdb_id) m = re_title.match(imdb_data['title']) if m: imdb_title = m['title'] imdb_year = int(m['year']) else: imdb_title = imdb_data['title'] imdb_year = None imdbs.append( ImdbMovieMatch(imdb_id=imdb_id, title=imdb_title, year=imdb_year)) return imdbs def ping(self): log.debug('ping()') if not self.logged_in(): raise ProviderNotConnectedError() def run_query(): return self._xmlrpc.NoOperation(self._token) result = self._safe_exec(run_query, None) self.check_result(result) def provider_info(self): if self.connected(): def run_query(): return self._xmlrpc.ServerInfo() result = self._safe_exec(run_query, None) data = [ (_('XML-RPC version'), result['xmlrpc_version']), (_('XML-RPC url'), result['xmlrpc_url']), (_('Application'), result['application']), (_('Contact'), result['contact']), (_('Website url'), result['website_url']), (_('Users online'), result['users_online_total']), (_('Programs online'), result['users_online_program']), (_('Users logged in'), result['users_loggedin']), (_('Max users online'), result['users_max_alltime']), (_('Users registered'), result['users_registered']), (_('Subtitles downloaded'), result['subs_downloads']), (_('Subtitles available'), result['subs_subtitle_files']), (_('Number movies'), result['movies_total']), (_('Number languages'), result['total_subtitles_languages']), (_('Client IP'), result['download_limits']['client_ip']), (_('24h global download limit'), result['download_limits']['global_24h_download_limit']), (_('24h client download limit'), result['download_limits']['client_24h_download_limit']), (_('24h client download count'), result['download_limits']['client_24h_download_count']), (_('Client download quota'), result['download_limits']['client_download_quota']), ] else: data = [] return data @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str @classmethod def get_name(cls): return 'opensubtitles' @classmethod def get_short_name(cls): return 'os' @classmethod def get_icon(cls): return ':/images/sites/opensubtitles.png' def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): self._ensure_connection() try: result = query() return result except (ProtocolError, CannotSendRequest, SocketError, ExpatError) as e: self._signal_connection_failed() log.debug('Query failed: {} {}'.format(type(e), e.args)) return default STATUS_CODE_RE = re.compile(r'(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise OpenSubtitlesProviderConnectionError(None, _('No message')) log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format( status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise OpenSubtitlesProviderConnectionError( None, _('Server returned status="{status}". Expected "200 OK".'). format(status=data['status']), data['status']) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise OpenSubtitlesProviderConnectionError(code, message) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)')
class Agent(): """ opensubtitle 的用户代理""" # __username="" # __password="" # __languae="" # __osd_server="" # __session="" opt_selection_mode = 'default' opt_search_mode = "filename" opt_language_separator = '_' def __init__(self, username, password, language="en"): self.__username = username self.__password = password self.__languae = language self.__osd_server = ServerProxy('http://api.opensubtitles.org/xml-rpc') def LogIn(self): try: # ==== Connection try: self.__session = self.__osd_server.LogIn( self.__username, self.__password, self.__languae, 'opensubtitles-download 4.0') except Exception: # Retry once, it could be a momentary overloaded server? time.sleep(3) try: self.__session = self.__osd_server.LogIn( self.__username, self.__password, self.__languae, 'opensubtitles-download 4.0') except Exception: print("error1") #superPrint("error", "Connection error!", "Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!") sys.exit(2) # Connection refused? if self.__session['status'] != '200 OK': print("error2") #superPrint("error", "Connection error!", "Opensubtitles.org servers refused the connection: " + session['status'] + ".\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!") sys.exit(2) print("login success!") except: print("login error3") sys.exit(2) def LogOut(self): if self.__session and self.__session['token']: self.__osd_server.LogOut(self.__session['token']) print("logout success") def __search(self, title): """ 查询服务器,寻找潜在的字幕""" searchList = [] searchList.append({'sublanguageid': 'eng', 'query': title}) try: subtitlesList = self.__osd_server.SearchSubtitles( self.__session['token'], searchList) return subtitlesList # time.sleep(1) except Exception: # Retry once, we are already connected, the server maybe momentary overloaded time.sleep(3) try: subtitlesList = self.__osd_server.SearchSubtitles( self.__session['token'], searchList) return subtitlesList except Exception: print("Search error!") #superPrint("error", "Search error!", "Unable to reach opensubtitles.org servers!\n<b>Search error</b>") return -1 def __selectionAuto(self, subtitlesList, year, imdb_ID): """ 从众多字幕中寻找合适的字幕 """ """Automatic subtitles selection using filename match""" # 需要用到的属性 # IDMovieImdb the most import # MovieKind # MovieYear # SubDownloadLink # SubFileName # SubFormat # 顺序选择即可。 # if len(subtitlesList['data'])==1: # return 0 data = subtitlesList['data'] for i in range(len(data)): this = data[i] if this["IDMovieImdb"] != imdb_ID: continue # if this["SubFormat"]=="srt": # return i if this["MovieKind"] != "movie": continue if this["SubFormat"] == "srt" and abs(year - int(this["MovieYear"])) < 2: return i return -1 def __down(self, subtitlesSelected, subtitlesList, subPath): # 此处变量与下载有直接关系 subLangId = self.opt_language_separator + subtitlesList['data'][ subtitlesSelected]['ISO639'] subLangName = subtitlesList['data'][subtitlesSelected]['LanguageName'] subURL = subtitlesList['data'][subtitlesSelected]['SubDownloadLink'] # subPath 为存储路径 # subPath="1.txt" #subPath = videoPath.rsplit('.', 1)[0] + '.' + subtitlesList['data'][subIndex]['SubFormat'] if sys.version_info >= (3, 0): tmpFile1, headers = urllib.request.urlretrieve(subURL) tmpFile2 = gzip.GzipFile(tmpFile1) byteswritten = open(subPath, 'wb').write(tmpFile2.read()) if byteswritten > 0: process_subtitlesDownload = 0 else: process_subtitlesDownload = 1 else: # python 2 tmpFile1, headers = urllib.urlretrieve(subURL) tmpFile2 = gzip.GzipFile(tmpFile1) open(subPath, 'wb').write(tmpFile2.read()) process_subtitlesDownload = 0 # If an error occurs, say so if process_subtitlesDownload != 0: print("Subtitling error!") #superPrint("error", "Subtitling error!", "An error occurred while downloading or writing <b>" + subtitlesList['data'][subIndex]['LanguageName'] + "</b> subtitles for <b>" + videoTitle + "</b>.") self.__osd_server.LogOut(self.__session['token']) sys.exit(2) def __result(self, subtitlesList, year, imdb_ID, subPath): """ 主体函数 """ # Parse the results of the XML-RPC query if ('data' in subtitlesList) and (len(subtitlesList['data']) > 0): # print("find it") #找到数据 # Mark search as successful subtitlesSelected = '' # If there is only one subtitles (matched by file hash), auto-select it (except in CLI mode) if (len(subtitlesList['data']) == 1) and (subtitlesList['data'][0]['MatchedBy'] == 'moviehash'): if opt_selection_mode != 'manual': # subtitlesSelected = subtitlesList['data'][0]['SubFileName'] subtitlesSelected = 0 # Get video title videoTitle = subtitlesList['data'][0]['MovieName'] # If there is more than one subtitles and opt_selection_mode != 'auto', # then let the user decide which one will be downloaded if subtitlesSelected == '': subtitlesSelected = self.__selectionAuto( subtitlesList, year, imdb_ID) if subtitlesSelected == -1: return -1 # If a subtitles has been selected at this point, download it! if subtitlesSelected == -1: # 记录未下载的电影名与ID #print("find error!") return -1 else: # print("ready to down") self.__down(subtitlesSelected, subtitlesList, subPath) return 0 else: # print("not find it!") return -1 def work(self, title, year, imdb_ID, path): """ 查找电影字幕并下载到指定路径中""" subtitlesList = self.__search(title) time.sleep(2) if subtitlesList == -1: return -1 return self.__result(subtitlesList, year, imdb_ID, path) pass def work_by_imda(self, year, imdb_ID): list = self.__search(imdb_ID) if list == -1: return -1 return self.__selectionAuto(list, year, imdb_ID) pass pass
class OpenSubtitles(QtCore.QThread): api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None def __init__(self, parent=None): QtCore.QThread.__init__(self, parent) self.stopping = False def process(self, files_list, lang='English'): self.moviefiles = files_list self.imdbid_to_hash = {} self.lang = LANGUAGES[lang][1] if not self.stopping: self.start() def __del__(self): if self.login_token: self.logout() def stopTask(self): self.stopping = True def run(self): utils.communicator.updategui.emit('Querying OpenSubtitles.org...', 'info' ) try: if not self.login_token: self.login() self.search_subtitles() except utils.NoInternetConnectionFound: utils.communicator.updategui.emit('No active Internet connection found. Kindly check and try again.', 'error') def login(self): '''Log in to OpenSubtitles.org''' self.server = ServerProxy(self.api_url, verbose=False) utils.communicator.updategui.emit('Logging in to OpenSubtitles.org...', 'info' ) try: resp = self.server.LogIn('', '', 'en', 'PySubD v2.0') self.check_status(resp) self.login_token = resp['token'] except Exception as e: if e.args[0] == 11004: utils.communicator.updategui.emit("No Internet Connection Found", 'error') else: utils.communicator.updategui.emit(str(e), 'error') def logout(self): '''Log out from OpenSubtitles''' utils.communicator.updategui.emit('Logging out...', 'info') resp = self.server.LogOut(self.login_token) self.check_status(resp) def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise utils.IncorrectResponseRecieved('Response error from %s. Response status: %s' % (self.api_url, resp['status'])) def _query_opensubs(self, search): results = [] while search[:500]: if not self.stopping: try: tempresp = self.server.SearchSubtitles(self.login_token, search[:500]) except Exception as e: if e.args[0] == 11004: utils.communicator.updategui.emit("No Internet Connection Found", 'error') else: utils.communicator.updategui.emit(str(e), 'error') return results if tempresp['data']: results.extend(tempresp['data']) self.check_status(tempresp) search = search[500:] else: return [] logger.debug('Results: %s' % results) return results def clean_results(self, results, imdb=False): subtitles = {} user_ranks = { 'administrator': 1, 'platinum member': 2, 'vip member': 3, 'gold member': 4, 'trusted': 5, 'silver member': 6, 'bronze member': 7, 'sub leecher': 8, '': 9, } for result in results: if result['SubBad'] != '1': movie_hash = result.get('MovieHash') if not movie_hash: movie_hash = self.imdbid_to_hash[int(result['IDMovieImdb'])] subid = result['IDSubtitleFile'] downcount = int(result['SubDownloadsCnt']) rating = float(result['SubRating']) if rating and rating < 8: # Ignore poorly rated subtitles, while not # penalizing the ones that haven't yet been rated continue user_rank = user_ranks[result['UserRank']] if imdb: cleaned_release_name = utils.clean_name(result['MovieReleaseName']) file_name = self.moviefiles[movie_hash]['file_name'] cleaned_file_name = utils.clean_name(file_name) overlap = len(set.intersection(set(cleaned_release_name), set(cleaned_file_name))) else: overlap = 0 subtitles.setdefault(movie_hash, []).append({ 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank, 'overlap' : overlap }) return subtitles def search_subtitles(self): search = [] for video_file_details in self.moviefiles.itervalues(): video_file_details['sublanguageid'] = self.lang search.append(video_file_details) results = self._query_opensubs(search) subtitles = self.clean_results(results) for (hash, found_matches) in subtitles.iteritems(): subtitles[hash] = utils.multikeysort(found_matches, ['overlap', 'user_rank', '-rating', '-downcount'])[0] for (hash, filedetails) in self.moviefiles.iteritems(): if not self.stopping: if subtitles.get(hash): utils.communicator.updategui.emit('Saving subtitles for %s'%filedetails['file_name'], 'success') subtitle = \ self.download_subtitles([subtitles[hash]['subid' ]]) utils.communicator.downloaded_sub.emit() utils.save_subs(subtitle, filedetails['save_subs_to'], subtitles[hash]) else: utils.communicator.no_sub_found.emit(filedetails['file_name']) else: return def download_subtitles(self, subparam): resp = self.server.DownloadSubtitles(self.login_token, subparam) self.check_status(resp) decoded = base64.standard_b64decode(resp['data'][0]['data' ].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed
class OpenSubtitles(object): def __init__(self): self.xmlrpc = ServerProxy(Settings.OPENSUBTITLES_SERVER, allow_none=True) self.language = Settings.LANGUAGE self.token = None def _get_from_data_or_none(self, key): """ Проверка на аутентификацию Args: key (str): строка с параметром который нужно вернуть Returns: (str) : возвращаемое значение """ status = self.data.get('status').split()[0] return self.data.get(key) if '200' == status else None def login(self): """ Авторизация Returns: (str) : токен """ self.data = self.xmlrpc.LogIn(Settings.USER_NAME, Settings.USER_PASSWORD, self.language, Settings.USER_AGENT) token = self._get_from_data_or_none('token') if token: self.token = token return token def logout(self): """ Логаут Returns: (str) : статус """ data = self.xmlrpc.LogOut(self.token) return '200' in data.get('status') def search_subtitles(self, params): """ Поиск субтитров Args: params(dict): + язык субтитров + id фильма на сайте *imdb.com* Returns: (dic) : + id субтитров + формат субтитров + id фильма """ self.data = self.xmlrpc.SearchSubtitles(self.token, params) return self.data def search_movies_on_imdb(self, params): """ Поиск фильмов Args: params(dict): + название фильма Returns: (dic) : + id фильма + название фильма """ self.data = self.xmlrpc.SearchMoviesOnIMDB(self.token, params) return self.data def download_subtitles(self, params): """ Загрузка субтитров Args: params(int): id субтитров Returns: (dic) : + id субтитров + строка с субтитрами """ self.data = self.xmlrpc.DownloadSubtitles(self.token, params) return self.data
class OpenSubtitles(SubtitleProvider): URL = 'http://api.opensubtitles.org/xml-rpc' def __init__(self): SubtitleProvider.__init__(self) self._xmlrpc = None self._token = None self._settings = OpenSubtitlesSettings() def get_settings(self): return self._settings def set_settings(self, settings): self._settings = settings def connect(self): log.debug('connect()') if self.connected(): return self._xmlrpc = ServerProxy(self.URL, allow_none=False) def disconnect(self): log.debug('disconnect()') if self.logged_in(): self.logout() if self.connected(): self._xmlrpc = None def connected(self): return self._xmlrpc is not None def login(self): log.debug('login()') if self.logged_in(): return if not self.connected(): raise ProviderNotConnectedError() def login_query(): # FIXME: 'en' language ok??? or '' as in the original return self._xmlrpc.LogIn(str(self._settings.username), str(self._settings.password), 'en', str(self._settings.get_user_agent())) result = self._safe_exec(login_query, None) self.check_result(result) self._token = result['token'] def logout(self): log.debug('logout()') if self.logged_in(): def logout_query(): return self._xmlrpc.LogOut(self._token) result = self._safe_exec(logout_query, None) self.check_result(result) self._token = None def logged_in(self): return self._token is not None SEARCH_LIMIT = 500 def search_videos(self, videos, callback, languages=None): log.debug('search_videos(#videos={})'.format(len(videos))) if not self.logged_in(): raise ProviderNotConnectedError() lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate( window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc.SearchSubtitles( self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown( remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime( rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitlesSubtitleFile( filename=remote_filename, file_size=remote_file_size, md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, age=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception( 'Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format( queries=queries)) log.error('Offending result is: {remote_sub}'.format( remote_sub=rsub_raw)) callback.finish() return remote_subtitles def query_text(self, query): return OpenSubtitlesTextQuery(query=query) def download_subtitles(self, os_rsubs): log.debug('download_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate( window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({ item['idsubtitlefile']: item['data'] for item in result['data'] }) subtitles = [ unzip_bytes(base64.b64decode( map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs ] return subtitles def ping(self): log.debug('ping()') if not self.logged_in(): raise ProviderNotConnectedError() def run_query(): return self._xmlrpc.NoOperation(self._token) result = self._safe_exec(run_query, None) self.check_result(result) @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str @classmethod def get_name(cls): return 'opensubtitles' @classmethod def get_short_name(cls): return 'os' def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): try: result = query() return result except (ProtocolError, CannotSendRequest, SocketError): self._signal_connection_failed() log.warning('Query failed', exc_info=sys.exc_info()) return default STATUS_CODE_RE = re.compile('(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise ProviderConnectionError(_('No message')) log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format( status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError( _('Server returned status="{status}". Expected "200 OK".'). format(status=data['status']), data['status']) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(message, code) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)')
class SubtitleDownload(QtCore.QThread): '''Traverses a directory and all subdirectories and downloads subtitles. Relies on an undocumented OpenSubtitles feature where subtitles seems to be sorted by filehash and popularity. ''' api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None def __init__(self, parent=None): QtCore.QThread.__init__(self, parent) def init(self, movie_paths): self._movie_paths = movie_paths self.start() def __del__(self): if self.login_token: self.logout() def stopTask(self): self.stopping = True def run(self): self.stopping = False self.moviefiles = {} self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Searching for video files...") """Check if the movie_path is a file or directory. Calculate the file stats if its a file, else traverse the directory tree and calculate filestats for all movie files""" for path in self._movie_paths: if os.path.isfile(path): self.calcfilestats(path, os.path.dirname(path)) else: for root, _, files in os.walk(path): for file in files: if not self.stopping: self.calcfilestats(file, root) else: return try: if self.moviefiles: if not self.login_token: try: self.login() except (noInternetConnection): self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, No active Internet connection found. Re-Check and try again." ) my_logger.debug( "Sorry, No active Internet connection found. Re-Check and try again." ) self.emit( QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"), self._movie_paths) return self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Searching for subtitles...") self.search_subtitles() self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Done...") my_logger.debug("Done...") else: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, no video files were found!") self.emit(QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"), self._movie_paths) except Error as e: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), ("XML-RPC error:", e)) except UserWarning as uw: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), uw) def calcfilestats(self, file, parentdir): ''' Calculates and adds the hash of a movie file to the moviefiles dictionary. Also emits a signal to update the LCD counter showing the found files.''' video_extns = [ '.avi', '.divx', '.mkv', '.mp4', '.ogg', '.rm', '.rmvb', '.vob', '.x264', '.xvid' ] if os.path.splitext(file)[1].lower() in video_extns: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Found: " + file) my_logger.debug("Found: " + file) self.emit(QtCore.SIGNAL("updateFound()")) filehash = self.hashFile(os.path.join(parentdir, file)) filesize = os.path.getsize(os.path.join(parentdir, file)) self.moviefiles[filehash] = { 'dir': parentdir, 'file': file, 'size': filesize } def login(self): '''Log in to OpenSubtitles''' self.server = ServerProxy(self.api_url, verbose=False) self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging in...") my_logger.debug("Logging in...") try: resp = self.server.LogIn('', '', 'en', 'PySubD v1.0') self.check_status(resp) self.login_token = resp['token'] except Error as e: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), e) my_logger.debug(str(e)) def logout(self): '''Log out from OpenSubtitles''' self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging out...") resp = self.server.LogOut(self.login_token) self.check_status(resp) def search_subtitles(self): '''Search OpenSubtitles for matching subtitles using moviehash and filebytesize''' search = [] for hash in self.moviefiles.keys(): search.append({ 'sublanguageid': 'eng', 'moviehash': hash, 'moviebytesize': str(self.moviefiles[hash]['size']) }) my_logger.debug("Length of search string: " + str(len(search))) resp = {} resp['data'] = [] while search[:500]: if not self.stopping: tempresp = self.server.SearchSubtitles(self.login_token, search[:500]) if tempresp['data'] != False: resp['data'].extend(tempresp['data']) self.check_status(tempresp) search = search[500:] else: return #Check if we actually got some matching subtitles to download, else return if not resp['data']: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, no subtitles were found!") my_logger.debug("Sorry, no subtitles were found!") return #User Ranks on OpenSubtitles.org.These are decided based upon the number of subtitles uploaded by the member. #A better rank is often an indication of a better subtitle quality. user_ranks = { 'admin': 1, 'platinum member': 2, 'vip member': 3, 'gold member': 4, 'trusted': 5, 'silver member': 6, 'bronze member': 7, 'sub leecher': 8, '': 9 } # A dictionary to store the subtitle id's found corresponding to every file hash subtitles = {} for result in resp['data']: #The subtitle must not be rated bad if int(result['SubBad']) != 1: hash = result['MovieHash'] subid = result['IDSubtitleFile'] downcount = result['SubDownloadsCnt'] rating = result['SubRating'] user_rank = user_ranks[result['UserRank']] #First good matching subtitle found if not subtitles.get(hash): subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } self.emit(QtCore.SIGNAL("updateAvailable()")) #Another good quality subtitle found uploaded by a more reputed user elif subtitles[hash]['user_rank'] > user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } #Another good quality subtitle found with a perfect rating, uploaded by a more or equally reputed user elif float(rating) == 10.0 and subtitles[hash][ 'user_rank'] >= user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } #Another good quality subtitle found with the better rating, higher download count and uploaded by a more or equally trusted user elif float(subtitles[hash]['rating']) >= float(rating) \ and int(subtitles[hash]['downcount']) < int(downcount) and subtitles[hash]['user_rank'] >= user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } my_logger.debug("Total number of subtitles found: " + str(len(subtitles))) notfound = [] for hash, filedetails in self.moviefiles.iteritems(): if not self.stopping: if subtitles.get(hash): try: subtitle = self.download_subtitles( [subtitles[hash]['subid']]) self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Saving subtitle for: " + filedetails['file']) my_logger.debug("Saving subtitle for: " + filedetails['file'] + " Hash : " + hash + " Rating: " + subtitles[hash]['rating'] + " DwnCnt: " + subtitles[hash]['downcount'] + " UpldrRnk:" + \ str(subtitles[hash]['user_rank'])) self.emit(QtCore.SIGNAL("updateDownloaded()")) filename = os.path.join( filedetails['dir'], os.path.splitext(filedetails['file'])[0] + ".srt") file = open(filename, "wb") file.write(subtitle) file.close() except IOError: self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "IO Error in saving subs for " + filedetails['file']) else: notfound.append(filedetails['file']) else: return #Report all the files for which no subtitles were found in an alphabetically sorted order. notfound.sort(key=str.lower) for file in notfound: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "No subtitles found for: " + file) my_logger.debug("No subtitles were found for: " + file) def download_subtitles(self, subparam): resp = self.server.DownloadSubtitles(self.login_token, subparam) self.check_status(resp) decoded = base64.standard_b64decode( resp['data'][0]['data'].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise UserWarning("Response error from " + self.api_url + ". Response status was: " + resp['status']) def hashFile(self, name): ''' Calculates the hash value of a movie. Copied from OpenSubtitles own examples: http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes ''' try: longlongformat = 'q' # long long bytesize = struct.calcsize(longlongformat) f = open(name, "rb") filesize = os.path.getsize(name) hash = filesize if filesize < 65536 * 2: return "SizeError: Minimum file size must be 120Kb" for x in range(65536 // bytesize): buffer = f.read(bytesize) (l_value, ) = struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number f.seek(max(0, filesize - 65536), 0) for x in range(65536 // bytesize): buffer = f.read(bytesize) (l_value, ) = struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF f.close() returnedhash = "%016x" % hash return returnedhash except (IOError): return "IOError"
if opt_search_mode in ('hash', 'hash_then_filename', 'hash_and_filename'): subtitlesSearchList.append({ 'sublanguageid': currentLanguage, 'moviehash': videoHash, 'moviebytesize': str(videoSize) }) if opt_search_mode in ('filename', 'hash_and_filename'): subtitlesSearchList.append({ 'sublanguageid': currentLanguage, 'query': videoFileName }) ## Primary search try: subtitlesResultList = osd_server.SearchSubtitles( session['token'], subtitlesSearchList) except Exception: # Retry once after a delay (we are already connected, the server may be momentary overloaded) time.sleep(3) try: subtitlesResultList = osd_server.SearchSubtitles( session['token'], subtitlesSearchList) except Exception: superPrint( "error", "Search error!", "Unable to reach opensubtitles.org servers!\n<b>Search error</b>" ) #if (opt_search_mode == 'hash_and_filename'): # TODO Cleanup duplicate between moviehash and filename results
videoSize = os.path.getsize(videoPath) videoFileName = os.path.basename(videoPath) # Count languages marked for this search for SubLanguageID in SubLanguageIDs: searchLanguage += len(SubLanguageID.split(',')) # Search for available subtitles (using file hash and size) for SubLanguageID in SubLanguageIDs: searchList = [] searchList.append({ 'sublanguageid': SubLanguageID, 'moviehash': videoHash, 'moviebytesize': str(videoSize) }) subtitlesList = server.SearchSubtitles(token, searchList) subtitlesSelected = '' # Parse the results of the XML-RPC query if subtitlesList['data']: # Mark search as successful searchLanguageResult += 1 # If there is only one subtitles, auto-select it if len(subtitlesList['data']) == 1: subtitlesSelected = subtitlesList['data'][0]['SubFileName'] # If there is more than one subtitles and opt_selection_mode != 'auto', # then let the user decide which one will be downloaded if len(subtitlesList['data']) > 1:
class OpenSub(object): def __init__(self, path): self.TAGS = [ 'bluray', 'cam', 'dvb', 'dvd', 'hd-dvd', 'hdtv', 'ppv', 'telecine', 'telesync', 'tv', 'vhs', 'vod', 'web-dl', 'webrip', 'workprint' ] self.OPENSUBTITLES_SERVER = 'https://api.opensubtitles.org:443/xml-rpc' self.USER_AGENT = 'TemporaryUserAgent' # needs to be changed from time to time to the recent temporary UA self.xmlrpc = ServerProxy(self.OPENSUBTITLES_SERVER, allow_none=True) self.language = 'eng' self.token = None self.mv_name = None self.path = path try: with open("opensubtitles_userpass.txt", 'r') as f: f = f.read() user = f.split('\n')[0].split('=')[1] password = f.split('\n')[1].split('=')[1] self.user = user self.passw = password except FileNotFoundError: print( "Please fill OpenSubtitles credentials in opensubtitles_userpass.txt" ) if self.login(self.user, self.passw): print("[OpenSubtitles] Login Successful") else: print("[OpenSubtitles] Login Failed") def _get_from_data_or_none(self, key): '''Return the key recieved from data if the status is 200, otherwise return None. ''' status = self.data.get('status').split()[0] return self.data.get(key) if '200' == status else None def login(self, username, password): '''Returns token is login is ok, otherwise None. ''' self.data = self.xmlrpc.LogIn(username, password, self.language, self.USER_AGENT) token = self._get_from_data_or_none('token') if token: self.token = token return token def logout(self): '''Returns True if logout is ok, otherwise None. ''' data = self.xmlrpc.LogOut(self.token) return '200' in data.get('status') def search_subtitles(self, params): '''Returns a list with the subtitles info. ''' self.data = self.xmlrpc.SearchSubtitles(self.token, params) return self._get_from_data_or_none('data') def download_subtitles(self, payload): """ :return: """ print("Attempting to download subtitles...") search_result = self.search_subtitles([payload]) if search_result: dllink = self.analyse_result(search_result, payload) gzfile = request.urlopen(dllink) try: with gzip.open(gzfile, 'rb') as f: with open(self.path + "\\" + self.mv_name + '.srt', 'wb') as sub_file: sub_file.write(f.read()) print("[OpenSubtitles] Subtitles downloaded for: " + self.mv_name) # Update stats with open("metadata.json", "r") as jsonFile: data = json.load(jsonFile) data['num_of_movies_in_db'] = int( data['num_of_movies_in_db']) + 1 data['latest_movie_added']['name'] = self.mv_name data['latest_movie_added']['time'] = str( datetime.datetime.now()) with open("metadata.json", "w") as jsonFile: json.dump(data, jsonFile, indent=4) # End Update return os.path.join(self.path, self.mv_name + ".srt") except PermissionError: print( "[OpenSubtitles] Permission Error: when creating subtitles for {}" .format(self.mv_name)) def analyse_result(self, result, payload): """ :param result: Search result to find appropriate subtitles :return: Download Link of best match for subtitles """ score = 0 dllink = None for record in result: if record.get("MovieName").lower() == payload['query'].lower( ) and record.get("SubLanguageID") == 'eng': self.mv_name = record.get("MovieName") dllink = record.get('SubDownloadLink') break if record.get('Score', 0) > score and record.get("SubLanguageID") == 'eng': self.mv_name = record.get("MovieName") score = record.get('Score', 0) dllink = record.get('SubDownloadLink') return dllink def create_payload(self, mv_name): """ :param lang: Subtitle Language :param mv_name: Movie name to search :return: Payload containing data about file """ payload = {} payload['query'] = mv_name payload['SubLanguageID'] = self.language payload['SubFormat'] = 'srt' return payload def search_and_download(self, mv_name): movie_db = [ os.path.splitext(name)[0] for name in os.listdir(self.path) ] for local in movie_db: if mv_name.lower() == local.lower(): return os.path.join(self.path, mv_name + ".srt") try: return self.download_subtitles(self.create_payload(mv_name)) except: print("[OpenSubtitles] Could not download subtitles for " + mv_name) def download_top_1000(self): with open(os.path.join(os.getcwd(), "top1000.txt"), "r") as f: f = f.read() i = 1 for mv_name in f.split("\n"): try: self.search_and_download(mv_name) except: e = sys.exc_info()[0] print("ERROR: " + str(e)) print("Error: subtitles not downloaded for: " + mv_name) i = i + 1 if i % 39 == 0: time.sleep(11)
def on_task_download(self, task, config): # filter all entries that have IMDB ID set try: entries = [e for e in task.accepted if e['imdb_id'] is not None] except KeyError: # No imdb urls on this task, skip it # TODO: should do lookup via imdb_lookup plugin? return try: s = ServerProxy("http://api.opensubtitles.org/xml-rpc") res = s.LogIn("", "", "en", "FlexGet") except Exception: logger.warning('Error connecting to opensubtitles.org') return if res['status'] != '200 OK': raise Exception("Login to opensubtitles.org XML-RPC interface failed") config = self.prepare_config(config, task) token = res['token'] # configuration languages = config['languages'] min_sub_rating = config['min_sub_rating'] match_limit = config[ 'match_limit' ] # no need to change this, but it should be configurable # loop through the entries for entry in entries: imdbid = entry.get('imdb_id') if not imdbid: logger.debug('no match for {}', entry['title']) continue query = [] for language in languages: query.append({'sublanguageid': language, 'imdbid': imdbid}) subtitles = s.SearchSubtitles(token, query) subtitles = subtitles['data'] # nothing found -> continue if not subtitles: continue # filter bad subs subtitles = [x for x in subtitles if x['SubBad'] == '0'] # some quality required (0.0 == not reviewed) subtitles = [ x for x in subtitles if float(x['SubRating']) >= min_sub_rating or float(x['SubRating']) == 0.0 ] filtered_subs = [] # find the best rated subs for each language for language in languages: langsubs = [x for x in subtitles if x['SubLanguageID'] == language] # did we find any subs for this language? if langsubs: def seqmatch(subfile): s = difflib.SequenceMatcher(lambda x: x in " ._", entry['title'], subfile) # print "matching: ", entry['title'], subfile, s.ratio() return s.ratio() > match_limit # filter only those that have matching release names langsubs = [x for x in subtitles if seqmatch(x['MovieReleaseName'])] if langsubs: # find the best one by SubRating langsubs.sort(key=lambda x: float(x['SubRating'])) langsubs.reverse() filtered_subs.append(langsubs[0]) # download for sub in filtered_subs: logger.debug( 'SUBS FOUND: {} {} {}', sub['MovieReleaseName'], sub['SubRating'], sub['SubLanguageID'], ) f = task.requests.get(sub['ZipDownloadLink']) subfilename = re.match( '^attachment; filename="(.*)"$', f.headers['content-disposition'] ).group(1) outfile = os.path.join(config['output'], subfilename) fp = open(outfile, 'w') fp.write(f.raw) fp.close() f.close() s.LogOut(token)
videoSize = os.path.getsize(videoPath) videoFileName = os.path.basename(videoPath) # ==== Search for available subtitles on OpenSubtitlesDownload for SubLanguageID in opt_languages: searchList = [] subtitlesList = {} if opt_search_mode in ('hash', 'hash_then_filename', 'hash_and_filename'): searchList.append({'sublanguageid':SubLanguageID, 'moviehash':videoHash, 'moviebytesize':str(videoSize)}) if opt_search_mode in ('filename', 'hash_and_filename'): searchList.append({'sublanguageid':SubLanguageID, 'query':videoFileName}) ## Primary search try: subtitlesList = osd_server.SearchSubtitles(session['token'], searchList) except Exception: # Retry once after a delay (we are already connected, the server may be momentary overloaded) time.sleep(3) try: subtitlesList = osd_server.SearchSubtitles(session['token'], searchList) except Exception: superPrint("error", "Search error!", "Unable to reach opensubtitles.org servers!\n<b>Search error</b>") #if (opt_search_mode == 'hash_and_filename'): # TODO Cleanup duplicate between moviehash and filename results ## Fallback search if ((opt_search_mode == 'hash_then_filename') and (('data' in subtitlesList) and (not subtitlesList['data']))): searchList[:] = [] # searchList.clear() searchList.append({'sublanguageid':SubLanguageID, 'query':videoFileName})
class SDService(object): """ Contains the class that represents the OSDB RPC Server. Encapsules all the XMLRPC methods. Consult the OSDB API methods at http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC If it fails to connect directly to the XMLRPC server, it will try to do so through a default proxy. Default proxy uses a form to set which URL to open. We will try to change this in later stage. """ def __init__(self, server=None, proxy=None): self.log = logging.getLogger("subdownloader.SDService.SDService") self.log.debug( "Creating Server with server = %s and proxy = %r" % (server, proxy)) self.timeout = 30 self.user_agent = USER_AGENT self.language = '' if server: self.server = server else: self.server = DEFAULT_OSDB_SERVER self.proxy = proxy self.logged_as = None self._xmlrpc_server = None self._token = None def connected(self): return self._token is not None def connect(self): server = self.server proxy = self.proxy self.log.debug("connect()... to server %s with proxy %s" % (server, proxy)) connect_res = False try: self.log.debug( "Connecting with parameters (%r, %r)" % (server, proxy)) connect = TimeoutFunction(self._connect) connect_res = connect(server, proxy) except TimeoutFunctionException as e: self.log.error("Connection timed out. Maybe you need a proxy.") raise except: self.log.exception("connect: Unexpected error") raise finally: self.log.debug("connection connected %s" % connect_res) return connect_res def _connect(self, server, proxy): try: if proxy: self.log.debug("Trying proxied connection... ({})".format(proxy)) self.proxied_transport = ProxiedTransport() self.proxied_transport.set_proxy(proxy) self._xmlrpc_server = ServerProxy( server, transport=self.proxied_transport, allow_none=True) # self.ServerInfo() self.log.debug("...connected") return True elif test_connection(TEST_URL): self.log.debug("Trying direct connection...") self._xmlrpc_server = ServerProxy( server, allow_none=True) # self.ServerInfo() self.log.debug("...connected") return True else: self.log.debug("...failed") self.log.error("Unable to connect. Try setting a proxy.") return False except ProtocolError as e: self._connection_failed() self.log.debug("error in HTTP/HTTPS transport layer") raise except Fault as e: self.log.debug("error in xml-rpc server") raise except: self.log.exception("Connection to the server failed/other error") raise def logged_in(self): return self._token is not None def login(self, username="", password=""): try: login = TimeoutFunction(self._login) return login(username, password) except TimeoutFunctionException: self.log.error("login timed out") except: self.log.exception("login: other issue") raise def _login(self, username="", password=""): """Login to the Server using username/password, empty parameters means an anonymously login Returns True if login sucessful, and False if not. """ self.log.debug("----------------") self.log.debug("Logging in (username: %s)..." % username) def run_query(): return self._xmlrpc_server.LogIn( username, password, self.language, self.user_agent) info = self._safe_exec(run_query, None) if info is None: self._token = None return False self.log.debug("Login ended in %s with status: %s" % (info['seconds'], info['status'])) if info['status'] == "200 OK": self.log.debug("Session ID: %s" % info['token']) self.log.debug("----------------") self._token = info['token'] return True else: # force token reset self.log.debug("----------------") self._token = None return False def logout(self): try: logout = TimeoutFunction(self._logout) result = logout() self._token = None return result except TimeoutFunctionException: self.log.error("logout timed out") def _logout(self): """Logout from current session(token) This functions doesn't return any boolean value, since it can 'fail' for anonymous logins """ self.log.debug("Logging out from session ID: %s" % self._token) try: info = self._xmlrpc_server.LogOut(self._token) self.log.debug("Logout ended in %s with status: %s" % (info['seconds'], info['status'])) except ProtocolError as e: self.log.debug("error in HTTP/HTTPS transport layer") raise except Fault as e: self.log.debug("error in xml-rpc server") raise except: self.log.exception("Connection to the server failed/other error") raise finally: # force token reset self._token = None STATUS_CODE_RE = re.compile('(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise ProviderConnectionError(None, 'No message') log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format(status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(None, 'Server returned status="{status}". Expected "200 OK".'.format( status=data['status'])) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(code, message) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)') @classmethod def name(cls): return "opensubtitles" def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): try: result = query() return result except (ProtocolError, xml.parsers.expat.ExpatError): self._signal_connection_failed() log.debug("Query failed", exc_info=sys.exc_info()) return default @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str def imdb_query(self, query): if not self.connected(): return None def run_query(): return self._xmlrpc_server.SearchMoviesOnIMDB(self._token, query) result = self._safe_exec(run_query, None) if result is None: return None provider_identities = [] for imdb_data in result['data']: if not imdb_data: continue imdb_identity = ImdbIdentity(imdb_id=imdb_data['id'], imdb_rating=None) video_identity = VideoIdentity(name=imdb_data['title'], year=None) provider_identities.append(ProviderIdentities( video_identity=video_identity, imdb_identity=imdb_identity, provider=self)) return provider_identities SEARCH_LIMIT = 500 def search_text(self, text, languages=None): lang_str = self._languages_to_str(languages) query = { 'sublanguageid': lang_str, 'query': str(text), } queries = [query] def run_query(): return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT}) self._safe_exec(run_query, None) def search_videos(self, videos, callback, languages=None): if not self.connected(): return None lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate(window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown(remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime(rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitles_SubtitleFile( filename=remote_filename, file_size=remote_file_size , md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, age=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception('Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format(queries=queries)) log.error('Offending result is: {remote_sub}'.format(remote_sub=rsub_raw)) callback.finish() return remote_subtitles def _video_info_to_identification(self, video_info): name = video_info['MovieName'] year = int(video_info['MovieYear']) imdb_id = video_info['MovieImdbID'] video_identity = VideoIdentity(name=name, year=year) imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) episode_identity = None movie_kind = video_info['MovieKind'] if movie_kind == 'episode': season = int(video_info['SeriesSeason']) episode = int(video_info['SeriesEpisode']) episode_identity = EpisodeIdentity(season=season, episode=episode) elif movie_kind == 'movie': pass else: log.warning('Unknown MoviesKind="{}"'.format(video_info['MovieKind'])) return ProviderIdentities(video_identity=video_identity, episode_identity=episode_identity, imdb_identity=imdb_identity, provider=self) def identify_videos(self, videos): if not self.connected(): return for part_videos in window_iterator(videos, 200): hashes = [video.get_osdb_hash() for video in part_videos] hash_video = {hash: video for hash, video in zip(hashes, part_videos)} def run_query(): return self._xmlrpc_server.CheckMovieHash2(self._token, hashes) result = self._safe_exec(run_query, None) self.check_result(result) for video_hash, video_info in result['data'].items(): identification = self._video_info_to_identification(video_info[0]) video = hash_video[video_hash] video.add_identity(identification) def download_subtitles(self, os_rsubs): if not self.connected(): return None window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate(window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc_server.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({item['idsubtitlefile']: item['data'] for item in result['data']}) subtitles = [unzip_bytes(base64.b64decode(map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs] return subtitles def can_upload_subtitles(self, local_movie): if not self.connected(): return False query = {} for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()): # sub_bytes = open(subtitle.get_filepath(), mode='rb').read() # sub_tx_data = b64encode(zlib.compress(sub_bytes)) cd = "cd{i}".format(i=i+1) cd_data = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()), 'moviefps': video.get_fps(), 'movieframes': str(video.get_framecount()), 'moviefilename': video.get_filename(), } query[cd] = cd_data def run_query(): return self._xmlrpc_server.TryUploadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) movie_already_in_db = int(result['alreadyindb']) != 0 if movie_already_in_db: return False return True def upload_subtitles(self, local_movie): query = { 'baseinfo': { 'idmovieimdb': local_movie.get_imdb_id(), 'moviereleasename': local_movie.get_release_name(), 'movieaka': local_movie.get_movie_name(), 'sublanguageid': local_movie.get_language().xxx(), 'subauthorcomment': local_movie.get_comments(), }, } if local_movie.is_hearing_impaired() is not None: query['hearingimpaired'] = local_movie.is_hearing_impaired() if local_movie.is_high_definition() is not None: query['highdefinition'] = local_movie.is_high_definition() if local_movie.is_automatic_translation() is not None: query['automatictranslation'] = local_movie.is_automatic_translation() if local_movie.get_subtitle_author() is not None: query['subtranslator'] = local_movie.get_subtitle_author() if local_movie.is_foreign_only() is not None: query['foreignpartsonly'] = local_movie.is_foreign_only() for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()): sub_bytes = subtitle.get_filepath().open(mode='rb').read() sub_tx_data = b64encode(zlib.compress(sub_bytes)) cd = "cd{i}".format(i=i+1) cd_data = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None, 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), 'subcontent': sub_tx_data, } query[cd] = cd_data def run_query(): return self._xmlrpc_server.UploadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result)