class SubtitlesConncector: RETURNED_OK_STATUS = '200 OK' def __init__(self, user_name='', password='', language='', agent='OS Test User Agent'): self.user_name = user_name self.password = password self.language = language self.agent = agent self._api_adres = 'http://api.opensubtitles.org/xml-rpc' self.proxy_server = ServerProxy(self._api_adres) self.token = self._get_token() def _is_status_ok(self, status): if status == SubtitlesConncector.RETURNED_OK_STATUS: return True return False def _get_token(self): data = self.proxy_server.LogIn(self.user_name, self.password, self.language, self.agent) status = StatusParsser(data=data).parse() if self._is_status_ok(status): return data['token'] else: raise StatusError(status) def _save_zip(self, data_to_zip, path, file_name): b64img = base64.b64decode(data_to_zip.encode('ascii')) b64imgfile = open(str(path + "/" + file_name + ".zip"), 'w') b64imgfile.write(str(b64img)) def get_subtitles(self, subtitle, subtitles_dir=None): if not subtitles_dir: subtitles_dir = os.path.expanduser("~") + "/subtitles" if not os.path.exists(subtitles_dir): os.makedirs(subtitles_dir) data = self.proxy_server.DownloadSubtitles(self.token, [subtitle.idsubtitlefile]) try: data = data['data'][0]['data'] except Exception: raise GetSubtitlesError("Error: cant get data") self._save_zip(data, subtitles_dir, subtitle.subfilename) def search_subtitles(self, subtitles): parsed_data_to_send = ParseSubtitlesToSend(subtitles).parse() data = self.proxy_server.SearchSubtitles(self.token, [parsed_data_to_send]) data = ParseIncomingData(data).parse() collect = SubtitlesFindedCollection() for d in data: collect.append( SubtitlesFindedProperties(sublangid=d['SubLanguageID'], subfilename=d['SubFileName'], idsubtitlefile=d['IDSubtitleFile'])) return collect
def query_open_subtitles(movie_filename, language): uri = 'http://api.opensubtitles.org/xml-rpc' server = ServerProxy(uri, verbose=0, allow_none=True, use_datetime=True) login_info = server.LogIn('', '', 'en', 'ss v' + __version__) token = login_info['token'] try: guessit_query = obtain_guessit_query(movie_filename, language) search_queries = [ guessit_query, obtain_movie_hash_query(movie_filename, language), ] response = server.SearchSubtitles(token, search_queries) try: search_results = response['data'] except KeyError: # noqa raise KeyError('"data" key not found in response: %r' % response) if search_results: search_results = filter_bad_results(search_results, guessit_query) return search_results finally: server.LogOut(token)
def get_subtitles(db, l_post_id, l_imdbid): try: server = ServerProxy('http://api.opensubtitles.org/xml-rpc') token = server.LogIn('dh_moviebarcodes', 'dh_ws2015', 'eng', 'Moviebarcode Analyzer')['token'] imdb_id = int(l_imdbid[2:]) search_request = [] search_request.append({'imdbid': imdb_id, 'sublanguageid': 'eng'}) resp = server.SearchSubtitles(token, search_request) subtitle_id = [] try: sub = get_best_subtitle(resp['data']) subtitle_id.append(sub['IDSubtitleFile']) subtitle_data = server.DownloadSubtitles(token, subtitle_id) except IndexError: print("No subtitle available") return "" if subtitle_data['status'] == '200 OK': compressed_data = subtitle_data['data'][0]['data'] decoded_subtitle = base64.b64decode(compressed_data) # subtitle in byte format decoded_subtitle = gzip.GzipFile(fileobj=io.BytesIO(decoded_subtitle)).read() clean_subtitle = remove_invalid_characters(decoded_subtitle) allLemmasAndMostFrequentLemmas = lemmatize(clean_subtitle) print(allLemmasAndMostFrequentLemmas[0]) print(allLemmasAndMostFrequentLemmas[1]) update_value_in_db(db, l_post_id, "subtitlesLemmatisation", allLemmasAndMostFrequentLemmas[0]) update_value_in_db(db, l_post_id, "subtitlesMostFrequentWords", allLemmasAndMostFrequentLemmas[1]) except ValueError: print("No subtitle available")
class OpenSubtitlesClient: def __init__(self, username, password, language): self.client = ServerProxy('http://api.opensubtitles.org/xml-rpc', allow_none=True) self.token = self.client.LogIn(username, password, language, "TemporaryUserAgent")['token'] def get_list_subtitle_response(self, query, language): response = self.client.SearchSubtitles(self.token, [{ 'query': query, 'sublanguageid': language }], [{ 'limit': 3 }]) return response['data'] def download_encrypted_subtitles(self, subtitle_file_id): response = self.client.DownloadSubtitles(self.token, [subtitle_file_id]) decompresses_subtitle_file = zlib.decompress( base64.b64decode(response['data'][0]['data']), 16 + zlib.MAX_WBITS) return decompresses_subtitle_file def get_encryption_method(self, encrypted_subtitles): return chardet.detect(encrypted_subtitles)['encoding'] def decode(self, encryption_method, data): return data.decode(encryption_method)
class Auth: def __init__(self): self.config = Config.get_instance() self.rpc = ServerProxy( self.config.get("server_url"), allow_none=True ) # allow_none is mandatory to make some methods work self.Token = None self.user_agent = self.config.get("user_agent") def get_rpc(self): return self.rpc def get_user_agent(self): return self.user_agent def login(self): User = self.config.get("username") Pass = self.config.get("password") self.logindata = self.rpc.LogIn(User, Pass, "eng", self.user_agent) if "200 OK" not in self.logindata["status"]: return False self.Token = self.logindata["token"] return self.Token def get_token(self): # function to login the user if self.Token is None: return self.login() return self.Token def logout(self): # function to logout self.rpc.LogOut(self.Token)
class OpenSubtitlesApiWrapper(): def __init__(self): try: self.auth_token = "-1" ### state to check if initialized properly self.server_connector = ServerProxy(SUBTITLE_SERVER, allow_none=True) auth_response = self.server_connector.LogIn( USER_NAME, PASSWORD, "en", USER_AGENT_OPEN_SUTBTITLE) if auth_response['status'] != HTTP_CODE_200: raise Exception("Auth failed!!!") self.auth_token = auth_response["token"] except Exception as e: LOGGER.exception(str(e)) def pull_subtitles(self, media_source): movie_vs_id = {} movie_ids = [] ### to preserve ordering with relevancy media_file_name = os.path.basename(media_source) media_dir = media_source.replace(media_file_name, "") media_name = os.path.splitext(media_file_name)[0] response = self.server_connector.SearchMoviesOnIMDB( self.auth_token, media_name) if len(response['data']) > 0: for movie in response['data']: movie_vs_id[movie['id']] = movie['title'] movie_ids.append(movie['id']) #os.system("autosub " + media_source + " -o " + PROJECT_HOME + "/tmp/tmp.srt") print(movie_vs_id[movie_ids[0]]) if input( "Is this your media file for which you are searching subs y/n? " ) == "y": media_id = movie_ids[0] else: count = 0 for movie_id in movie_ids: print(str(count + 1) + ") " + movie_vs_id[movie_id]) count += 1 choice = int( input("Please choose between the options [1-" + str(count) + "] ")) media_id = movie_ids[choice - 1] print("Pulling subtitles from opensubtitle for " + movie_vs_id[media_id]) subtitles = self.server_connector.SearchSubtitles( self.auth_token, [{ "sublanguageid": "eng", "imdbid": media_id }])['data'] print(subtitles) counter = 1 for subtitle in subtitles: download_link = subtitle['SubDownloadLink'] print("Downloading sub(s) from " + download_link) downloaded_file = os.path.basename(download_link) os.system("cd " + media_dir + "; wget " + download_link + "; gunzip -c " + downloaded_file + " > " + media_name + "." + str(counter) + ".srt; rm -rf '*.gz'") counter += 1
class Downloader: """ Connects to the OpenSubtitles server and allows for file downloading. """ def __init__(self, user_agent, server='http://api.opensubtitles.org/xml-rpc', language='eng'): self.server = server self.language = language trans = Transport() trans.user_agent = user_agent self._rpc = ServerProxy(self.server, allow_none=True, transport=trans) login_response = self._rpc.LogIn('', '', language, user_agent) assert_status(login_response) self._token = login_response.get('token') def download(self, *ids): """ Downloads the subtitles with the given ids. :param ids: The subtitles to download :return: Result instances :raises NotOKException """ bundles = sublists_of(ids, 20) # 20 files at once is an API restriction for bundle in bundles: download_response = self._rpc.DownloadSubtitles( self._token, bundle) assert_status(download_response) download_data = download_response.get('data') for item in download_data: subtitle_id = item['idsubtitlefile'] subtitle_data = item['data'] decompressed = decompress(subtitle_data) yield Result(subtitle_id, decompressed) def close(self): """ Closes the connection. """ self._rpc.LogOut(self._token) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close()
class OpenSubtitles: def __init__(self): # noinspection SpellCheckingInspection opensubtitles_ua = b'VkxzdWIgMC4xMC4y' self.ua = base64.b64decode(opensubtitles_ua).decode() self.opensubtitles_api_url = 'http://api.opensubtitles.org/xml-rpc' self.opensubtitles_lang = 'en' self.token = None self.xmlrpc_transport = Transport() self.xmlrpc_transport.user_agent = self.ua self.xmlrpc = ServerProxy(self.opensubtitles_api_url, allow_none=True, transport=self.xmlrpc_transport) def retrieve_subtitles(self, ids: List[str]) -> Dict[str, str]: response = self.xmlrpc.DownloadSubtitles(self.token, ids) status = response.get('status', '').split()[0] if status == '200': subtitles_content = {} encoded_data = response.get('data') for item in encoded_data: subfile_id = item['idsubtitlefile'] try: decoded_data = decompress(item['data'], 'utf_8_sig') or decompress( item['data'], 'utf-8') except UnicodeDecodeError: decoded_data = decompress(item['data'], 'latin1') if not decoded_data: logger.error('An error occurred while decoding subtitle' 'file ID {}.'.format(subfile_id)) else: subtitles_content[subfile_id] = decoded_data return subtitles_content else: raise RuntimeWarning( f'Unable to get subtitles, returned status is {status}') def search_subtitles(self, param: List[dict]) -> List[Dict[str, str]]: response = self.xmlrpc.SearchSubtitles(self.token, param) status = response.get('status', '').split()[0] if status == '200': return response.get('data') else: raise RuntimeWarning( f'Unable to search subtitles, returned status is {status}') def login(self, username, password): response = self.xmlrpc.LogIn(username, password, self.opensubtitles_lang, self.ua) status = response.get('status', '').split()[0] if status == '200': self.token = response.get('token') else: raise RuntimeWarning( f'Unable to authenticate, returned status is {status}')
def log_in_opensubtitles(self, proxy: ServerProxy) -> str: """ Logs in the user to OpenSubtitles. This function should be called always when starting talking with server. It returns token, which must be used in later communication. If user has no account, blank username and password should be OK. As language - use ISO639 2 letter code. :param proxy: ServerProxy.LogIn(username, password, language, useragent) username: (string) Can be blank since anonymous users are allowed password: (string) Can be blank since anonymous users are allowed language: (string) Either HTTP ACCEPT-LANGUAGE header or ISO639 2 useragent: (string) Use your registered useragent, also provide version number - we need tracking version numbers of your program. If your UA is not registered, you will get error 414 Unknown User Agent :return: token or error message Link to request useragent: http://trac.opensubtitles.org/projects/opensubtitles/wiki/DevReadFirst """ try: self.prompt_label.setText( "Logging in to OpenSubtitles, please wait ...") username = "******" password = "******" user_agent = "SubcrawlProjectUserAgent" login = proxy.LogIn(username, password, self.preference.language_iso3, user_agent) except Fault: self.prompt_label.setText( "There was a fault while logging in to OpenSubtitles. Please try again." ) return "error" except ProtocolError: self.prompt_label.setText( "There was an error with the server. Please try again later.") return "error" except ConnectionResetError or ConnectionError or ConnectionAbortedError or ConnectionRefusedError: self.prompt_label.setText("Please check your internet connection.") return "error" except expat.ExpatError: # https://stackoverflow.com/questions/3664084/xml-parser-syntax-error self.prompt_label.setText( "The received payload is probably incorrect") return "error" except gaierror: self.prompt_label.setText( "Please check your internet connection and try again") return "error" except Exception as e: self.prompt_label.setText( "Be sure to send us this error: {}".format(str(e))) return "error" else: if login["status"] == "200 OK": return login["token"] else: return "error"
def login(): transport = Transport() transport.user_agent = constants.USER_AGENT_OPENSUBS xmlrpc = ServerProxy(constants.OPENSUBTITLES_URL, allow_none=True, transport=transport) try: data = xmlrpc.LogIn(constants.USERNAME, constants.PASSWORD, constants.LANGUAGE, constants.USER_AGENT_OPENSUBS) except: logging.warning( "Error occured while establishing connection to opensubtitles...") return None, None if '200' == data.get('status').split()[0]: logging.info("Got token from opensubtitles") return data.get('token'), xmlrpc else: logging.warning( "Error occured while getting opensubtitles token. Returned status as " + data.get('status').split()[0]) return None
def query_open_subtitles(movie_filename, language): uri = 'http://api.opensubtitles.org/xml-rpc' server = ServerProxy(uri, verbose=0, allow_none=True, use_datetime=True) login_info = server.LogIn('', '', 'en', 'OS Test User Agent') token = login_info['token'] try: guessit_query = obtain_guessit_query(movie_filename, language) search_queries = [ guessit_query, obtain_movie_hash_query(movie_filename, language), ] response = server.SearchSubtitles(token, search_queries) search_results = response['data'] if search_results: search_results = filter_bad_results(search_results, guessit_query) return search_results finally: server.LogOut(token)
time.sleep(2) if opt_gui == 'cli' and opt_selection_mode != 'auto': # Synchronous call process_videoDispatched = subprocess.call(command) else: # Asynchronous call process_videoDispatched = subprocess.Popen(command) # ==== Search and download subtitles =========================================== try: # ==== Connection to OpenSubtitlesDownload try: session = osd_server.LogIn( osd_username, hashlib.md5(osd_password[0:32].encode('utf-8')).hexdigest(), osd_language, 'opensubtitles-download 4.2') except Exception: # Retry once after a delay (could just be a momentary overloaded server?) time.sleep(3) try: session = osd_server.LogIn(osd_username, osd_password, osd_language, 'opensubtitles-download 4.2') except Exception: superPrint("error", "Connection error!", "Unable to reach OpenSubtitles.org servers!\n\nPlease check:\n" + \ "- Your Internet connection status\n" + \ "- www.opensubtitles.org availability\n" + \ "The subtitles search and download service is powered by <a href=\"https://opensubtitles.org\">opensubtitles.org</a>.\n" + \ "Be sure to donate if you appreciate the service provided!") sys.exit(2)
class OpenSubtitles(SubtitleProvider): URL = 'http://api.opensubtitles.org/xml-rpc' def __init__(self): SubtitleProvider.__init__(self) self._xmlrpc = None self._token = None self._settings = OpenSubtitlesSettings() def get_settings(self): return self._settings def set_settings(self, settings): self._settings = settings def connect(self): log.debug('connect()') if self.connected(): return self._xmlrpc = ServerProxy(self.URL, allow_none=False) def disconnect(self): log.debug('disconnect()') if self.logged_in(): self.logout() if self.connected(): self._xmlrpc = None def connected(self): return self._xmlrpc is not None def login(self): log.debug('login()') if self.logged_in(): return if not self.connected(): raise ProviderNotConnectedError() def login_query(): # FIXME: 'en' language ok??? or '' as in the original return self._xmlrpc.LogIn(str(self._settings.username), str(self._settings.password), 'en', str(self._settings.get_user_agent())) result = self._safe_exec(login_query, None) self.check_result(result) self._token = result['token'] def logout(self): log.debug('logout()') if self.logged_in(): def logout_query(): return self._xmlrpc.LogOut(self._token) result = self._safe_exec(logout_query, None) self.check_result(result) self._token = None def logged_in(self): return self._token is not None SEARCH_LIMIT = 500 def search_videos(self, videos, callback, languages=None): log.debug('search_videos(#videos={})'.format(len(videos))) if not self.logged_in(): raise ProviderNotConnectedError() lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate( window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc.SearchSubtitles( self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown( remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime( rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitlesSubtitleFile( filename=remote_filename, file_size=remote_file_size, md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, age=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception( 'Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format( queries=queries)) log.error('Offending result is: {remote_sub}'.format( remote_sub=rsub_raw)) callback.finish() return remote_subtitles def query_text(self, query): return OpenSubtitlesTextQuery(query=query) def download_subtitles(self, os_rsubs): log.debug('download_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate( window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({ item['idsubtitlefile']: item['data'] for item in result['data'] }) subtitles = [ unzip_bytes(base64.b64decode( map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs ] return subtitles def ping(self): log.debug('ping()') if not self.logged_in(): raise ProviderNotConnectedError() def run_query(): return self._xmlrpc.NoOperation(self._token) result = self._safe_exec(run_query, None) self.check_result(result) @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str @classmethod def get_name(cls): return 'opensubtitles' @classmethod def get_short_name(cls): return 'os' def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): try: result = query() return result except (ProtocolError, CannotSendRequest, SocketError): self._signal_connection_failed() log.warning('Query failed', exc_info=sys.exc_info()) return default STATUS_CODE_RE = re.compile('(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise ProviderConnectionError(_('No message')) log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format( status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError( _('Server returned status="{status}". Expected "200 OK".'). format(status=data['status']), data['status']) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(message, code) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)')
class SubtitleDownload: '''Traverses a directory and all subdirectories and downloads subtitles. Relies on an undocumented OpenSubtitles feature where subtitles seems to be sorted by filehash and popularity. ''' api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None moviefiles = [] movie_exts = (".avi", ".mkv", ".mp4") subb_exts = (".srt", ".sub", ".mpl") def __init__(self, movie_path, lang = "en"): print("OpenSubtitles Subtitle Downloader".center(78)) print("===================================".center(78)) self.server = ServerProxy(self.api_url, verbose=False) self.lang_id = lang # Traverse the directory tree and select all movie files for root, _, files in os.walk(movie_path): for file in files: if self.is_movie(file): file_path = os.path.join(root, file) if not self.subtitles_already_present(file_path): print("Found: " + file) filehash = self.hashFile(file_path) filesize = os.path.getsize(file_path) self.moviefiles.append({'dir': root, 'file': file, 'hash': filehash, 'size': filesize, 'subtitleid': None}) try: print("Login...") self.login() print("Searching for subtitles...") self.search_subtitles() print("Logout...") self.logout() except Error as e: print("XML-RPC error:", e) except UserWarning as uw: print(uw) def login(self): '''Log in to OpenSubtitles''' resp = self.server.LogIn('', '', 'en', 'OS Test User Agent') self.check_status(resp) self.login_token = resp['token'] def logout(self): '''Log out from OpenSubtitles''' resp = self.server.LogOut(self.login_token) self.check_status(resp) def search_subtitles(self): '''Search OpenSubtitles for matching subtitles''' search = [] for movie in self.moviefiles: search.append({'sublanguageid': self.lang_id, 'moviehash': movie['hash'], 'moviebytesize': str(movie['size'])}) # fixes weird problem where subs aren't found when only searching # for one movie. if len(search) == 1: search.append(search[0]) resp = self.server.SearchSubtitles(self.login_token, search) self.check_status(resp) if resp['data'] == False: print("No subtitles found") return subtitles = [] for result in resp['data']: if int(result['SubBad']) != 1: subtitles.append({'subid': result['IDSubtitleFile'], 'hash': result['MovieHash']}) downloadable_subs = subtitles[:] hash = None for s in subtitles: if hash == s['hash']: downloadable_subs.remove(s) hash = s['hash'] for ds in downloadable_subs: sub = self.download_subtitles(ds) for movie in self.moviefiles: if movie['hash'] == ds['hash']: print("Saving subtitle for: " + movie['file']) filename = os.path.join(movie['dir'], os.path.splitext(movie['file'])[0] + ".srt") file = open(filename, "wb") file.write(sub) file.close() def download_subtitles(self, subtitle): resp = self.server.DownloadSubtitles(self.login_token, [subtitle['subid']]) self.check_status(resp) decoded = base64.standard_b64decode(resp['data'][0]['data'].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed def is_movie(self, file): return os.path.splitext(file.lower())[1] in self.movie_exts def subtitles_already_present(self, file): file_base = os.path.splitext(file.lower())[0] for ext in self.subb_exts: if os.path.exists(file_base + ext): return True return False def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise UserWarning("Response error from " + self.api_url + ". Response status was: " + resp['status']) def hashFile(self, name): '''Calculates the hash value of a movie. Copied from OpenSubtitles own examples: http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes ''' try: longlongformat = 'q' # long long bytesize = struct.calcsize(longlongformat) f = open(name, "rb") filesize = os.path.getsize(name) hash = filesize if filesize < 65536 * 2: return "SizeError" for x in range(65536//bytesize): buffer = f.read(bytesize) (l_value,)= struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number f.seek(max(0,filesize-65536),0) for x in range(65536//bytesize): buffer = f.read(bytesize) (l_value,)= struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF f.close() returnedhash = "%016x" % hash return returnedhash except(IOError): return "IOError"
if gui == 'cli' and opt_selection_mode == 'manual': # Synchronous call process_videoDispatched = subprocess.call(command_splitted) else: # Asynchronous call process_videoDispatched = subprocess.Popen(command_splitted) # Do not spawn too many instances at the same time time.sleep(0.33) # ==== Search and download subtitles try: try: # Connection to opensubtitles.org server session = server.LogIn('', '', 'en', 'opensubtitles-download 3.2') except Exception: # Retry once, it could be a momentary overloaded server? time.sleep(3) try: # Connection to opensubtitles.org server session = server.LogIn('', '', 'en', 'opensubtitles-download 3.2') except Exception: # Failed connection attempts? superPrint( "error", "Connection error!", "Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your 200 downloads per 24h limit\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!" ) sys.exit(1) # Connection refused?
# Asynchronous dispatch process_videoDispatched = subprocess.Popen(command_splitted) # Do not spawn too many instances at the same time time.sleep(0.33) # The first video file will be processed by this instance videoPathList.clear() videoPathList.append(videoPath) # ==== Search and download subtitles =========================================== try: # ==== Connection try: session = osd_server.LogIn(osd_username, osd_password, "en", 'opensubtitles-download 5.0') except Exception: # Retry once, it could be a momentary overloaded server? time.sleep(3) try: session = osd_server.LogIn(osd_username, osd_password, "en", 'opensubtitles-download 5.0') except Exception: superPrint( "error", "Connection error!", "Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!" ) sys.exit(2) # Connection refused? if session['status'] != '200 OK':
class SDService(object): """ Contains the class that represents the OSDB RPC Server. Encapsules all the XMLRPC methods. Consult the OSDB API methods at http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC If it fails to connect directly to the XMLRPC server, it will try to do so through a default proxy. Default proxy uses a form to set which URL to open. We will try to change this in later stage. """ def __init__(self, server=None, proxy=None): self.log = logging.getLogger("subdownloader.SDService.SDService") self.log.debug( "Creating Server with server = %s and proxy = %r" % (server, proxy)) self.timeout = 30 self.user_agent = USER_AGENT self.language = '' if server: self.server = server else: self.server = DEFAULT_OSDB_SERVER self.proxy = proxy self.logged_as = None self._xmlrpc_server = None self._token = None def connected(self): return self._token is not None def connect(self): server = self.server proxy = self.proxy self.log.debug("connect()... to server %s with proxy %s" % (server, proxy)) connect_res = False try: self.log.debug( "Connecting with parameters (%r, %r)" % (server, proxy)) connect = TimeoutFunction(self._connect) connect_res = connect(server, proxy) except TimeoutFunctionException as e: self.log.error("Connection timed out. Maybe you need a proxy.") raise except: self.log.exception("connect: Unexpected error") raise finally: self.log.debug("connection connected %s" % connect_res) return connect_res def _connect(self, server, proxy): try: if proxy: self.log.debug("Trying proxied connection... ({})".format(proxy)) self.proxied_transport = ProxiedTransport() self.proxied_transport.set_proxy(proxy) self._xmlrpc_server = ServerProxy( server, transport=self.proxied_transport, allow_none=True) # self.ServerInfo() self.log.debug("...connected") return True elif test_connection(TEST_URL): self.log.debug("Trying direct connection...") self._xmlrpc_server = ServerProxy( server, allow_none=True) # self.ServerInfo() self.log.debug("...connected") return True else: self.log.debug("...failed") self.log.error("Unable to connect. Try setting a proxy.") return False except ProtocolError as e: self._connection_failed() self.log.debug("error in HTTP/HTTPS transport layer") raise except Fault as e: self.log.debug("error in xml-rpc server") raise except: self.log.exception("Connection to the server failed/other error") raise def logged_in(self): return self._token is not None def login(self, username="", password=""): try: login = TimeoutFunction(self._login) return login(username, password) except TimeoutFunctionException: self.log.error("login timed out") except: self.log.exception("login: other issue") raise def _login(self, username="", password=""): """Login to the Server using username/password, empty parameters means an anonymously login Returns True if login sucessful, and False if not. """ self.log.debug("----------------") self.log.debug("Logging in (username: %s)..." % username) def run_query(): return self._xmlrpc_server.LogIn( username, password, self.language, self.user_agent) info = self._safe_exec(run_query, None) if info is None: self._token = None return False self.log.debug("Login ended in %s with status: %s" % (info['seconds'], info['status'])) if info['status'] == "200 OK": self.log.debug("Session ID: %s" % info['token']) self.log.debug("----------------") self._token = info['token'] return True else: # force token reset self.log.debug("----------------") self._token = None return False def logout(self): try: logout = TimeoutFunction(self._logout) result = logout() self._token = None return result except TimeoutFunctionException: self.log.error("logout timed out") def _logout(self): """Logout from current session(token) This functions doesn't return any boolean value, since it can 'fail' for anonymous logins """ self.log.debug("Logging out from session ID: %s" % self._token) try: info = self._xmlrpc_server.LogOut(self._token) self.log.debug("Logout ended in %s with status: %s" % (info['seconds'], info['status'])) except ProtocolError as e: self.log.debug("error in HTTP/HTTPS transport layer") raise except Fault as e: self.log.debug("error in xml-rpc server") raise except: self.log.exception("Connection to the server failed/other error") raise finally: # force token reset self._token = None STATUS_CODE_RE = re.compile('(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise ProviderConnectionError(None, 'No message') log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format(status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(None, 'Server returned status="{status}". Expected "200 OK".'.format( status=data['status'])) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise ProviderConnectionError(code, message) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)') @classmethod def name(cls): return "opensubtitles" def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): try: result = query() return result except (ProtocolError, xml.parsers.expat.ExpatError): self._signal_connection_failed() log.debug("Query failed", exc_info=sys.exc_info()) return default @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str def imdb_query(self, query): if not self.connected(): return None def run_query(): return self._xmlrpc_server.SearchMoviesOnIMDB(self._token, query) result = self._safe_exec(run_query, None) if result is None: return None provider_identities = [] for imdb_data in result['data']: if not imdb_data: continue imdb_identity = ImdbIdentity(imdb_id=imdb_data['id'], imdb_rating=None) video_identity = VideoIdentity(name=imdb_data['title'], year=None) provider_identities.append(ProviderIdentities( video_identity=video_identity, imdb_identity=imdb_identity, provider=self)) return provider_identities SEARCH_LIMIT = 500 def search_text(self, text, languages=None): lang_str = self._languages_to_str(languages) query = { 'sublanguageid': lang_str, 'query': str(text), } queries = [query] def run_query(): return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT}) self._safe_exec(run_query, None) def search_videos(self, videos, callback, languages=None): if not self.connected(): return None lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate(window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown(remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime(rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitles_SubtitleFile( filename=remote_filename, file_size=remote_file_size , md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, age=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception('Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format(queries=queries)) log.error('Offending result is: {remote_sub}'.format(remote_sub=rsub_raw)) callback.finish() return remote_subtitles def _video_info_to_identification(self, video_info): name = video_info['MovieName'] year = int(video_info['MovieYear']) imdb_id = video_info['MovieImdbID'] video_identity = VideoIdentity(name=name, year=year) imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None) episode_identity = None movie_kind = video_info['MovieKind'] if movie_kind == 'episode': season = int(video_info['SeriesSeason']) episode = int(video_info['SeriesEpisode']) episode_identity = EpisodeIdentity(season=season, episode=episode) elif movie_kind == 'movie': pass else: log.warning('Unknown MoviesKind="{}"'.format(video_info['MovieKind'])) return ProviderIdentities(video_identity=video_identity, episode_identity=episode_identity, imdb_identity=imdb_identity, provider=self) def identify_videos(self, videos): if not self.connected(): return for part_videos in window_iterator(videos, 200): hashes = [video.get_osdb_hash() for video in part_videos] hash_video = {hash: video for hash, video in zip(hashes, part_videos)} def run_query(): return self._xmlrpc_server.CheckMovieHash2(self._token, hashes) result = self._safe_exec(run_query, None) self.check_result(result) for video_hash, video_info in result['data'].items(): identification = self._video_info_to_identification(video_info[0]) video = hash_video[video_hash] video.add_identity(identification) def download_subtitles(self, os_rsubs): if not self.connected(): return None window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate(window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc_server.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({item['idsubtitlefile']: item['data'] for item in result['data']}) subtitles = [unzip_bytes(base64.b64decode(map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs] return subtitles def can_upload_subtitles(self, local_movie): if not self.connected(): return False query = {} for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()): # sub_bytes = open(subtitle.get_filepath(), mode='rb').read() # sub_tx_data = b64encode(zlib.compress(sub_bytes)) cd = "cd{i}".format(i=i+1) cd_data = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()), 'moviefps': video.get_fps(), 'movieframes': str(video.get_framecount()), 'moviefilename': video.get_filename(), } query[cd] = cd_data def run_query(): return self._xmlrpc_server.TryUploadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) movie_already_in_db = int(result['alreadyindb']) != 0 if movie_already_in_db: return False return True def upload_subtitles(self, local_movie): query = { 'baseinfo': { 'idmovieimdb': local_movie.get_imdb_id(), 'moviereleasename': local_movie.get_release_name(), 'movieaka': local_movie.get_movie_name(), 'sublanguageid': local_movie.get_language().xxx(), 'subauthorcomment': local_movie.get_comments(), }, } if local_movie.is_hearing_impaired() is not None: query['hearingimpaired'] = local_movie.is_hearing_impaired() if local_movie.is_high_definition() is not None: query['highdefinition'] = local_movie.is_high_definition() if local_movie.is_automatic_translation() is not None: query['automatictranslation'] = local_movie.is_automatic_translation() if local_movie.get_subtitle_author() is not None: query['subtranslator'] = local_movie.get_subtitle_author() if local_movie.is_foreign_only() is not None: query['foreignpartsonly'] = local_movie.is_foreign_only() for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()): sub_bytes = subtitle.get_filepath().open(mode='rb').read() sub_tx_data = b64encode(zlib.compress(sub_bytes)) cd = "cd{i}".format(i=i+1) cd_data = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None, 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), 'subcontent': sub_tx_data, } query[cd] = cd_data def run_query(): return self._xmlrpc_server.UploadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result)
class OpenSub(object): def __init__(self, path): self.TAGS = [ 'bluray', 'cam', 'dvb', 'dvd', 'hd-dvd', 'hdtv', 'ppv', 'telecine', 'telesync', 'tv', 'vhs', 'vod', 'web-dl', 'webrip', 'workprint' ] self.OPENSUBTITLES_SERVER = 'https://api.opensubtitles.org:443/xml-rpc' self.USER_AGENT = 'TemporaryUserAgent' # needs to be changed from time to time to the recent temporary UA self.xmlrpc = ServerProxy(self.OPENSUBTITLES_SERVER, allow_none=True) self.language = 'eng' self.token = None self.mv_name = None self.path = path try: with open("opensubtitles_userpass.txt", 'r') as f: f = f.read() user = f.split('\n')[0].split('=')[1] password = f.split('\n')[1].split('=')[1] self.user = user self.passw = password except FileNotFoundError: print( "Please fill OpenSubtitles credentials in opensubtitles_userpass.txt" ) if self.login(self.user, self.passw): print("[OpenSubtitles] Login Successful") else: print("[OpenSubtitles] Login Failed") def _get_from_data_or_none(self, key): '''Return the key recieved from data if the status is 200, otherwise return None. ''' status = self.data.get('status').split()[0] return self.data.get(key) if '200' == status else None def login(self, username, password): '''Returns token is login is ok, otherwise None. ''' self.data = self.xmlrpc.LogIn(username, password, self.language, self.USER_AGENT) token = self._get_from_data_or_none('token') if token: self.token = token return token def logout(self): '''Returns True if logout is ok, otherwise None. ''' data = self.xmlrpc.LogOut(self.token) return '200' in data.get('status') def search_subtitles(self, params): '''Returns a list with the subtitles info. ''' self.data = self.xmlrpc.SearchSubtitles(self.token, params) return self._get_from_data_or_none('data') def download_subtitles(self, payload): """ :return: """ print("Attempting to download subtitles...") search_result = self.search_subtitles([payload]) if search_result: dllink = self.analyse_result(search_result, payload) gzfile = request.urlopen(dllink) try: with gzip.open(gzfile, 'rb') as f: with open(self.path + "\\" + self.mv_name + '.srt', 'wb') as sub_file: sub_file.write(f.read()) print("[OpenSubtitles] Subtitles downloaded for: " + self.mv_name) # Update stats with open("metadata.json", "r") as jsonFile: data = json.load(jsonFile) data['num_of_movies_in_db'] = int( data['num_of_movies_in_db']) + 1 data['latest_movie_added']['name'] = self.mv_name data['latest_movie_added']['time'] = str( datetime.datetime.now()) with open("metadata.json", "w") as jsonFile: json.dump(data, jsonFile, indent=4) # End Update return os.path.join(self.path, self.mv_name + ".srt") except PermissionError: print( "[OpenSubtitles] Permission Error: when creating subtitles for {}" .format(self.mv_name)) def analyse_result(self, result, payload): """ :param result: Search result to find appropriate subtitles :return: Download Link of best match for subtitles """ score = 0 dllink = None for record in result: if record.get("MovieName").lower() == payload['query'].lower( ) and record.get("SubLanguageID") == 'eng': self.mv_name = record.get("MovieName") dllink = record.get('SubDownloadLink') break if record.get('Score', 0) > score and record.get("SubLanguageID") == 'eng': self.mv_name = record.get("MovieName") score = record.get('Score', 0) dllink = record.get('SubDownloadLink') return dllink def create_payload(self, mv_name): """ :param lang: Subtitle Language :param mv_name: Movie name to search :return: Payload containing data about file """ payload = {} payload['query'] = mv_name payload['SubLanguageID'] = self.language payload['SubFormat'] = 'srt' return payload def search_and_download(self, mv_name): movie_db = [ os.path.splitext(name)[0] for name in os.listdir(self.path) ] for local in movie_db: if mv_name.lower() == local.lower(): return os.path.join(self.path, mv_name + ".srt") try: return self.download_subtitles(self.create_payload(mv_name)) except: print("[OpenSubtitles] Could not download subtitles for " + mv_name) def download_top_1000(self): with open(os.path.join(os.getcwd(), "top1000.txt"), "r") as f: f = f.read() i = 1 for mv_name in f.split("\n"): try: self.search_and_download(mv_name) except: e = sys.exc_info()[0] print("ERROR: " + str(e)) print("Error: subtitles not downloaded for: " + mv_name) i = i + 1 if i % 39 == 0: time.sleep(11)
class Agent(): """ opensubtitle 的用户代理""" # __username="" # __password="" # __languae="" # __osd_server="" # __session="" opt_selection_mode = 'default' opt_search_mode = "filename" opt_language_separator = '_' def __init__(self, username, password, language="en"): self.__username = username self.__password = password self.__languae = language self.__osd_server = ServerProxy('http://api.opensubtitles.org/xml-rpc') def LogIn(self): try: # ==== Connection try: self.__session = self.__osd_server.LogIn( self.__username, self.__password, self.__languae, 'opensubtitles-download 4.0') except Exception: # Retry once, it could be a momentary overloaded server? time.sleep(3) try: self.__session = self.__osd_server.LogIn( self.__username, self.__password, self.__languae, 'opensubtitles-download 4.0') except Exception: print("error1") #superPrint("error", "Connection error!", "Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!") sys.exit(2) # Connection refused? if self.__session['status'] != '200 OK': print("error2") #superPrint("error", "Connection error!", "Opensubtitles.org servers refused the connection: " + session['status'] + ".\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!") sys.exit(2) print("login success!") except: print("login error3") sys.exit(2) def LogOut(self): if self.__session and self.__session['token']: self.__osd_server.LogOut(self.__session['token']) print("logout success") def __search(self, title): """ 查询服务器,寻找潜在的字幕""" searchList = [] searchList.append({'sublanguageid': 'eng', 'query': title}) try: subtitlesList = self.__osd_server.SearchSubtitles( self.__session['token'], searchList) return subtitlesList # time.sleep(1) except Exception: # Retry once, we are already connected, the server maybe momentary overloaded time.sleep(3) try: subtitlesList = self.__osd_server.SearchSubtitles( self.__session['token'], searchList) return subtitlesList except Exception: print("Search error!") #superPrint("error", "Search error!", "Unable to reach opensubtitles.org servers!\n<b>Search error</b>") return -1 def __selectionAuto(self, subtitlesList, year, imdb_ID): """ 从众多字幕中寻找合适的字幕 """ """Automatic subtitles selection using filename match""" # 需要用到的属性 # IDMovieImdb the most import # MovieKind # MovieYear # SubDownloadLink # SubFileName # SubFormat # 顺序选择即可。 # if len(subtitlesList['data'])==1: # return 0 data = subtitlesList['data'] for i in range(len(data)): this = data[i] if this["IDMovieImdb"] != imdb_ID: continue # if this["SubFormat"]=="srt": # return i if this["MovieKind"] != "movie": continue if this["SubFormat"] == "srt" and abs(year - int(this["MovieYear"])) < 2: return i return -1 def __down(self, subtitlesSelected, subtitlesList, subPath): # 此处变量与下载有直接关系 subLangId = self.opt_language_separator + subtitlesList['data'][ subtitlesSelected]['ISO639'] subLangName = subtitlesList['data'][subtitlesSelected]['LanguageName'] subURL = subtitlesList['data'][subtitlesSelected]['SubDownloadLink'] # subPath 为存储路径 # subPath="1.txt" #subPath = videoPath.rsplit('.', 1)[0] + '.' + subtitlesList['data'][subIndex]['SubFormat'] if sys.version_info >= (3, 0): tmpFile1, headers = urllib.request.urlretrieve(subURL) tmpFile2 = gzip.GzipFile(tmpFile1) byteswritten = open(subPath, 'wb').write(tmpFile2.read()) if byteswritten > 0: process_subtitlesDownload = 0 else: process_subtitlesDownload = 1 else: # python 2 tmpFile1, headers = urllib.urlretrieve(subURL) tmpFile2 = gzip.GzipFile(tmpFile1) open(subPath, 'wb').write(tmpFile2.read()) process_subtitlesDownload = 0 # If an error occurs, say so if process_subtitlesDownload != 0: print("Subtitling error!") #superPrint("error", "Subtitling error!", "An error occurred while downloading or writing <b>" + subtitlesList['data'][subIndex]['LanguageName'] + "</b> subtitles for <b>" + videoTitle + "</b>.") self.__osd_server.LogOut(self.__session['token']) sys.exit(2) def __result(self, subtitlesList, year, imdb_ID, subPath): """ 主体函数 """ # Parse the results of the XML-RPC query if ('data' in subtitlesList) and (len(subtitlesList['data']) > 0): # print("find it") #找到数据 # Mark search as successful subtitlesSelected = '' # If there is only one subtitles (matched by file hash), auto-select it (except in CLI mode) if (len(subtitlesList['data']) == 1) and (subtitlesList['data'][0]['MatchedBy'] == 'moviehash'): if opt_selection_mode != 'manual': # subtitlesSelected = subtitlesList['data'][0]['SubFileName'] subtitlesSelected = 0 # Get video title videoTitle = subtitlesList['data'][0]['MovieName'] # If there is more than one subtitles and opt_selection_mode != 'auto', # then let the user decide which one will be downloaded if subtitlesSelected == '': subtitlesSelected = self.__selectionAuto( subtitlesList, year, imdb_ID) if subtitlesSelected == -1: return -1 # If a subtitles has been selected at this point, download it! if subtitlesSelected == -1: # 记录未下载的电影名与ID #print("find error!") return -1 else: # print("ready to down") self.__down(subtitlesSelected, subtitlesList, subPath) return 0 else: # print("not find it!") return -1 def work(self, title, year, imdb_ID, path): """ 查找电影字幕并下载到指定路径中""" subtitlesList = self.__search(title) time.sleep(2) if subtitlesList == -1: return -1 return self.__result(subtitlesList, year, imdb_ID, path) pass def work_by_imda(self, year, imdb_ID): list = self.__search(imdb_ID) if list == -1: return -1 return self.__selectionAuto(list, year, imdb_ID) pass pass
class OpenSubtitles(QtCore.QThread): api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None def __init__(self, parent=None): QtCore.QThread.__init__(self, parent) self.stopping = False def process(self, files_list, lang='English'): self.moviefiles = files_list self.imdbid_to_hash = {} self.lang = LANGUAGES[lang][1] if not self.stopping: self.start() def __del__(self): if self.login_token: self.logout() def stopTask(self): self.stopping = True def run(self): utils.communicator.updategui.emit('Querying OpenSubtitles.org...', 'info' ) try: if not self.login_token: self.login() self.search_subtitles() except utils.NoInternetConnectionFound: utils.communicator.updategui.emit('No active Internet connection found. Kindly check and try again.', 'error') def login(self): '''Log in to OpenSubtitles.org''' self.server = ServerProxy(self.api_url, verbose=False) utils.communicator.updategui.emit('Logging in to OpenSubtitles.org...', 'info' ) try: resp = self.server.LogIn('', '', 'en', 'PySubD v2.0') self.check_status(resp) self.login_token = resp['token'] except Exception as e: if e.args[0] == 11004: utils.communicator.updategui.emit("No Internet Connection Found", 'error') else: utils.communicator.updategui.emit(str(e), 'error') def logout(self): '''Log out from OpenSubtitles''' utils.communicator.updategui.emit('Logging out...', 'info') resp = self.server.LogOut(self.login_token) self.check_status(resp) def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise utils.IncorrectResponseRecieved('Response error from %s. Response status: %s' % (self.api_url, resp['status'])) def _query_opensubs(self, search): results = [] while search[:500]: if not self.stopping: try: tempresp = self.server.SearchSubtitles(self.login_token, search[:500]) except Exception as e: if e.args[0] == 11004: utils.communicator.updategui.emit("No Internet Connection Found", 'error') else: utils.communicator.updategui.emit(str(e), 'error') return results if tempresp['data']: results.extend(tempresp['data']) self.check_status(tempresp) search = search[500:] else: return [] logger.debug('Results: %s' % results) return results def clean_results(self, results, imdb=False): subtitles = {} user_ranks = { 'administrator': 1, 'platinum member': 2, 'vip member': 3, 'gold member': 4, 'trusted': 5, 'silver member': 6, 'bronze member': 7, 'sub leecher': 8, '': 9, } for result in results: if result['SubBad'] != '1': movie_hash = result.get('MovieHash') if not movie_hash: movie_hash = self.imdbid_to_hash[int(result['IDMovieImdb'])] subid = result['IDSubtitleFile'] downcount = int(result['SubDownloadsCnt']) rating = float(result['SubRating']) if rating and rating < 8: # Ignore poorly rated subtitles, while not # penalizing the ones that haven't yet been rated continue user_rank = user_ranks[result['UserRank']] if imdb: cleaned_release_name = utils.clean_name(result['MovieReleaseName']) file_name = self.moviefiles[movie_hash]['file_name'] cleaned_file_name = utils.clean_name(file_name) overlap = len(set.intersection(set(cleaned_release_name), set(cleaned_file_name))) else: overlap = 0 subtitles.setdefault(movie_hash, []).append({ 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank, 'overlap' : overlap }) return subtitles def search_subtitles(self): search = [] for video_file_details in self.moviefiles.itervalues(): video_file_details['sublanguageid'] = self.lang search.append(video_file_details) results = self._query_opensubs(search) subtitles = self.clean_results(results) for (hash, found_matches) in subtitles.iteritems(): subtitles[hash] = utils.multikeysort(found_matches, ['overlap', 'user_rank', '-rating', '-downcount'])[0] for (hash, filedetails) in self.moviefiles.iteritems(): if not self.stopping: if subtitles.get(hash): utils.communicator.updategui.emit('Saving subtitles for %s'%filedetails['file_name'], 'success') subtitle = \ self.download_subtitles([subtitles[hash]['subid' ]]) utils.communicator.downloaded_sub.emit() utils.save_subs(subtitle, filedetails['save_subs_to'], subtitles[hash]) else: utils.communicator.no_sub_found.emit(filedetails['file_name']) else: return def download_subtitles(self, subparam): resp = self.server.DownloadSubtitles(self.login_token, subparam) self.check_status(resp) decoded = base64.standard_b64decode(resp['data'][0]['data' ].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed
class OpenSubtitles(SubtitleProvider): URL = 'http://api.opensubtitles.org/xml-rpc' def __init__(self, settings=None): SubtitleProvider.__init__(self) self._xmlrpc = None self._token = None self._last_time = None if settings is None: settings = OpenSubtitlesSettings() self._settings = settings def get_settings(self): return self._settings def set_settings(self, settings): if self.connected(): raise RuntimeError( 'Cannot set settings while connected') # FIXME: change error self._settings = settings def connect(self): log.debug('connect()') if self.connected(): return self._xmlrpc = ServerProxy(self.URL, allow_none=False) self._last_time = time.time() def disconnect(self): log.debug('disconnect()') if self.logged_in(): self.logout() if self.connected(): self._xmlrpc = None def connected(self): return self._xmlrpc is not None def login(self): log.debug('login()') if self.logged_in(): return if not self.connected(): self.connect() def login_query(): # FIXME: 'en' language ok??? or '' as in the original return self._xmlrpc.LogIn(str(self._settings.username), str(self._settings.password), 'en', str(self._settings.get_user_agent())) result = self._safe_exec(login_query, None) self.check_result(result) self._token = result['token'] def logout(self): log.debug('logout()') if self.logged_in(): def logout_query(): return self._xmlrpc.LogOut(self._token) # Do no check result of this call. Assume connection closed. self._safe_exec(logout_query, None) self._token = None def logged_in(self): return self._token is not None def reestablish(self): log.debug('reestablish()') connected = self.connected() logged_in = self.logged_in() self.disconnect() if connected: self.connect() if logged_in: self.login() _TIMEOUT_MS = 60000 def _ensure_connection(self): now = time.time() if now - time.time() > self._TIMEOUT_MS: self.reestablish() self._last_time = now SEARCH_LIMIT = 500 def search_videos(self, videos, callback, languages=None): log.debug('search_videos(#videos={})'.format(len(videos))) if not self.logged_in(): raise ProviderNotConnectedError() lang_str = self._languages_to_str(languages) window_size = 5 callback.set_range(0, (len(videos) + (window_size - 1)) // window_size) remote_subtitles = [] for window_i, video_window in enumerate( window_iterator(videos, window_size)): callback.update(window_i) if callback.canceled(): break queries = [] hash_video = {} for video in video_window: query = { 'sublanguageid': lang_str, 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), } if video.get_osdb_hash() is None: log.debug('osdb hash of "{}" is empty -> skip'.format( video.get_filepath())) self._signal_connection_failed( ) # FIXME: other name + general signaling continue queries.append(query) hash_video[video.get_osdb_hash()] = video def run_query(): return self._xmlrpc.SearchSubtitles( self._token, queries, {'limit': self.SEARCH_LIMIT}) result = self._safe_exec(run_query, None) self.check_result(result) if result is None: continue for rsub_raw in result['data']: try: remote_filename = rsub_raw['SubFileName'] remote_file_size = int(rsub_raw['SubSize']) remote_id = rsub_raw['IDSubtitleFile'] remote_md5_hash = rsub_raw['SubHash'] remote_download_link = rsub_raw['SubDownloadLink'] remote_link = rsub_raw['SubtitlesLink'] remote_uploader = rsub_raw['UserNickName'].strip() remote_language_raw = rsub_raw['SubLanguageID'] try: remote_language = Language.from_unknown( remote_language_raw, xx=True, xxx=True) except NotALanguageException: remote_language = UnknownLanguage(remote_language_raw) remote_rating = float(rsub_raw['SubRating']) remote_date = datetime.datetime.strptime( rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S') remote_subtitle = OpenSubtitlesSubtitleFile( filename=remote_filename, file_size=remote_file_size, md5_hash=remote_md5_hash, id_online=remote_id, download_link=remote_download_link, link=remote_link, uploader=remote_uploader, language=remote_language, rating=remote_rating, date=remote_date, ) movie_hash = '{:>016}'.format(rsub_raw['MovieHash']) video = hash_video[movie_hash] imdb_id = rsub_raw['IDMovieImdb'] try: imdb_rating = float(rsub_raw['MovieImdbRating']) except (ValueError, KeyError): imdb_rating = None imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=imdb_rating) video_name = rsub_raw['MovieName'] try: video_year = int(rsub_raw['MovieYear']) except (ValueError, KeyError): video_year = None video_identity = VideoIdentity(name=video_name, year=video_year) try: series_season = int(rsub_raw['SeriesSeason']) except (KeyError, ValueError): series_season = None try: series_episode = int(rsub_raw['SeriesEpisode']) except (KeyError, ValueError): series_episode = None series_identity = SeriesIdentity(season=series_season, episode=series_episode) identity = ProviderIdentities( video_identity=video_identity, imdb_identity=imdb_identity, episode_identity=series_identity, provider=self) video.add_subtitle(remote_subtitle) video.add_identity(identity) remote_subtitles.append(remote_subtitle) except (KeyError, ValueError): log.exception( 'Error parsing result of SearchSubtitles(...)') log.error('Offending query is: {queries}'.format( queries=queries)) log.error('Offending result is: {remote_sub}'.format( remote_sub=rsub_raw)) callback.finish() return remote_subtitles def query_text(self, query): return OpenSubtitlesTextQuery(query=query) def download_subtitles(self, os_rsubs): log.debug('download_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() window_size = 20 map_id_data = {} for window_i, os_rsub_window in enumerate( window_iterator(os_rsubs, window_size)): query = [subtitle.get_id_online() for subtitle in os_rsub_window] def run_query(): return self._xmlrpc.DownloadSubtitles(self._token, query) result = self._safe_exec(run_query, None) self.check_result(result) map_id_data.update({ item['idsubtitlefile']: item['data'] for item in result['data'] }) subtitles = [ unzip_bytes(base64.b64decode( map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs ] return subtitles def upload_subtitles(self, local_movie): log.debug('upload_subtitles()') if not self.logged_in(): raise ProviderNotConnectedError() video_subtitles = list(local_movie.iter_video_subtitles()) if not video_subtitles: return UploadResult( type=UploadResult.Type.MISSINGDATA, reason=_('Need at least one subtitle to upload')) query_try = dict() for sub_i, (video, subtitle) in enumerate(video_subtitles): if not video: return UploadResult( type=UploadResult.Type.MISSINGDATA, reason=_('Each subtitle needs an accompanying video')) query_try['cd{}'.format(sub_i + 1)] = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), } def run_query_try_upload(): return self._xmlrpc.TryUploadSubtitles(self._token, query_try) try_result = self._safe_exec(run_query_try_upload, None) self.check_result(try_result) if int(try_result['alreadyindb']): return UploadResult(type=UploadResult.Type.DUPLICATE, reason=_('Subtitle is already in database')) if local_movie.get_imdb_id() is None: return UploadResult(type=UploadResult.Type.MISSINGDATA, reason=_('Need IMDb id')) upload_base_info = { 'idmovieimdb': local_movie.get_imdb_id(), } if local_movie.get_comments() is not None: upload_base_info['subauthorcomment'] = local_movie.get_comments() if not local_movie.get_language().is_generic(): upload_base_info['sublanguageid'] = local_movie.get_language().xxx( ) if local_movie.get_release_name() is not None: upload_base_info[ 'moviereleasename'] = local_movie.get_release_name() if local_movie.get_movie_name() is not None: upload_base_info['movieaka'] = local_movie.get_movie_name() if local_movie.is_hearing_impaired() is not None: upload_base_info[ 'hearingimpaired'] = local_movie.is_hearing_impaired() if local_movie.is_high_definition() is not None: upload_base_info[ 'highdefinition'] = local_movie.is_high_definition() if local_movie.is_automatic_translation() is not None: upload_base_info[ 'automatictranslation'] = local_movie.is_automatic_translation( ) if local_movie.get_author() is not None: upload_base_info['subtranslator'] = local_movie.get_author() if local_movie.is_foreign_only() is not None: upload_base_info['foreignpartsonly'] = local_movie.is_foreign_only( ) query_upload = { 'baseinfo': upload_base_info, } for sub_i, (video, subtitle) in enumerate(video_subtitles): sub_bytes = subtitle.get_filepath().open(mode='rb').read() sub_tx_data = base64.b64encode(zlib.compress(sub_bytes)).decode() query_upload['cd{}'.format(sub_i + 1)] = { 'subhash': subtitle.get_md5_hash(), 'subfilename': subtitle.get_filename(), 'moviehash': video.get_osdb_hash(), 'moviebytesize': str(video.get_size()), 'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None, 'moviefps': str(video.get_fps()) if video.get_fps() else None, 'movieframes': str(video.get_framecount()) if video.get_framecount() else None, 'moviefilename': video.get_filename(), 'subcontent': sub_tx_data, } def run_query_upload(): return self._xmlrpc.UploadSubtitles(self._token, query_upload) result = self._safe_exec(run_query_upload, None) self.check_result(result) rsubs = [] for sub_data in result['data']: filename = sub_data['SubFileName'] file_size = sub_data['SubSize'] md5_hash = sub_data['SubHash'] id_online = sub_data['IDSubMOvieFile'] download_link = sub_data['SubDownloadLink'] link = None uploader = sub_data['UserNickName'] language = Language.from_xxx(sub_data['SubLanguageID']) rating = float(sub_data['SubRating']) add_date = datetime.datetime.strptime(sub_data['SubAddDate'], '%Y-%m-%d %H:%M:%S') sub = OpenSubtitlesSubtitleFile(filename=filename, file_size=file_size, md5_hash=md5_hash, id_online=id_online, download_link=download_link, link=link, uploader=uploader, language=language, rating=rating, date=add_date) rsubs.append(sub) return UploadResult(type=UploadResult.Type.OK, rsubs=rsubs) def imdb_search_title(self, title): self._ensure_connection() def run_query(): return self._xmlrpc.SearchMoviesOnIMDB(self._token, title.strip()) result = self._safe_exec(run_query, default=None) self.check_result(result) imdbs = [] re_title = re.compile(r'(?P<title>.*) \((?P<year>[0-9]+)\)') for imdb_data in result['data']: imdb_id = imdb_data['id'] if all(c in string.digits for c in imdb_id): imdb_id = 'tt{}'.format(imdb_id) m = re_title.match(imdb_data['title']) if m: imdb_title = m['title'] imdb_year = int(m['year']) else: imdb_title = imdb_data['title'] imdb_year = None imdbs.append( ImdbMovieMatch(imdb_id=imdb_id, title=imdb_title, year=imdb_year)) return imdbs def ping(self): log.debug('ping()') if not self.logged_in(): raise ProviderNotConnectedError() def run_query(): return self._xmlrpc.NoOperation(self._token) result = self._safe_exec(run_query, None) self.check_result(result) def provider_info(self): if self.connected(): def run_query(): return self._xmlrpc.ServerInfo() result = self._safe_exec(run_query, None) data = [ (_('XML-RPC version'), result['xmlrpc_version']), (_('XML-RPC url'), result['xmlrpc_url']), (_('Application'), result['application']), (_('Contact'), result['contact']), (_('Website url'), result['website_url']), (_('Users online'), result['users_online_total']), (_('Programs online'), result['users_online_program']), (_('Users logged in'), result['users_loggedin']), (_('Max users online'), result['users_max_alltime']), (_('Users registered'), result['users_registered']), (_('Subtitles downloaded'), result['subs_downloads']), (_('Subtitles available'), result['subs_subtitle_files']), (_('Number movies'), result['movies_total']), (_('Number languages'), result['total_subtitles_languages']), (_('Client IP'), result['download_limits']['client_ip']), (_('24h global download limit'), result['download_limits']['global_24h_download_limit']), (_('24h client download limit'), result['download_limits']['client_24h_download_limit']), (_('24h client download count'), result['download_limits']['client_24h_download_count']), (_('Client download quota'), result['download_limits']['client_download_quota']), ] else: data = [] return data @staticmethod def _languages_to_str(languages): if languages: lang_str = ','.join([language.xxx() for language in languages]) else: lang_str = 'all' return lang_str @classmethod def get_name(cls): return 'opensubtitles' @classmethod def get_short_name(cls): return 'os' @classmethod def get_icon(cls): return ':/images/sites/opensubtitles.png' def _signal_connection_failed(self): # FIXME: set flag/... to signal users that the connection has failed pass def _safe_exec(self, query, default): self._ensure_connection() try: result = query() return result except (ProtocolError, CannotSendRequest, SocketError, ExpatError) as e: self._signal_connection_failed() log.debug('Query failed: {} {}'.format(type(e), e.args)) return default STATUS_CODE_RE = re.compile(r'(\d+) (.+)') @classmethod def check_result(cls, data): log.debug('check_result(<data>)') if data is None: log.warning('data is None ==> FAIL') raise OpenSubtitlesProviderConnectionError(None, _('No message')) log.debug('checking presence of "status" in result ...') if 'status' not in data: log.debug('... no "status" in result ==> assuming SUCCESS') return log.debug('... FOUND') status = data['status'] log.debug('result["status"]="{status}"'.format(status=status)) log.debug('applying regex to status ...') try: code, message = cls.STATUS_CODE_RE.match(status).groups() log.debug('... regex SUCCEEDED') code = int(code) except (AttributeError, ValueError): log.debug('... regex FAILED') log.warning('Got unexpected status="{status}" from server.'.format( status=status)) log.debug('Checking for presence of "200" ...') if '200' not in data['status']: log.debug('... FAIL. Raising ProviderConnectionError.') raise OpenSubtitlesProviderConnectionError( None, _('Server returned status="{status}". Expected "200 OK".'). format(status=data['status']), data['status']) log.debug('... SUCCESS') code, message = 200, 'OK' log.debug('Checking code={code} ...'.format(code=code)) if code != 200: log.debug('... FAIL. Raising ProviderConnectionError.') raise OpenSubtitlesProviderConnectionError(code, message) log.debug('... SUCCESS.') log.debug('check_result() finished (data is ok)')
class SubtitleDownload(QtCore.QThread): '''Traverses a directory and all subdirectories and downloads subtitles. Relies on an undocumented OpenSubtitles feature where subtitles seems to be sorted by filehash and popularity. ''' api_url = 'http://api.opensubtitles.org/xml-rpc' login_token = None server = None def __init__(self, parent=None): QtCore.QThread.__init__(self, parent) def init(self, movie_paths): self._movie_paths = movie_paths self.start() def __del__(self): if self.login_token: self.logout() def stopTask(self): self.stopping = True def run(self): self.stopping = False self.moviefiles = {} self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Searching for video files...") """Check if the movie_path is a file or directory. Calculate the file stats if its a file, else traverse the directory tree and calculate filestats for all movie files""" for path in self._movie_paths: if os.path.isfile(path): self.calcfilestats(path, os.path.dirname(path)) else: for root, _, files in os.walk(path): for file in files: if not self.stopping: self.calcfilestats(file, root) else: return try: if self.moviefiles: if not self.login_token: try: self.login() except (noInternetConnection): self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, No active Internet connection found. Re-Check and try again." ) my_logger.debug( "Sorry, No active Internet connection found. Re-Check and try again." ) self.emit( QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"), self._movie_paths) return self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Searching for subtitles...") self.search_subtitles() self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Done...") my_logger.debug("Done...") else: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, no video files were found!") self.emit(QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"), self._movie_paths) except Error as e: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), ("XML-RPC error:", e)) except UserWarning as uw: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), uw) def calcfilestats(self, file, parentdir): ''' Calculates and adds the hash of a movie file to the moviefiles dictionary. Also emits a signal to update the LCD counter showing the found files.''' video_extns = [ '.avi', '.divx', '.mkv', '.mp4', '.ogg', '.rm', '.rmvb', '.vob', '.x264', '.xvid' ] if os.path.splitext(file)[1].lower() in video_extns: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Found: " + file) my_logger.debug("Found: " + file) self.emit(QtCore.SIGNAL("updateFound()")) filehash = self.hashFile(os.path.join(parentdir, file)) filesize = os.path.getsize(os.path.join(parentdir, file)) self.moviefiles[filehash] = { 'dir': parentdir, 'file': file, 'size': filesize } def login(self): '''Log in to OpenSubtitles''' self.server = ServerProxy(self.api_url, verbose=False) self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging in...") my_logger.debug("Logging in...") try: resp = self.server.LogIn('', '', 'en', 'PySubD v1.0') self.check_status(resp) self.login_token = resp['token'] except Error as e: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), e) my_logger.debug(str(e)) def logout(self): '''Log out from OpenSubtitles''' self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging out...") resp = self.server.LogOut(self.login_token) self.check_status(resp) def search_subtitles(self): '''Search OpenSubtitles for matching subtitles using moviehash and filebytesize''' search = [] for hash in self.moviefiles.keys(): search.append({ 'sublanguageid': 'eng', 'moviehash': hash, 'moviebytesize': str(self.moviefiles[hash]['size']) }) my_logger.debug("Length of search string: " + str(len(search))) resp = {} resp['data'] = [] while search[:500]: if not self.stopping: tempresp = self.server.SearchSubtitles(self.login_token, search[:500]) if tempresp['data'] != False: resp['data'].extend(tempresp['data']) self.check_status(tempresp) search = search[500:] else: return #Check if we actually got some matching subtitles to download, else return if not resp['data']: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Sorry, no subtitles were found!") my_logger.debug("Sorry, no subtitles were found!") return #User Ranks on OpenSubtitles.org.These are decided based upon the number of subtitles uploaded by the member. #A better rank is often an indication of a better subtitle quality. user_ranks = { 'admin': 1, 'platinum member': 2, 'vip member': 3, 'gold member': 4, 'trusted': 5, 'silver member': 6, 'bronze member': 7, 'sub leecher': 8, '': 9 } # A dictionary to store the subtitle id's found corresponding to every file hash subtitles = {} for result in resp['data']: #The subtitle must not be rated bad if int(result['SubBad']) != 1: hash = result['MovieHash'] subid = result['IDSubtitleFile'] downcount = result['SubDownloadsCnt'] rating = result['SubRating'] user_rank = user_ranks[result['UserRank']] #First good matching subtitle found if not subtitles.get(hash): subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } self.emit(QtCore.SIGNAL("updateAvailable()")) #Another good quality subtitle found uploaded by a more reputed user elif subtitles[hash]['user_rank'] > user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } #Another good quality subtitle found with a perfect rating, uploaded by a more or equally reputed user elif float(rating) == 10.0 and subtitles[hash][ 'user_rank'] >= user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } #Another good quality subtitle found with the better rating, higher download count and uploaded by a more or equally trusted user elif float(subtitles[hash]['rating']) >= float(rating) \ and int(subtitles[hash]['downcount']) < int(downcount) and subtitles[hash]['user_rank'] >= user_rank: subtitles[hash] = { 'subid': subid, 'downcount': downcount, 'rating': rating, 'user_rank': user_rank } my_logger.debug("Total number of subtitles found: " + str(len(subtitles))) notfound = [] for hash, filedetails in self.moviefiles.iteritems(): if not self.stopping: if subtitles.get(hash): try: subtitle = self.download_subtitles( [subtitles[hash]['subid']]) self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Saving subtitle for: " + filedetails['file']) my_logger.debug("Saving subtitle for: " + filedetails['file'] + " Hash : " + hash + " Rating: " + subtitles[hash]['rating'] + " DwnCnt: " + subtitles[hash]['downcount'] + " UpldrRnk:" + \ str(subtitles[hash]['user_rank'])) self.emit(QtCore.SIGNAL("updateDownloaded()")) filename = os.path.join( filedetails['dir'], os.path.splitext(filedetails['file'])[0] + ".srt") file = open(filename, "wb") file.write(subtitle) file.close() except IOError: self.emit( QtCore.SIGNAL("updategui(PyQt_PyObject)"), "IO Error in saving subs for " + filedetails['file']) else: notfound.append(filedetails['file']) else: return #Report all the files for which no subtitles were found in an alphabetically sorted order. notfound.sort(key=str.lower) for file in notfound: self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "No subtitles found for: " + file) my_logger.debug("No subtitles were found for: " + file) def download_subtitles(self, subparam): resp = self.server.DownloadSubtitles(self.login_token, subparam) self.check_status(resp) decoded = base64.standard_b64decode( resp['data'][0]['data'].encode('ascii')) decompressed = zlib.decompress(decoded, 15 + 32) return decompressed def check_status(self, resp): '''Check the return status of the request. Anything other than "200 OK" raises a UserWarning ''' if resp['status'].upper() != '200 OK': raise UserWarning("Response error from " + self.api_url + ". Response status was: " + resp['status']) def hashFile(self, name): ''' Calculates the hash value of a movie. Copied from OpenSubtitles own examples: http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes ''' try: longlongformat = 'q' # long long bytesize = struct.calcsize(longlongformat) f = open(name, "rb") filesize = os.path.getsize(name) hash = filesize if filesize < 65536 * 2: return "SizeError: Minimum file size must be 120Kb" for x in range(65536 // bytesize): buffer = f.read(bytesize) (l_value, ) = struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number f.seek(max(0, filesize - 65536), 0) for x in range(65536 // bytesize): buffer = f.read(bytesize) (l_value, ) = struct.unpack(longlongformat, buffer) hash += l_value hash = hash & 0xFFFFFFFFFFFFFFFF f.close() returnedhash = "%016x" % hash return returnedhash except (IOError): return "IOError"
class OpenSubtitles(object): def __init__(self): self.xmlrpc = ServerProxy(Settings.OPENSUBTITLES_SERVER, allow_none=True) self.language = Settings.LANGUAGE self.token = None def _get_from_data_or_none(self, key): """ Проверка на аутентификацию Args: key (str): строка с параметром который нужно вернуть Returns: (str) : возвращаемое значение """ status = self.data.get('status').split()[0] return self.data.get(key) if '200' == status else None def login(self): """ Авторизация Returns: (str) : токен """ self.data = self.xmlrpc.LogIn(Settings.USER_NAME, Settings.USER_PASSWORD, self.language, Settings.USER_AGENT) token = self._get_from_data_or_none('token') if token: self.token = token return token def logout(self): """ Логаут Returns: (str) : статус """ data = self.xmlrpc.LogOut(self.token) return '200' in data.get('status') def search_subtitles(self, params): """ Поиск субтитров Args: params(dict): + язык субтитров + id фильма на сайте *imdb.com* Returns: (dic) : + id субтитров + формат субтитров + id фильма """ self.data = self.xmlrpc.SearchSubtitles(self.token, params) return self.data def search_movies_on_imdb(self, params): """ Поиск фильмов Args: params(dict): + название фильма Returns: (dic) : + id фильма + название фильма """ self.data = self.xmlrpc.SearchMoviesOnIMDB(self.token, params) return self.data def download_subtitles(self, params): """ Загрузка субтитров Args: params(int): id субтитров Returns: (dic) : + id субтитров + строка с субтитрами """ self.data = self.xmlrpc.DownloadSubtitles(self.token, params) return self.data
filePath = os.environ['NAUTILUS_SCRIPT_SELECTED_FILE_PATHS'].splitlines()[0] print('File path:', filePath) folders = filePath.split('/') fileName = folders[len(folders)-1] print('File name:', fileName) folderPath = filePath[0:filePath.index(fileName)] print('Folder path:', folderPath) print('Searching the best subtitle for file', fileName, '...') print('Retrieving loginToken...') server = ServerProxy('https://api.opensubtitles.org/xml-rpc') session = server.LogIn('Lodono', 'bocegajen', languageID, 'opensubtitles-download 4.1') if session['status'] != '200 OK': print('Login failed. Exiting...') sys.exit() loginToken = session['token'] print('Success:', loginToken) print('First trying to query using file hash and size...') tryHashAndSize = queryByHashAndSize(filePath, languageID, server, loginToken) if tryHashAndSize['success']:
def on_task_download(self, task, config): # filter all entries that have IMDB ID set try: entries = [e for e in task.accepted if e['imdb_id'] is not None] except KeyError: # No imdb urls on this task, skip it # TODO: should do lookup via imdb_lookup plugin? return try: s = ServerProxy("http://api.opensubtitles.org/xml-rpc") res = s.LogIn("", "", "en", "FlexGet") except Exception: logger.warning('Error connecting to opensubtitles.org') return if res['status'] != '200 OK': raise Exception("Login to opensubtitles.org XML-RPC interface failed") config = self.prepare_config(config, task) token = res['token'] # configuration languages = config['languages'] min_sub_rating = config['min_sub_rating'] match_limit = config[ 'match_limit' ] # no need to change this, but it should be configurable # loop through the entries for entry in entries: imdbid = entry.get('imdb_id') if not imdbid: logger.debug('no match for {}', entry['title']) continue query = [] for language in languages: query.append({'sublanguageid': language, 'imdbid': imdbid}) subtitles = s.SearchSubtitles(token, query) subtitles = subtitles['data'] # nothing found -> continue if not subtitles: continue # filter bad subs subtitles = [x for x in subtitles if x['SubBad'] == '0'] # some quality required (0.0 == not reviewed) subtitles = [ x for x in subtitles if float(x['SubRating']) >= min_sub_rating or float(x['SubRating']) == 0.0 ] filtered_subs = [] # find the best rated subs for each language for language in languages: langsubs = [x for x in subtitles if x['SubLanguageID'] == language] # did we find any subs for this language? if langsubs: def seqmatch(subfile): s = difflib.SequenceMatcher(lambda x: x in " ._", entry['title'], subfile) # print "matching: ", entry['title'], subfile, s.ratio() return s.ratio() > match_limit # filter only those that have matching release names langsubs = [x for x in subtitles if seqmatch(x['MovieReleaseName'])] if langsubs: # find the best one by SubRating langsubs.sort(key=lambda x: float(x['SubRating'])) langsubs.reverse() filtered_subs.append(langsubs[0]) # download for sub in filtered_subs: logger.debug( 'SUBS FOUND: {} {} {}', sub['MovieReleaseName'], sub['SubRating'], sub['SubLanguageID'], ) f = task.requests.get(sub['ZipDownloadLink']) subfilename = re.match( '^attachment; filename="(.*)"$', f.headers['content-disposition'] ).group(1) outfile = os.path.join(config['output'], subfilename) fp = open(outfile, 'w') fp.write(f.raw) fp.close() f.close() s.LogOut(token)
# Do not spawn too many instances at once time.sleep(0.33) if opt_gui == 'cli' and opt_selection_mode != 'auto': # Synchronous call process_videoDispatched = subprocess.call(command) else: # Asynchronous call process_videoDispatched = subprocess.Popen(command) # ==== Search and download subtitles =========================================== try: # ==== Connection to OpenSubtitlesDownload try: session = osd_server.LogIn(osd_username, osd_password, osd_language, 'opensubtitles-download 4.1') except Exception: # Retry once after a delay (could just be a momentary overloaded server?) time.sleep(3) try: session = osd_server.LogIn(osd_username, osd_password, osd_language, 'opensubtitles-download 4.1') except Exception: superPrint("error", "Connection error!", "Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!") sys.exit(2) # Connection refused? if session['status'] != '200 OK': superPrint("error", "Connection error!", "Opensubtitles.org servers refused the connection: " + session['status'] + ".\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!") sys.exit(2) # Count languages marked for this search
filePath = os.environ['NAUTILUS_SCRIPT_SELECTED_FILE_PATHS'].splitlines()[0] print('File path:', filePath) folders = filePath.split('/') fileName = folders[len(folders) - 1] print('File name:', fileName) folderPath = filePath[0:filePath.index(fileName)] print('Folder path:', folderPath) print('Searching the best subtitle for file', fileName, '...') print('Retrieving loginToken...') server = ServerProxy('https://api.opensubtitles.org/xml-rpc') session = server.LogIn('Lodono', 'bocegajen', languageID, 'TemporaryUserAgent') if session['status'] != '200 OK': print('Login failed. Exiting...') sys.exit() loginToken = session['token'] print('Success:', loginToken) print('First trying to query using file hash and size...') tryHashAndSize = queryByHashAndSize(filePath, languageID, server, loginToken) if tryHashAndSize['success']: