class OpenSubtitlesClient:
def __init__(self, username, password, language):
self.client = ServerProxy('http://api.opensubtitles.org/xml-rpc',
allow_none=True)
self.token = self.client.LogIn(username, password, language,
"TemporaryUserAgent")['token']
def get_list_subtitle_response(self, query, language):
response = self.client.SearchSubtitles(self.token,
[{
'query': query,
'sublanguageid': language
}], [{
'limit': 3
}])
return response['data']
def download_encrypted_subtitles(self, subtitle_file_id):
response = self.client.DownloadSubtitles(self.token,
[subtitle_file_id])
decompresses_subtitle_file = zlib.decompress(
base64.b64decode(response['data'][0]['data']), 16 + zlib.MAX_WBITS)
return decompresses_subtitle_file
def get_encryption_method(self, encrypted_subtitles):
return chardet.detect(encrypted_subtitles)['encoding']
def decode(self, encryption_method, data):
return data.decode(encryption_method)
def _perform_query_and_store(self, payload_for_sub_search: dict,
proxy: ServerProxy):
"""
Searches for the desired subtitles through the OpenSubtitles API and writes the download URL information
to a table ("search_subs").
:param payload_for_sub_search: (dictionary) contains the information about the movie for which
the subtitle will download
:param proxy: ServerProxy.LogIn(username, password, language, useragent)
:return download_data: (dictionary) contains crucial information for file downloading
"""
try:
query_result = proxy.SearchSubtitles(self.opensubs_token,
[payload_for_sub_search],
{"limit": 10})
except Fault:
self.prompt_label.setText("A fault has occurred")
except ProtocolError:
self.prompt_label.setText("A ProtocolError has occurred.")
else:
if query_result["status"] == "200 OK":
if query_result["data"]:
payload_for_download = self._create_download_data(
query_result["data"], payload_for_sub_search)
self.interactor.add_subtitle_search_data_to_db(
payload_for_download)
else:
self.prompt_label.setText(
"There is no subtitles in this language for {}".format(
payload_for_sub_search["query"]))
else:
self.prompt_label.setText("Wrong status code: {}".format(
query_result["status"]))
def query_open_subtitles(movie_filename, language):
uri = 'http://api.opensubtitles.org/xml-rpc'
server = ServerProxy(uri, verbose=0, allow_none=True, use_datetime=True)
login_info = server.LogIn('', '', 'en', 'ss v' + __version__)
token = login_info['token']
try:
guessit_query = obtain_guessit_query(movie_filename, language)
search_queries = [
guessit_query,
obtain_movie_hash_query(movie_filename, language),
]
response = server.SearchSubtitles(token, search_queries)
try:
search_results = response['data']
except KeyError: # noqa
raise KeyError('"data" key not found in response: %r' % response)
if search_results:
search_results = filter_bad_results(search_results, guessit_query)
return search_results
finally:
server.LogOut(token)
class SubtitlesConncector:
RETURNED_OK_STATUS = '200 OK'
def __init__(self,
user_name='',
password='',
language='',
agent='OS Test User Agent'):
self.user_name = user_name
self.password = password
self.language = language
self.agent = agent
self._api_adres = 'http://api.opensubtitles.org/xml-rpc'
self.proxy_server = ServerProxy(self._api_adres)
self.token = self._get_token()
def _is_status_ok(self, status):
if status == SubtitlesConncector.RETURNED_OK_STATUS:
return True
return False
def _get_token(self):
data = self.proxy_server.LogIn(self.user_name, self.password,
self.language, self.agent)
status = StatusParsser(data=data).parse()
if self._is_status_ok(status):
return data['token']
else:
raise StatusError(status)
def _save_zip(self, data_to_zip, path, file_name):
b64img = base64.b64decode(data_to_zip.encode('ascii'))
b64imgfile = open(str(path + "/" + file_name + ".zip"), 'w')
b64imgfile.write(str(b64img))
def get_subtitles(self, subtitle, subtitles_dir=None):
if not subtitles_dir:
subtitles_dir = os.path.expanduser("~") + "/subtitles"
if not os.path.exists(subtitles_dir):
os.makedirs(subtitles_dir)
data = self.proxy_server.DownloadSubtitles(self.token,
[subtitle.idsubtitlefile])
try:
data = data['data'][0]['data']
except Exception:
raise GetSubtitlesError("Error: cant get data")
self._save_zip(data, subtitles_dir, subtitle.subfilename)
def search_subtitles(self, subtitles):
parsed_data_to_send = ParseSubtitlesToSend(subtitles).parse()
data = self.proxy_server.SearchSubtitles(self.token,
[parsed_data_to_send])
data = ParseIncomingData(data).parse()
collect = SubtitlesFindedCollection()
for d in data:
collect.append(
SubtitlesFindedProperties(sublangid=d['SubLanguageID'],
subfilename=d['SubFileName'],
idsubtitlefile=d['IDSubtitleFile']))
return collect
def get_subtitles(db, l_post_id, l_imdbid):
try:
server = ServerProxy('http://api.opensubtitles.org/xml-rpc')
token = server.LogIn('dh_moviebarcodes', 'dh_ws2015', 'eng', 'Moviebarcode Analyzer')['token']
imdb_id = int(l_imdbid[2:])
search_request = []
search_request.append({'imdbid': imdb_id, 'sublanguageid': 'eng'})
resp = server.SearchSubtitles(token, search_request)
subtitle_id = []
try:
sub = get_best_subtitle(resp['data'])
subtitle_id.append(sub['IDSubtitleFile'])
subtitle_data = server.DownloadSubtitles(token, subtitle_id)
except IndexError:
print("No subtitle available")
return ""
if subtitle_data['status'] == '200 OK':
compressed_data = subtitle_data['data'][0]['data']
decoded_subtitle = base64.b64decode(compressed_data)
# subtitle in byte format
decoded_subtitle = gzip.GzipFile(fileobj=io.BytesIO(decoded_subtitle)).read()
clean_subtitle = remove_invalid_characters(decoded_subtitle)
allLemmasAndMostFrequentLemmas = lemmatize(clean_subtitle)
print(allLemmasAndMostFrequentLemmas[0])
print(allLemmasAndMostFrequentLemmas[1])
update_value_in_db(db, l_post_id, "subtitlesLemmatisation", allLemmasAndMostFrequentLemmas[0])
update_value_in_db(db, l_post_id, "subtitlesMostFrequentWords", allLemmasAndMostFrequentLemmas[1])
except ValueError:
print("No subtitle available")
class OpenSubtitlesApiWrapper():
def __init__(self):
try:
self.auth_token = "-1" ### state to check if initialized properly
self.server_connector = ServerProxy(SUBTITLE_SERVER,
allow_none=True)
auth_response = self.server_connector.LogIn(
USER_NAME, PASSWORD, "en", USER_AGENT_OPEN_SUTBTITLE)
if auth_response['status'] != HTTP_CODE_200:
raise Exception("Auth failed!!!")
self.auth_token = auth_response["token"]
except Exception as e:
LOGGER.exception(str(e))
def pull_subtitles(self, media_source):
movie_vs_id = {}
movie_ids = [] ### to preserve ordering with relevancy
media_file_name = os.path.basename(media_source)
media_dir = media_source.replace(media_file_name, "")
media_name = os.path.splitext(media_file_name)[0]
response = self.server_connector.SearchMoviesOnIMDB(
self.auth_token, media_name)
if len(response['data']) > 0:
for movie in response['data']:
movie_vs_id[movie['id']] = movie['title']
movie_ids.append(movie['id'])
#os.system("autosub " + media_source + " -o " + PROJECT_HOME + "/tmp/tmp.srt")
print(movie_vs_id[movie_ids[0]])
if input(
"Is this your media file for which you are searching subs y/n? "
) == "y":
media_id = movie_ids[0]
else:
count = 0
for movie_id in movie_ids:
print(str(count + 1) + ") " + movie_vs_id[movie_id])
count += 1
choice = int(
input("Please choose between the options [1-" + str(count) +
"] "))
media_id = movie_ids[choice - 1]
print("Pulling subtitles from opensubtitle for " +
movie_vs_id[media_id])
subtitles = self.server_connector.SearchSubtitles(
self.auth_token, [{
"sublanguageid": "eng",
"imdbid": media_id
}])['data']
print(subtitles)
counter = 1
for subtitle in subtitles:
download_link = subtitle['SubDownloadLink']
print("Downloading sub(s) from " + download_link)
downloaded_file = os.path.basename(download_link)
os.system("cd " + media_dir + "; wget " + download_link +
"; gunzip -c " + downloaded_file + " > " + media_name +
"." + str(counter) + ".srt; rm -rf '*.gz'")
counter += 1
class OpenSubtitles:
def __init__(self):
# noinspection SpellCheckingInspection
opensubtitles_ua = b'VkxzdWIgMC4xMC4y'
self.ua = base64.b64decode(opensubtitles_ua).decode()
self.opensubtitles_api_url = 'http://api.opensubtitles.org/xml-rpc'
self.opensubtitles_lang = 'en'
self.token = None
self.xmlrpc_transport = Transport()
self.xmlrpc_transport.user_agent = self.ua
self.xmlrpc = ServerProxy(self.opensubtitles_api_url,
allow_none=True,
transport=self.xmlrpc_transport)
def retrieve_subtitles(self, ids: List[str]) -> Dict[str, str]:
response = self.xmlrpc.DownloadSubtitles(self.token, ids)
status = response.get('status', '').split()[0]
if status == '200':
subtitles_content = {}
encoded_data = response.get('data')
for item in encoded_data:
subfile_id = item['idsubtitlefile']
try:
decoded_data = decompress(item['data'],
'utf_8_sig') or decompress(
item['data'], 'utf-8')
except UnicodeDecodeError:
decoded_data = decompress(item['data'], 'latin1')
if not decoded_data:
logger.error('An error occurred while decoding subtitle'
'file ID {}.'.format(subfile_id))
else:
subtitles_content[subfile_id] = decoded_data
return subtitles_content
else:
raise RuntimeWarning(
f'Unable to get subtitles, returned status is {status}')
def search_subtitles(self, param: List[dict]) -> List[Dict[str, str]]:
response = self.xmlrpc.SearchSubtitles(self.token, param)
status = response.get('status', '').split()[0]
if status == '200':
return response.get('data')
else:
raise RuntimeWarning(
f'Unable to search subtitles, returned status is {status}')
def login(self, username, password):
response = self.xmlrpc.LogIn(username, password,
self.opensubtitles_lang, self.ua)
status = response.get('status', '').split()[0]
if status == '200':
self.token = response.get('token')
else:
raise RuntimeWarning(
f'Unable to authenticate, returned status is {status}')
def query_open_subtitles(movie_filename, language):
uri = 'http://api.opensubtitles.org/xml-rpc'
server = ServerProxy(uri, verbose=0, allow_none=True, use_datetime=True)
login_info = server.LogIn('', '', 'en', 'OS Test User Agent')
token = login_info['token']
try:
guessit_query = obtain_guessit_query(movie_filename, language)
search_queries = [
guessit_query,
obtain_movie_hash_query(movie_filename, language),
]
response = server.SearchSubtitles(token, search_queries)
search_results = response['data']
if search_results:
search_results = filter_bad_results(search_results, guessit_query)
return search_results
finally:
server.LogOut(token)
class SubtitleDownload:
'''Traverses a directory and all subdirectories and downloads subtitles.
Relies on an undocumented OpenSubtitles feature where subtitles seems to be
sorted by filehash and popularity.
'''
api_url = 'http://api.opensubtitles.org/xml-rpc'
login_token = None
server = None
moviefiles = []
movie_exts = (".avi", ".mkv", ".mp4")
subb_exts = (".srt", ".sub", ".mpl")
def __init__(self, movie_path, lang = "en"):
print("OpenSubtitles Subtitle Downloader".center(78))
print("===================================".center(78))
self.server = ServerProxy(self.api_url, verbose=False)
self.lang_id = lang
# Traverse the directory tree and select all movie files
for root, _, files in os.walk(movie_path):
for file in files:
if self.is_movie(file):
file_path = os.path.join(root, file)
if not self.subtitles_already_present(file_path):
print("Found: " + file)
filehash = self.hashFile(file_path)
filesize = os.path.getsize(file_path)
self.moviefiles.append({'dir': root,
'file': file,
'hash': filehash,
'size': filesize,
'subtitleid': None})
try:
print("Login...")
self.login()
print("Searching for subtitles...")
self.search_subtitles()
print("Logout...")
self.logout()
except Error as e:
print("XML-RPC error:", e)
except UserWarning as uw:
print(uw)
def login(self):
'''Log in to OpenSubtitles'''
resp = self.server.LogIn('', '', 'en', 'OS Test User Agent')
self.check_status(resp)
self.login_token = resp['token']
def logout(self):
'''Log out from OpenSubtitles'''
resp = self.server.LogOut(self.login_token)
self.check_status(resp)
def search_subtitles(self):
'''Search OpenSubtitles for matching subtitles'''
search = []
for movie in self.moviefiles:
search.append({'sublanguageid': self.lang_id,
'moviehash': movie['hash'],
'moviebytesize': str(movie['size'])})
# fixes weird problem where subs aren't found when only searching
# for one movie.
if len(search) == 1:
search.append(search[0])
resp = self.server.SearchSubtitles(self.login_token, search)
self.check_status(resp)
if resp['data'] == False:
print("No subtitles found")
return
subtitles = []
for result in resp['data']:
if int(result['SubBad']) != 1:
subtitles.append({'subid': result['IDSubtitleFile'],
'hash': result['MovieHash']})
downloadable_subs = subtitles[:]
hash = None
for s in subtitles:
if hash == s['hash']:
downloadable_subs.remove(s)
hash = s['hash']
for ds in downloadable_subs:
sub = self.download_subtitles(ds)
for movie in self.moviefiles:
if movie['hash'] == ds['hash']:
print("Saving subtitle for: " + movie['file'])
filename = os.path.join(movie['dir'], os.path.splitext(movie['file'])[0] + ".srt")
file = open(filename, "wb")
file.write(sub)
file.close()
def download_subtitles(self, subtitle):
resp = self.server.DownloadSubtitles(self.login_token, [subtitle['subid']])
self.check_status(resp)
decoded = base64.standard_b64decode(resp['data'][0]['data'].encode('ascii'))
decompressed = zlib.decompress(decoded, 15 + 32)
return decompressed
def is_movie(self, file):
return os.path.splitext(file.lower())[1] in self.movie_exts
def subtitles_already_present(self, file):
file_base = os.path.splitext(file.lower())[0]
for ext in self.subb_exts:
if os.path.exists(file_base + ext):
return True
return False
def check_status(self, resp):
'''Check the return status of the request.
Anything other than "200 OK" raises a UserWarning
'''
if resp['status'].upper() != '200 OK':
raise UserWarning("Response error from " + self.api_url + ". Response status was: " + resp['status'])
def hashFile(self, name):
'''Calculates the hash value of a movie.
Copied from OpenSubtitles own examples:
http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
'''
try:
longlongformat = 'q' # long long
bytesize = struct.calcsize(longlongformat)
f = open(name, "rb")
filesize = os.path.getsize(name)
hash = filesize
if filesize < 65536 * 2:
return "SizeError"
for x in range(65536//bytesize):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
f.seek(max(0,filesize-65536),0)
for x in range(65536//bytesize):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF
f.close()
returnedhash = "%016x" % hash
return returnedhash
except(IOError):
return "IOError"
class OpenSubtitles(SubtitleProvider):
URL = 'http://api.opensubtitles.org/xml-rpc'
def __init__(self, settings=None):
SubtitleProvider.__init__(self)
self._xmlrpc = None
self._token = None
self._last_time = None
if settings is None:
settings = OpenSubtitlesSettings()
self._settings = settings
def get_settings(self):
return self._settings
def set_settings(self, settings):
if self.connected():
raise RuntimeError(
'Cannot set settings while connected') # FIXME: change error
self._settings = settings
def connect(self):
log.debug('connect()')
if self.connected():
return
self._xmlrpc = ServerProxy(self.URL, allow_none=False)
self._last_time = time.time()
def disconnect(self):
log.debug('disconnect()')
if self.logged_in():
self.logout()
if self.connected():
self._xmlrpc = None
def connected(self):
return self._xmlrpc is not None
def login(self):
log.debug('login()')
if self.logged_in():
return
if not self.connected():
self.connect()
def login_query():
# FIXME: 'en' language ok??? or '' as in the original
return self._xmlrpc.LogIn(str(self._settings.username),
str(self._settings.password), 'en',
str(self._settings.get_user_agent()))
result = self._safe_exec(login_query, None)
self.check_result(result)
self._token = result['token']
def logout(self):
log.debug('logout()')
if self.logged_in():
def logout_query():
return self._xmlrpc.LogOut(self._token)
# Do no check result of this call. Assume connection closed.
self._safe_exec(logout_query, None)
self._token = None
def logged_in(self):
return self._token is not None
def reestablish(self):
log.debug('reestablish()')
connected = self.connected()
logged_in = self.logged_in()
self.disconnect()
if connected:
self.connect()
if logged_in:
self.login()
_TIMEOUT_MS = 60000
def _ensure_connection(self):
now = time.time()
if now - time.time() > self._TIMEOUT_MS:
self.reestablish()
self._last_time = now
SEARCH_LIMIT = 500
def search_videos(self, videos, callback, languages=None):
log.debug('search_videos(#videos={})'.format(len(videos)))
if not self.logged_in():
raise ProviderNotConnectedError()
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(
window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
if video.get_osdb_hash() is None:
log.debug('osdb hash of "{}" is empty -> skip'.format(
video.get_filepath()))
self._signal_connection_failed(
) # FIXME: other name + general signaling
continue
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc.SearchSubtitles(
self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(
remote_language_raw, xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(
rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitlesSubtitleFile(
filename=remote_filename,
file_size=remote_file_size,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
date=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
try:
imdb_rating = float(rsub_raw['MovieImdbRating'])
except (ValueError, KeyError):
imdb_rating = None
imdb_identity = ImdbIdentity(imdb_id=imdb_id,
imdb_rating=imdb_rating)
video_name = rsub_raw['MovieName']
try:
video_year = int(rsub_raw['MovieYear'])
except (ValueError, KeyError):
video_year = None
video_identity = VideoIdentity(name=video_name,
year=video_year)
try:
series_season = int(rsub_raw['SeriesSeason'])
except (KeyError, ValueError):
series_season = None
try:
series_episode = int(rsub_raw['SeriesEpisode'])
except (KeyError, ValueError):
series_episode = None
series_identity = SeriesIdentity(season=series_season,
episode=series_episode)
identity = ProviderIdentities(
video_identity=video_identity,
imdb_identity=imdb_identity,
episode_identity=series_identity,
provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception(
'Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(
queries=queries))
log.error('Offending result is: {remote_sub}'.format(
remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def query_text(self, query):
return OpenSubtitlesTextQuery(query=query)
def download_subtitles(self, os_rsubs):
log.debug('download_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(
window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({
item['idsubtitlefile']: item['data']
for item in result['data']
})
subtitles = [
unzip_bytes(base64.b64decode(
map_id_data[os_rsub.get_id_online()])).read()
for os_rsub in os_rsubs
]
return subtitles
def upload_subtitles(self, local_movie):
log.debug('upload_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
video_subtitles = list(local_movie.iter_video_subtitles())
if not video_subtitles:
return UploadResult(
type=UploadResult.Type.MISSINGDATA,
reason=_('Need at least one subtitle to upload'))
query_try = dict()
for sub_i, (video, subtitle) in enumerate(video_subtitles):
if not video:
return UploadResult(
type=UploadResult.Type.MISSINGDATA,
reason=_('Each subtitle needs an accompanying video'))
query_try['cd{}'.format(sub_i + 1)] = {
'subhash':
subtitle.get_md5_hash(),
'subfilename':
subtitle.get_filename(),
'moviehash':
video.get_osdb_hash(),
'moviebytesize':
str(video.get_size()),
'moviefps':
str(video.get_fps()) if video.get_fps() else None,
'movieframes':
str(video.get_framecount())
if video.get_framecount() else None,
'moviefilename':
video.get_filename(),
}
def run_query_try_upload():
return self._xmlrpc.TryUploadSubtitles(self._token, query_try)
try_result = self._safe_exec(run_query_try_upload, None)
self.check_result(try_result)
if int(try_result['alreadyindb']):
return UploadResult(type=UploadResult.Type.DUPLICATE,
reason=_('Subtitle is already in database'))
if local_movie.get_imdb_id() is None:
return UploadResult(type=UploadResult.Type.MISSINGDATA,
reason=_('Need IMDb id'))
upload_base_info = {
'idmovieimdb': local_movie.get_imdb_id(),
}
if local_movie.get_comments() is not None:
upload_base_info['subauthorcomment'] = local_movie.get_comments()
if not local_movie.get_language().is_generic():
upload_base_info['sublanguageid'] = local_movie.get_language().xxx(
)
if local_movie.get_release_name() is not None:
upload_base_info[
'moviereleasename'] = local_movie.get_release_name()
if local_movie.get_movie_name() is not None:
upload_base_info['movieaka'] = local_movie.get_movie_name()
if local_movie.is_hearing_impaired() is not None:
upload_base_info[
'hearingimpaired'] = local_movie.is_hearing_impaired()
if local_movie.is_high_definition() is not None:
upload_base_info[
'highdefinition'] = local_movie.is_high_definition()
if local_movie.is_automatic_translation() is not None:
upload_base_info[
'automatictranslation'] = local_movie.is_automatic_translation(
)
if local_movie.get_author() is not None:
upload_base_info['subtranslator'] = local_movie.get_author()
if local_movie.is_foreign_only() is not None:
upload_base_info['foreignpartsonly'] = local_movie.is_foreign_only(
)
query_upload = {
'baseinfo': upload_base_info,
}
for sub_i, (video, subtitle) in enumerate(video_subtitles):
sub_bytes = subtitle.get_filepath().open(mode='rb').read()
sub_tx_data = base64.b64encode(zlib.compress(sub_bytes)).decode()
query_upload['cd{}'.format(sub_i + 1)] = {
'subhash':
subtitle.get_md5_hash(),
'subfilename':
subtitle.get_filename(),
'moviehash':
video.get_osdb_hash(),
'moviebytesize':
str(video.get_size()),
'movietimems':
str(video.get_time_ms()) if video.get_time_ms() else None,
'moviefps':
str(video.get_fps()) if video.get_fps() else None,
'movieframes':
str(video.get_framecount())
if video.get_framecount() else None,
'moviefilename':
video.get_filename(),
'subcontent':
sub_tx_data,
}
def run_query_upload():
return self._xmlrpc.UploadSubtitles(self._token, query_upload)
result = self._safe_exec(run_query_upload, None)
self.check_result(result)
rsubs = []
for sub_data in result['data']:
filename = sub_data['SubFileName']
file_size = sub_data['SubSize']
md5_hash = sub_data['SubHash']
id_online = sub_data['IDSubMOvieFile']
download_link = sub_data['SubDownloadLink']
link = None
uploader = sub_data['UserNickName']
language = Language.from_xxx(sub_data['SubLanguageID'])
rating = float(sub_data['SubRating'])
add_date = datetime.datetime.strptime(sub_data['SubAddDate'],
'%Y-%m-%d %H:%M:%S')
sub = OpenSubtitlesSubtitleFile(filename=filename,
file_size=file_size,
md5_hash=md5_hash,
id_online=id_online,
download_link=download_link,
link=link,
uploader=uploader,
language=language,
rating=rating,
date=add_date)
rsubs.append(sub)
return UploadResult(type=UploadResult.Type.OK, rsubs=rsubs)
def imdb_search_title(self, title):
self._ensure_connection()
def run_query():
return self._xmlrpc.SearchMoviesOnIMDB(self._token, title.strip())
result = self._safe_exec(run_query, default=None)
self.check_result(result)
imdbs = []
re_title = re.compile(r'(?P<title>.*) \((?P<year>[0-9]+)\)')
for imdb_data in result['data']:
imdb_id = imdb_data['id']
if all(c in string.digits for c in imdb_id):
imdb_id = 'tt{}'.format(imdb_id)
m = re_title.match(imdb_data['title'])
if m:
imdb_title = m['title']
imdb_year = int(m['year'])
else:
imdb_title = imdb_data['title']
imdb_year = None
imdbs.append(
ImdbMovieMatch(imdb_id=imdb_id,
title=imdb_title,
year=imdb_year))
return imdbs
def ping(self):
log.debug('ping()')
if not self.logged_in():
raise ProviderNotConnectedError()
def run_query():
return self._xmlrpc.NoOperation(self._token)
result = self._safe_exec(run_query, None)
self.check_result(result)
def provider_info(self):
if self.connected():
def run_query():
return self._xmlrpc.ServerInfo()
result = self._safe_exec(run_query, None)
data = [
(_('XML-RPC version'), result['xmlrpc_version']),
(_('XML-RPC url'), result['xmlrpc_url']),
(_('Application'), result['application']),
(_('Contact'), result['contact']),
(_('Website url'), result['website_url']),
(_('Users online'), result['users_online_total']),
(_('Programs online'), result['users_online_program']),
(_('Users logged in'), result['users_loggedin']),
(_('Max users online'), result['users_max_alltime']),
(_('Users registered'), result['users_registered']),
(_('Subtitles downloaded'), result['subs_downloads']),
(_('Subtitles available'), result['subs_subtitle_files']),
(_('Number movies'), result['movies_total']),
(_('Number languages'), result['total_subtitles_languages']),
(_('Client IP'), result['download_limits']['client_ip']),
(_('24h global download limit'),
result['download_limits']['global_24h_download_limit']),
(_('24h client download limit'),
result['download_limits']['client_24h_download_limit']),
(_('24h client download count'),
result['download_limits']['client_24h_download_count']),
(_('Client download quota'),
result['download_limits']['client_download_quota']),
]
else:
data = []
return data
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
@classmethod
def get_name(cls):
return 'opensubtitles'
@classmethod
def get_short_name(cls):
return 'os'
@classmethod
def get_icon(cls):
return ':/images/sites/opensubtitles.png'
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
self._ensure_connection()
try:
result = query()
return result
except (ProtocolError, CannotSendRequest, SocketError,
ExpatError) as e:
self._signal_connection_failed()
log.debug('Query failed: {} {}'.format(type(e), e.args))
return default
STATUS_CODE_RE = re.compile(r'(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise OpenSubtitlesProviderConnectionError(None, _('No message'))
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(
status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise OpenSubtitlesProviderConnectionError(
None,
_('Server returned status="{status}". Expected "200 OK".').
format(status=data['status']), data['status'])
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise OpenSubtitlesProviderConnectionError(code, message)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')
class Agent():
""" opensubtitle 的用户代理"""
# __username=""
# __password=""
# __languae=""
# __osd_server=""
# __session=""
opt_selection_mode = 'default'
opt_search_mode = "filename"
opt_language_separator = '_'
def __init__(self, username, password, language="en"):
self.__username = username
self.__password = password
self.__languae = language
self.__osd_server = ServerProxy('http://api.opensubtitles.org/xml-rpc')
def LogIn(self):
try:
# ==== Connection
try:
self.__session = self.__osd_server.LogIn(
self.__username, self.__password, self.__languae,
'opensubtitles-download 4.0')
except Exception:
# Retry once, it could be a momentary overloaded server?
time.sleep(3)
try:
self.__session = self.__osd_server.LogIn(
self.__username, self.__password, self.__languae,
'opensubtitles-download 4.0')
except Exception:
print("error1")
#superPrint("error", "Connection error!", "Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!")
sys.exit(2)
# Connection refused?
if self.__session['status'] != '200 OK':
print("error2")
#superPrint("error", "Connection error!", "Opensubtitles.org servers refused the connection: " + session['status'] + ".\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!")
sys.exit(2)
print("login success!")
except:
print("login error3")
sys.exit(2)
def LogOut(self):
if self.__session and self.__session['token']:
self.__osd_server.LogOut(self.__session['token'])
print("logout success")
def __search(self, title):
""" 查询服务器，寻找潜在的字幕"""
searchList = []
searchList.append({'sublanguageid': 'eng', 'query': title})
try:
subtitlesList = self.__osd_server.SearchSubtitles(
self.__session['token'], searchList)
return subtitlesList
# time.sleep(1)
except Exception:
# Retry once, we are already connected, the server maybe momentary overloaded
time.sleep(3)
try:
subtitlesList = self.__osd_server.SearchSubtitles(
self.__session['token'], searchList)
return subtitlesList
except Exception:
print("Search error!")
#superPrint("error", "Search error!", "Unable to reach opensubtitles.org servers!\n<b>Search error</b>")
return -1
def __selectionAuto(self, subtitlesList, year, imdb_ID):
""" 从众多字幕中寻找合适的字幕 """
"""Automatic subtitles selection using filename match"""
# 需要用到的属性
# IDMovieImdb the most import
# MovieKind
# MovieYear
# SubDownloadLink
# SubFileName
# SubFormat
# 顺序选择即可。
# if len(subtitlesList['data'])==1:
# return 0
data = subtitlesList['data']
for i in range(len(data)):
this = data[i]
if this["IDMovieImdb"] != imdb_ID:
continue
# if this["SubFormat"]=="srt":
# return i
if this["MovieKind"] != "movie":
continue
if this["SubFormat"] == "srt" and abs(year -
int(this["MovieYear"])) < 2:
return i
return -1
def __down(self, subtitlesSelected, subtitlesList, subPath):
# 此处变量与下载有直接关系
subLangId = self.opt_language_separator + subtitlesList['data'][
subtitlesSelected]['ISO639']
subLangName = subtitlesList['data'][subtitlesSelected]['LanguageName']
subURL = subtitlesList['data'][subtitlesSelected]['SubDownloadLink']
# subPath 为存储路径
# subPath="1.txt"
#subPath = videoPath.rsplit('.', 1)[0] + '.' + subtitlesList['data'][subIndex]['SubFormat']
if sys.version_info >= (3, 0):
tmpFile1, headers = urllib.request.urlretrieve(subURL)
tmpFile2 = gzip.GzipFile(tmpFile1)
byteswritten = open(subPath, 'wb').write(tmpFile2.read())
if byteswritten > 0:
process_subtitlesDownload = 0
else:
process_subtitlesDownload = 1
else: # python 2
tmpFile1, headers = urllib.urlretrieve(subURL)
tmpFile2 = gzip.GzipFile(tmpFile1)
open(subPath, 'wb').write(tmpFile2.read())
process_subtitlesDownload = 0
# If an error occurs, say so
if process_subtitlesDownload != 0:
print("Subtitling error!")
#superPrint("error", "Subtitling error!", "An error occurred while downloading or writing <b>" + subtitlesList['data'][subIndex]['LanguageName'] + "</b> subtitles for <b>" + videoTitle + "</b>.")
self.__osd_server.LogOut(self.__session['token'])
sys.exit(2)
def __result(self, subtitlesList, year, imdb_ID, subPath):
""" 主体函数 """
# Parse the results of the XML-RPC query
if ('data' in subtitlesList) and (len(subtitlesList['data']) > 0):
# print("find it")
#找到数据
# Mark search as successful
subtitlesSelected = ''
# If there is only one subtitles (matched by file hash), auto-select it (except in CLI mode)
if (len(subtitlesList['data'])
== 1) and (subtitlesList['data'][0]['MatchedBy']
== 'moviehash'):
if opt_selection_mode != 'manual':
# subtitlesSelected = subtitlesList['data'][0]['SubFileName']
subtitlesSelected = 0
# Get video title
videoTitle = subtitlesList['data'][0]['MovieName']
# If there is more than one subtitles and opt_selection_mode != 'auto',
# then let the user decide which one will be downloaded
if subtitlesSelected == '':
subtitlesSelected = self.__selectionAuto(
subtitlesList, year, imdb_ID)
if subtitlesSelected == -1:
return -1
# If a subtitles has been selected at this point, download it!
if subtitlesSelected == -1:
# 记录未下载的电影名与ID
#print("find error!")
return -1
else:
# print("ready to down")
self.__down(subtitlesSelected, subtitlesList, subPath)
return 0
else:
# print("not find it!")
return -1
def work(self, title, year, imdb_ID, path):
""" 查找电影字幕并下载到指定路径中"""
subtitlesList = self.__search(title)
time.sleep(2)
if subtitlesList == -1:
return -1
return self.__result(subtitlesList, year, imdb_ID, path)
pass
def work_by_imda(self, year, imdb_ID):
list = self.__search(imdb_ID)
if list == -1:
return -1
return self.__selectionAuto(list, year, imdb_ID)
pass
pass
class OpenSubtitles(QtCore.QThread):
api_url = 'http://api.opensubtitles.org/xml-rpc'
login_token = None
server = None
def __init__(self, parent=None):
QtCore.QThread.__init__(self, parent)
self.stopping = False
def process(self, files_list, lang='English'):
self.moviefiles = files_list
self.imdbid_to_hash = {}
self.lang = LANGUAGES[lang][1]
if not self.stopping:
self.start()
def __del__(self):
if self.login_token:
self.logout()
def stopTask(self):
self.stopping = True
def run(self):
utils.communicator.updategui.emit('Querying OpenSubtitles.org...', 'info'
)
try:
if not self.login_token:
self.login()
self.search_subtitles()
except utils.NoInternetConnectionFound:
utils.communicator.updategui.emit('No active Internet connection found. Kindly check and try again.',
'error')
def login(self):
'''Log in to OpenSubtitles.org'''
self.server = ServerProxy(self.api_url, verbose=False)
utils.communicator.updategui.emit('Logging in to OpenSubtitles.org...', 'info'
)
try:
resp = self.server.LogIn('', '', 'en', 'PySubD v2.0')
self.check_status(resp)
self.login_token = resp['token']
except Exception as e:
if e.args[0] == 11004:
utils.communicator.updategui.emit("No Internet Connection Found", 'error')
else:
utils.communicator.updategui.emit(str(e), 'error')
def logout(self):
'''Log out from OpenSubtitles'''
utils.communicator.updategui.emit('Logging out...', 'info')
resp = self.server.LogOut(self.login_token)
self.check_status(resp)
def check_status(self, resp):
'''Check the return status of the request.
Anything other than "200 OK" raises a UserWarning
'''
if resp['status'].upper() != '200 OK':
raise utils.IncorrectResponseRecieved('Response error from %s. Response status: %s'
% (self.api_url, resp['status']))
def _query_opensubs(self, search):
results = []
while search[:500]:
if not self.stopping:
try:
tempresp = self.server.SearchSubtitles(self.login_token, search[:500])
except Exception as e:
if e.args[0] == 11004:
utils.communicator.updategui.emit("No Internet Connection Found", 'error')
else:
utils.communicator.updategui.emit(str(e), 'error')
return results
if tempresp['data']:
results.extend(tempresp['data'])
self.check_status(tempresp)
search = search[500:]
else:
return []
logger.debug('Results: %s' % results)
return results
def clean_results(self, results, imdb=False):
subtitles = {}
user_ranks = { 'administrator': 1,
'platinum member': 2,
'vip member': 3,
'gold member': 4,
'trusted': 5,
'silver member': 6,
'bronze member': 7,
'sub leecher': 8,
'': 9, }
for result in results:
if result['SubBad'] != '1':
movie_hash = result.get('MovieHash')
if not movie_hash:
movie_hash = self.imdbid_to_hash[int(result['IDMovieImdb'])]
subid = result['IDSubtitleFile']
downcount = int(result['SubDownloadsCnt'])
rating = float(result['SubRating'])
if rating and rating < 8:
# Ignore poorly rated subtitles, while not
# penalizing the ones that haven't yet been rated
continue
user_rank = user_ranks[result['UserRank']]
if imdb:
cleaned_release_name = utils.clean_name(result['MovieReleaseName'])
file_name = self.moviefiles[movie_hash]['file_name']
cleaned_file_name = utils.clean_name(file_name)
overlap = len(set.intersection(set(cleaned_release_name), set(cleaned_file_name)))
else:
overlap = 0
subtitles.setdefault(movie_hash, []).append({
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank,
'overlap' : overlap
})
return subtitles
def search_subtitles(self):
search = []
for video_file_details in self.moviefiles.itervalues():
video_file_details['sublanguageid'] = self.lang
search.append(video_file_details)
results = self._query_opensubs(search)
subtitles = self.clean_results(results)
for (hash, found_matches) in subtitles.iteritems():
subtitles[hash] = utils.multikeysort(found_matches,
['overlap', 'user_rank', '-rating', '-downcount'])[0]
for (hash, filedetails) in self.moviefiles.iteritems():
if not self.stopping:
if subtitles.get(hash):
utils.communicator.updategui.emit('Saving subtitles for %s'%filedetails['file_name'], 'success')
subtitle = \
self.download_subtitles([subtitles[hash]['subid'
]])
utils.communicator.downloaded_sub.emit()
utils.save_subs(subtitle, filedetails['save_subs_to'],
subtitles[hash])
else:
utils.communicator.no_sub_found.emit(filedetails['file_name'])
else:
return
def download_subtitles(self, subparam):
resp = self.server.DownloadSubtitles(self.login_token, subparam)
self.check_status(resp)
decoded = base64.standard_b64decode(resp['data'][0]['data'
].encode('ascii'))
decompressed = zlib.decompress(decoded, 15 + 32)
return decompressed
class OpenSubtitles(object):
def __init__(self):
self.xmlrpc = ServerProxy(Settings.OPENSUBTITLES_SERVER,
allow_none=True)
self.language = Settings.LANGUAGE
self.token = None
def _get_from_data_or_none(self, key):
"""
Проверка на аутентификацию
Args:
key (str): строка с параметром который нужно вернуть
Returns:
(str) : возвращаемое значение
"""
status = self.data.get('status').split()[0]
return self.data.get(key) if '200' == status else None
def login(self):
"""
Авторизация
Returns:
(str) : токен
"""
self.data = self.xmlrpc.LogIn(Settings.USER_NAME,
Settings.USER_PASSWORD, self.language,
Settings.USER_AGENT)
token = self._get_from_data_or_none('token')
if token:
self.token = token
return token
def logout(self):
"""
Логаут
Returns:
(str) : статус
"""
data = self.xmlrpc.LogOut(self.token)
return '200' in data.get('status')
def search_subtitles(self, params):
"""
Поиск субтитров
Args:
params(dict):
+ язык субтитров
+ id фильма на сайте *imdb.com*
Returns:
(dic) :
+ id субтитров
+ формат субтитров
+ id фильма
"""
self.data = self.xmlrpc.SearchSubtitles(self.token, params)
return self.data
def search_movies_on_imdb(self, params):
"""
Поиск фильмов
Args:
params(dict):
+ название фильма
Returns:
(dic) :
+ id фильма
+ название фильма
"""
self.data = self.xmlrpc.SearchMoviesOnIMDB(self.token, params)
return self.data
def download_subtitles(self, params):
"""
Загрузка субтитров
Args:
params(int): id субтитров
Returns:
(dic) :
+ id субтитров
+ строка с субтитрами
"""
self.data = self.xmlrpc.DownloadSubtitles(self.token, params)
return self.data
class OpenSubtitles(SubtitleProvider):
URL = 'http://api.opensubtitles.org/xml-rpc'
def __init__(self):
SubtitleProvider.__init__(self)
self._xmlrpc = None
self._token = None
self._settings = OpenSubtitlesSettings()
def get_settings(self):
return self._settings
def set_settings(self, settings):
self._settings = settings
def connect(self):
log.debug('connect()')
if self.connected():
return
self._xmlrpc = ServerProxy(self.URL, allow_none=False)
def disconnect(self):
log.debug('disconnect()')
if self.logged_in():
self.logout()
if self.connected():
self._xmlrpc = None
def connected(self):
return self._xmlrpc is not None
def login(self):
log.debug('login()')
if self.logged_in():
return
if not self.connected():
raise ProviderNotConnectedError()
def login_query():
# FIXME: 'en' language ok??? or '' as in the original
return self._xmlrpc.LogIn(str(self._settings.username),
str(self._settings.password), 'en',
str(self._settings.get_user_agent()))
result = self._safe_exec(login_query, None)
self.check_result(result)
self._token = result['token']
def logout(self):
log.debug('logout()')
if self.logged_in():
def logout_query():
return self._xmlrpc.LogOut(self._token)
result = self._safe_exec(logout_query, None)
self.check_result(result)
self._token = None
def logged_in(self):
return self._token is not None
SEARCH_LIMIT = 500
def search_videos(self, videos, callback, languages=None):
log.debug('search_videos(#videos={})'.format(len(videos)))
if not self.logged_in():
raise ProviderNotConnectedError()
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(
window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc.SearchSubtitles(
self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(
remote_language_raw, xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(
rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitlesSubtitleFile(
filename=remote_filename,
file_size=remote_file_size,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
age=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
imdb_identity = ImdbIdentity(imdb_id=imdb_id,
imdb_rating=None)
identity = ProviderIdentities(imdb_identity=imdb_identity,
provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception(
'Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(
queries=queries))
log.error('Offending result is: {remote_sub}'.format(
remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def query_text(self, query):
return OpenSubtitlesTextQuery(query=query)
def download_subtitles(self, os_rsubs):
log.debug('download_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(
window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({
item['idsubtitlefile']: item['data']
for item in result['data']
})
subtitles = [
unzip_bytes(base64.b64decode(
map_id_data[os_rsub.get_id_online()])).read()
for os_rsub in os_rsubs
]
return subtitles
def ping(self):
log.debug('ping()')
if not self.logged_in():
raise ProviderNotConnectedError()
def run_query():
return self._xmlrpc.NoOperation(self._token)
result = self._safe_exec(run_query, None)
self.check_result(result)
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
@classmethod
def get_name(cls):
return 'opensubtitles'
@classmethod
def get_short_name(cls):
return 'os'
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
try:
result = query()
return result
except (ProtocolError, CannotSendRequest, SocketError):
self._signal_connection_failed()
log.warning('Query failed', exc_info=sys.exc_info())
return default
STATUS_CODE_RE = re.compile('(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise ProviderConnectionError(_('No message'))
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(
status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(
_('Server returned status="{status}". Expected "200 OK".').
format(status=data['status']), data['status'])
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(message, code)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')
class SubtitleDownload(QtCore.QThread):
'''Traverses a directory and all subdirectories and downloads subtitles.
Relies on an undocumented OpenSubtitles feature where subtitles seems to be
sorted by filehash and popularity.
'''
api_url = 'http://api.opensubtitles.org/xml-rpc'
login_token = None
server = None
def __init__(self, parent=None):
QtCore.QThread.__init__(self, parent)
def init(self, movie_paths):
self._movie_paths = movie_paths
self.start()
def __del__(self):
if self.login_token:
self.logout()
def stopTask(self):
self.stopping = True
def run(self):
self.stopping = False
self.moviefiles = {}
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Searching for video files...")
"""Check if the movie_path is a file or directory.
Calculate the file stats if its a file, else traverse the directory tree and calculate filestats for all movie files"""
for path in self._movie_paths:
if os.path.isfile(path):
self.calcfilestats(path, os.path.dirname(path))
else:
for root, _, files in os.walk(path):
for file in files:
if not self.stopping:
self.calcfilestats(file, root)
else:
return
try:
if self.moviefiles:
if not self.login_token:
try:
self.login()
except (noInternetConnection):
self.emit(
QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Sorry, No active Internet connection found. Re-Check and try again."
)
my_logger.debug(
"Sorry, No active Internet connection found. Re-Check and try again."
)
self.emit(
QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"),
self._movie_paths)
return
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Searching for subtitles...")
self.search_subtitles()
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Done...")
my_logger.debug("Done...")
else:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Sorry, no video files were found!")
self.emit(QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"),
self._movie_paths)
except Error as e:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
("XML-RPC error:", e))
except UserWarning as uw:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), uw)
def calcfilestats(self, file, parentdir):
''' Calculates and adds the hash of a movie file to the moviefiles dictionary.
Also emits a signal to update the LCD counter showing the found files.'''
video_extns = [
'.avi', '.divx', '.mkv', '.mp4', '.ogg', '.rm', '.rmvb', '.vob',
'.x264', '.xvid'
]
if os.path.splitext(file)[1].lower() in video_extns:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Found: " + file)
my_logger.debug("Found: " + file)
self.emit(QtCore.SIGNAL("updateFound()"))
filehash = self.hashFile(os.path.join(parentdir, file))
filesize = os.path.getsize(os.path.join(parentdir, file))
self.moviefiles[filehash] = {
'dir': parentdir,
'file': file,
'size': filesize
}
def login(self):
'''Log in to OpenSubtitles'''
self.server = ServerProxy(self.api_url, verbose=False)
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging in...")
my_logger.debug("Logging in...")
try:
resp = self.server.LogIn('', '', 'en', 'PySubD v1.0')
self.check_status(resp)
self.login_token = resp['token']
except Error as e:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), e)
my_logger.debug(str(e))
def logout(self):
'''Log out from OpenSubtitles'''
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging out...")
resp = self.server.LogOut(self.login_token)
self.check_status(resp)
def search_subtitles(self):
'''Search OpenSubtitles for matching subtitles using moviehash and filebytesize'''
search = []
for hash in self.moviefiles.keys():
search.append({
'sublanguageid': 'eng',
'moviehash': hash,
'moviebytesize': str(self.moviefiles[hash]['size'])
})
my_logger.debug("Length of search string: " + str(len(search)))
resp = {}
resp['data'] = []
while search[:500]:
if not self.stopping:
tempresp = self.server.SearchSubtitles(self.login_token,
search[:500])
if tempresp['data'] != False:
resp['data'].extend(tempresp['data'])
self.check_status(tempresp)
search = search[500:]
else:
return
#Check if we actually got some matching subtitles to download, else return
if not resp['data']:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Sorry, no subtitles were found!")
my_logger.debug("Sorry, no subtitles were found!")
return
#User Ranks on OpenSubtitles.org.These are decided based upon the number of subtitles uploaded by the member.
#A better rank is often an indication of a better subtitle quality.
user_ranks = {
'admin': 1,
'platinum member': 2,
'vip member': 3,
'gold member': 4,
'trusted': 5,
'silver member': 6,
'bronze member': 7,
'sub leecher': 8,
'': 9
}
# A dictionary to store the subtitle id's found corresponding to every file hash
subtitles = {}
for result in resp['data']:
#The subtitle must not be rated bad
if int(result['SubBad']) != 1:
hash = result['MovieHash']
subid = result['IDSubtitleFile']
downcount = result['SubDownloadsCnt']
rating = result['SubRating']
user_rank = user_ranks[result['UserRank']]
#First good matching subtitle found
if not subtitles.get(hash):
subtitles[hash] = {
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank
}
self.emit(QtCore.SIGNAL("updateAvailable()"))
#Another good quality subtitle found uploaded by a more reputed user
elif subtitles[hash]['user_rank'] > user_rank:
subtitles[hash] = {
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank
}
#Another good quality subtitle found with a perfect rating, uploaded by a more or equally reputed user
elif float(rating) == 10.0 and subtitles[hash][
'user_rank'] >= user_rank:
subtitles[hash] = {
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank
}
#Another good quality subtitle found with the better rating, higher download count and uploaded by a more or equally trusted user
elif float(subtitles[hash]['rating']) >= float(rating) \
and int(subtitles[hash]['downcount']) < int(downcount) and subtitles[hash]['user_rank'] >= user_rank:
subtitles[hash] = {
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank
}
my_logger.debug("Total number of subtitles found: " +
str(len(subtitles)))
notfound = []
for hash, filedetails in self.moviefiles.iteritems():
if not self.stopping:
if subtitles.get(hash):
try:
subtitle = self.download_subtitles(
[subtitles[hash]['subid']])
self.emit(
QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Saving subtitle for: " + filedetails['file'])
my_logger.debug("Saving subtitle for: " + filedetails['file'] + " Hash : " + hash + " Rating: " +
subtitles[hash]['rating'] + " DwnCnt: " + subtitles[hash]['downcount'] + " UpldrRnk:" + \
str(subtitles[hash]['user_rank']))
self.emit(QtCore.SIGNAL("updateDownloaded()"))
filename = os.path.join(
filedetails['dir'],
os.path.splitext(filedetails['file'])[0] + ".srt")
file = open(filename, "wb")
file.write(subtitle)
file.close()
except IOError:
self.emit(
QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"IO Error in saving subs for " +
filedetails['file'])
else:
notfound.append(filedetails['file'])
else:
return
#Report all the files for which no subtitles were found in an alphabetically sorted order.
notfound.sort(key=str.lower)
for file in notfound:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"No subtitles found for: " + file)
my_logger.debug("No subtitles were found for: " + file)
def download_subtitles(self, subparam):
resp = self.server.DownloadSubtitles(self.login_token, subparam)
self.check_status(resp)
decoded = base64.standard_b64decode(
resp['data'][0]['data'].encode('ascii'))
decompressed = zlib.decompress(decoded, 15 + 32)
return decompressed
def check_status(self, resp):
'''Check the return status of the request.
Anything other than "200 OK" raises a UserWarning
'''
if resp['status'].upper() != '200 OK':
raise UserWarning("Response error from " + self.api_url +
". Response status was: " + resp['status'])
def hashFile(self, name):
''' Calculates the hash value of a movie.
Copied from OpenSubtitles own examples:
http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
'''
try:
longlongformat = 'q' # long long
bytesize = struct.calcsize(longlongformat)
f = open(name, "rb")
filesize = os.path.getsize(name)
hash = filesize
if filesize < 65536 * 2:
return "SizeError: Minimum file size must be 120Kb"
for x in range(65536 // bytesize):
buffer = f.read(bytesize)
(l_value, ) = struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number
f.seek(max(0, filesize - 65536), 0)
for x in range(65536 // bytesize):
buffer = f.read(bytesize)
(l_value, ) = struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF
f.close()
returnedhash = "%016x" % hash
return returnedhash
except (IOError):
return "IOError"
if opt_search_mode in ('hash', 'hash_then_filename',
'hash_and_filename'):
subtitlesSearchList.append({
'sublanguageid': currentLanguage,
'moviehash': videoHash,
'moviebytesize': str(videoSize)
})
if opt_search_mode in ('filename', 'hash_and_filename'):
subtitlesSearchList.append({
'sublanguageid': currentLanguage,
'query': videoFileName
})
## Primary search
try:
subtitlesResultList = osd_server.SearchSubtitles(
session['token'], subtitlesSearchList)
except Exception:
# Retry once after a delay (we are already connected, the server may be momentary overloaded)
time.sleep(3)
try:
subtitlesResultList = osd_server.SearchSubtitles(
session['token'], subtitlesSearchList)
except Exception:
superPrint(
"error", "Search error!",
"Unable to reach opensubtitles.org servers!\n<b>Search error</b>"
)
#if (opt_search_mode == 'hash_and_filename'):
# TODO Cleanup duplicate between moviehash and filename results
videoSize = os.path.getsize(videoPath)
videoFileName = os.path.basename(videoPath)
# Count languages marked for this search
for SubLanguageID in SubLanguageIDs:
searchLanguage += len(SubLanguageID.split(','))
# Search for available subtitles (using file hash and size)
for SubLanguageID in SubLanguageIDs:
searchList = []
searchList.append({
'sublanguageid': SubLanguageID,
'moviehash': videoHash,
'moviebytesize': str(videoSize)
})
subtitlesList = server.SearchSubtitles(token, searchList)
subtitlesSelected = ''
# Parse the results of the XML-RPC query
if subtitlesList['data']:
# Mark search as successful
searchLanguageResult += 1
# If there is only one subtitles, auto-select it
if len(subtitlesList['data']) == 1:
subtitlesSelected = subtitlesList['data'][0]['SubFileName']
# If there is more than one subtitles and opt_selection_mode != 'auto',
# then let the user decide which one will be downloaded
if len(subtitlesList['data']) > 1:
class OpenSub(object):
def __init__(self, path):
self.TAGS = [
'bluray', 'cam', 'dvb', 'dvd', 'hd-dvd', 'hdtv', 'ppv', 'telecine',
'telesync', 'tv', 'vhs', 'vod', 'web-dl', 'webrip', 'workprint'
]
self.OPENSUBTITLES_SERVER = 'https://api.opensubtitles.org:443/xml-rpc'
self.USER_AGENT = 'TemporaryUserAgent' # needs to be changed from time to time to the recent temporary UA
self.xmlrpc = ServerProxy(self.OPENSUBTITLES_SERVER, allow_none=True)
self.language = 'eng'
self.token = None
self.mv_name = None
self.path = path
try:
with open("opensubtitles_userpass.txt", 'r') as f:
f = f.read()
user = f.split('\n')[0].split('=')[1]
password = f.split('\n')[1].split('=')[1]
self.user = user
self.passw = password
except FileNotFoundError:
print(
"Please fill OpenSubtitles credentials in opensubtitles_userpass.txt"
)
if self.login(self.user, self.passw):
print("[OpenSubtitles] Login Successful")
else:
print("[OpenSubtitles] Login Failed")
def _get_from_data_or_none(self, key):
'''Return the key recieved from data if the status is 200,
otherwise return None.
'''
status = self.data.get('status').split()[0]
return self.data.get(key) if '200' == status else None
def login(self, username, password):
'''Returns token is login is ok, otherwise None.
'''
self.data = self.xmlrpc.LogIn(username, password, self.language,
self.USER_AGENT)
token = self._get_from_data_or_none('token')
if token:
self.token = token
return token
def logout(self):
'''Returns True if logout is ok, otherwise None.
'''
data = self.xmlrpc.LogOut(self.token)
return '200' in data.get('status')
def search_subtitles(self, params):
'''Returns a list with the subtitles info.
'''
self.data = self.xmlrpc.SearchSubtitles(self.token, params)
return self._get_from_data_or_none('data')
def download_subtitles(self, payload):
"""
:return:
"""
print("Attempting to download subtitles...")
search_result = self.search_subtitles([payload])
if search_result:
dllink = self.analyse_result(search_result, payload)
gzfile = request.urlopen(dllink)
try:
with gzip.open(gzfile, 'rb') as f:
with open(self.path + "\\" + self.mv_name + '.srt',
'wb') as sub_file:
sub_file.write(f.read())
print("[OpenSubtitles] Subtitles downloaded for: " +
self.mv_name)
# Update stats
with open("metadata.json", "r") as jsonFile:
data = json.load(jsonFile)
data['num_of_movies_in_db'] = int(
data['num_of_movies_in_db']) + 1
data['latest_movie_added']['name'] = self.mv_name
data['latest_movie_added']['time'] = str(
datetime.datetime.now())
with open("metadata.json", "w") as jsonFile:
json.dump(data, jsonFile, indent=4)
# End Update
return os.path.join(self.path, self.mv_name + ".srt")
except PermissionError:
print(
"[OpenSubtitles] Permission Error: when creating subtitles for {}"
.format(self.mv_name))
def analyse_result(self, result, payload):
"""
:param result: Search result to find appropriate subtitles
:return: Download Link of best match for subtitles
"""
score = 0
dllink = None
for record in result:
if record.get("MovieName").lower() == payload['query'].lower(
) and record.get("SubLanguageID") == 'eng':
self.mv_name = record.get("MovieName")
dllink = record.get('SubDownloadLink')
break
if record.get('Score',
0) > score and record.get("SubLanguageID") == 'eng':
self.mv_name = record.get("MovieName")
score = record.get('Score', 0)
dllink = record.get('SubDownloadLink')
return dllink
def create_payload(self, mv_name):
"""
:param lang: Subtitle Language
:param mv_name: Movie name to search
:return: Payload containing data about file
"""
payload = {}
payload['query'] = mv_name
payload['SubLanguageID'] = self.language
payload['SubFormat'] = 'srt'
return payload
def search_and_download(self, mv_name):
movie_db = [
os.path.splitext(name)[0] for name in os.listdir(self.path)
]
for local in movie_db:
if mv_name.lower() == local.lower():
return os.path.join(self.path, mv_name + ".srt")
try:
return self.download_subtitles(self.create_payload(mv_name))
except:
print("[OpenSubtitles] Could not download subtitles for " +
mv_name)
def download_top_1000(self):
with open(os.path.join(os.getcwd(), "top1000.txt"), "r") as f:
f = f.read()
i = 1
for mv_name in f.split("\n"):
try:
self.search_and_download(mv_name)
except:
e = sys.exc_info()[0]
print("ERROR: " + str(e))
print("Error: subtitles not downloaded for: " + mv_name)
i = i + 1
if i % 39 == 0:
time.sleep(11)
def on_task_download(self, task, config):
# filter all entries that have IMDB ID set
try:
entries = [e for e in task.accepted if e['imdb_id'] is not None]
except KeyError:
# No imdb urls on this task, skip it
# TODO: should do lookup via imdb_lookup plugin?
return
try:
s = ServerProxy("http://api.opensubtitles.org/xml-rpc")
res = s.LogIn("", "", "en", "FlexGet")
except Exception:
logger.warning('Error connecting to opensubtitles.org')
return
if res['status'] != '200 OK':
raise Exception("Login to opensubtitles.org XML-RPC interface failed")
config = self.prepare_config(config, task)
token = res['token']
# configuration
languages = config['languages']
min_sub_rating = config['min_sub_rating']
match_limit = config[
'match_limit'
] # no need to change this, but it should be configurable
# loop through the entries
for entry in entries:
imdbid = entry.get('imdb_id')
if not imdbid:
logger.debug('no match for {}', entry['title'])
continue
query = []
for language in languages:
query.append({'sublanguageid': language, 'imdbid': imdbid})
subtitles = s.SearchSubtitles(token, query)
subtitles = subtitles['data']
# nothing found -> continue
if not subtitles:
continue
# filter bad subs
subtitles = [x for x in subtitles if x['SubBad'] == '0']
# some quality required (0.0 == not reviewed)
subtitles = [
x
for x in subtitles
if float(x['SubRating']) >= min_sub_rating or float(x['SubRating']) == 0.0
]
filtered_subs = []
# find the best rated subs for each language
for language in languages:
langsubs = [x for x in subtitles if x['SubLanguageID'] == language]
# did we find any subs for this language?
if langsubs:
def seqmatch(subfile):
s = difflib.SequenceMatcher(lambda x: x in " ._", entry['title'], subfile)
# print "matching: ", entry['title'], subfile, s.ratio()
return s.ratio() > match_limit
# filter only those that have matching release names
langsubs = [x for x in subtitles if seqmatch(x['MovieReleaseName'])]
if langsubs:
# find the best one by SubRating
langsubs.sort(key=lambda x: float(x['SubRating']))
langsubs.reverse()
filtered_subs.append(langsubs[0])
# download
for sub in filtered_subs:
logger.debug(
'SUBS FOUND: {} {} {}',
sub['MovieReleaseName'],
sub['SubRating'],
sub['SubLanguageID'],
)
f = task.requests.get(sub['ZipDownloadLink'])
subfilename = re.match(
'^attachment; filename="(.*)"$', f.headers['content-disposition']
).group(1)
outfile = os.path.join(config['output'], subfilename)
fp = open(outfile, 'w')
fp.write(f.raw)
fp.close()
f.close()
s.LogOut(token)
videoSize = os.path.getsize(videoPath)
videoFileName = os.path.basename(videoPath)
# ==== Search for available subtitles on OpenSubtitlesDownload
for SubLanguageID in opt_languages:
searchList = []
subtitlesList = {}
if opt_search_mode in ('hash', 'hash_then_filename', 'hash_and_filename'):
searchList.append({'sublanguageid':SubLanguageID, 'moviehash':videoHash, 'moviebytesize':str(videoSize)})
if opt_search_mode in ('filename', 'hash_and_filename'):
searchList.append({'sublanguageid':SubLanguageID, 'query':videoFileName})
## Primary search
try:
subtitlesList = osd_server.SearchSubtitles(session['token'], searchList)
except Exception:
# Retry once after a delay (we are already connected, the server may be momentary overloaded)
time.sleep(3)
try:
subtitlesList = osd_server.SearchSubtitles(session['token'], searchList)
except Exception:
superPrint("error", "Search error!", "Unable to reach opensubtitles.org servers!\n<b>Search error</b>")
#if (opt_search_mode == 'hash_and_filename'):
# TODO Cleanup duplicate between moviehash and filename results
## Fallback search
if ((opt_search_mode == 'hash_then_filename') and (('data' in subtitlesList) and (not subtitlesList['data']))):
searchList[:] = [] # searchList.clear()
searchList.append({'sublanguageid':SubLanguageID, 'query':videoFileName})
class SDService(object):
"""
Contains the class that represents the OSDB RPC Server.
Encapsules all the XMLRPC methods.
Consult the OSDB API methods at http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC
If it fails to connect directly to the XMLRPC server, it will try to do so through a default proxy.
Default proxy uses a form to set which URL to open. We will try to change this in later stage.
"""
def __init__(self, server=None, proxy=None):
self.log = logging.getLogger("subdownloader.SDService.SDService")
self.log.debug(
"Creating Server with server = %s and proxy = %r" % (server, proxy))
self.timeout = 30
self.user_agent = USER_AGENT
self.language = ''
if server:
self.server = server
else:
self.server = DEFAULT_OSDB_SERVER
self.proxy = proxy
self.logged_as = None
self._xmlrpc_server = None
self._token = None
def connected(self):
return self._token is not None
def connect(self):
server = self.server
proxy = self.proxy
self.log.debug("connect()... to server %s with proxy %s" % (server, proxy))
connect_res = False
try:
self.log.debug(
"Connecting with parameters (%r, %r)" % (server, proxy))
connect = TimeoutFunction(self._connect)
connect_res = connect(server, proxy)
except TimeoutFunctionException as e:
self.log.error("Connection timed out. Maybe you need a proxy.")
raise
except:
self.log.exception("connect: Unexpected error")
raise
finally:
self.log.debug("connection connected %s" % connect_res)
return connect_res
def _connect(self, server, proxy):
try:
if proxy:
self.log.debug("Trying proxied connection... ({})".format(proxy))
self.proxied_transport = ProxiedTransport()
self.proxied_transport.set_proxy(proxy)
self._xmlrpc_server = ServerProxy(
server, transport=self.proxied_transport, allow_none=True)
# self.ServerInfo()
self.log.debug("...connected")
return True
elif test_connection(TEST_URL):
self.log.debug("Trying direct connection...")
self._xmlrpc_server = ServerProxy(
server, allow_none=True)
# self.ServerInfo()
self.log.debug("...connected")
return True
else:
self.log.debug("...failed")
self.log.error("Unable to connect. Try setting a proxy.")
return False
except ProtocolError as e:
self._connection_failed()
self.log.debug("error in HTTP/HTTPS transport layer")
raise
except Fault as e:
self.log.debug("error in xml-rpc server")
raise
except:
self.log.exception("Connection to the server failed/other error")
raise
def logged_in(self):
return self._token is not None
def login(self, username="", password=""):
try:
login = TimeoutFunction(self._login)
return login(username, password)
except TimeoutFunctionException:
self.log.error("login timed out")
except:
self.log.exception("login: other issue")
raise
def _login(self, username="", password=""):
"""Login to the Server using username/password,
empty parameters means an anonymously login
Returns True if login sucessful, and False if not.
"""
self.log.debug("----------------")
self.log.debug("Logging in (username: %s)..." % username)
def run_query():
return self._xmlrpc_server.LogIn(
username, password, self.language, self.user_agent)
info = self._safe_exec(run_query, None)
if info is None:
self._token = None
return False
self.log.debug("Login ended in %s with status: %s" %
(info['seconds'], info['status']))
if info['status'] == "200 OK":
self.log.debug("Session ID: %s" % info['token'])
self.log.debug("----------------")
self._token = info['token']
return True
else:
# force token reset
self.log.debug("----------------")
self._token = None
return False
def logout(self):
try:
logout = TimeoutFunction(self._logout)
result = logout()
self._token = None
return result
except TimeoutFunctionException:
self.log.error("logout timed out")
def _logout(self):
"""Logout from current session(token)
This functions doesn't return any boolean value, since it can 'fail' for anonymous logins
"""
self.log.debug("Logging out from session ID: %s" % self._token)
try:
info = self._xmlrpc_server.LogOut(self._token)
self.log.debug("Logout ended in %s with status: %s" %
(info['seconds'], info['status']))
except ProtocolError as e:
self.log.debug("error in HTTP/HTTPS transport layer")
raise
except Fault as e:
self.log.debug("error in xml-rpc server")
raise
except:
self.log.exception("Connection to the server failed/other error")
raise
finally:
# force token reset
self._token = None
STATUS_CODE_RE = re.compile('(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise ProviderConnectionError(None, 'No message')
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(None, 'Server returned status="{status}". Expected "200 OK".'.format(
status=data['status']))
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(code, message)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')
@classmethod
def name(cls):
return "opensubtitles"
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
try:
result = query()
return result
except (ProtocolError, xml.parsers.expat.ExpatError):
self._signal_connection_failed()
log.debug("Query failed", exc_info=sys.exc_info())
return default
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
def imdb_query(self, query):
if not self.connected():
return None
def run_query():
return self._xmlrpc_server.SearchMoviesOnIMDB(self._token, query)
result = self._safe_exec(run_query, None)
if result is None:
return None
provider_identities = []
for imdb_data in result['data']:
if not imdb_data:
continue
imdb_identity = ImdbIdentity(imdb_id=imdb_data['id'], imdb_rating=None)
video_identity = VideoIdentity(name=imdb_data['title'], year=None)
provider_identities.append(ProviderIdentities(
video_identity=video_identity, imdb_identity=imdb_identity,
provider=self))
return provider_identities
SEARCH_LIMIT = 500
def search_text(self, text, languages=None):
lang_str = self._languages_to_str(languages)
query = {
'sublanguageid': lang_str,
'query': str(text),
}
queries = [query]
def run_query():
return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT})
self._safe_exec(run_query, None)
def search_videos(self, videos, callback, languages=None):
if not self.connected():
return None
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(remote_language_raw,
xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitles_SubtitleFile(
filename=remote_filename,
file_size=remote_file_size ,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
age=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None)
identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception('Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(queries=queries))
log.error('Offending result is: {remote_sub}'.format(remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def _video_info_to_identification(self, video_info):
name = video_info['MovieName']
year = int(video_info['MovieYear'])
imdb_id = video_info['MovieImdbID']
video_identity = VideoIdentity(name=name, year=year)
imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None)
episode_identity = None
movie_kind = video_info['MovieKind']
if movie_kind == 'episode':
season = int(video_info['SeriesSeason'])
episode = int(video_info['SeriesEpisode'])
episode_identity = EpisodeIdentity(season=season, episode=episode)
elif movie_kind == 'movie':
pass
else:
log.warning('Unknown MoviesKind="{}"'.format(video_info['MovieKind']))
return ProviderIdentities(video_identity=video_identity, episode_identity=episode_identity,
imdb_identity=imdb_identity, provider=self)
def identify_videos(self, videos):
if not self.connected():
return
for part_videos in window_iterator(videos, 200):
hashes = [video.get_osdb_hash() for video in part_videos]
hash_video = {hash: video for hash, video in zip(hashes, part_videos)}
def run_query():
return self._xmlrpc_server.CheckMovieHash2(self._token, hashes)
result = self._safe_exec(run_query, None)
self.check_result(result)
for video_hash, video_info in result['data'].items():
identification = self._video_info_to_identification(video_info[0])
video = hash_video[video_hash]
video.add_identity(identification)
def download_subtitles(self, os_rsubs):
if not self.connected():
return None
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc_server.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({item['idsubtitlefile']: item['data'] for item in result['data']})
subtitles = [unzip_bytes(base64.b64decode(map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs]
return subtitles
def can_upload_subtitles(self, local_movie):
if not self.connected():
return False
query = {}
for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()):
# sub_bytes = open(subtitle.get_filepath(), mode='rb').read()
# sub_tx_data = b64encode(zlib.compress(sub_bytes))
cd = "cd{i}".format(i=i+1)
cd_data = {
'subhash': subtitle.get_md5_hash(),
'subfilename': subtitle.get_filename(),
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
'movietimems': str(video.get_time_ms()),
'moviefps': video.get_fps(),
'movieframes': str(video.get_framecount()),
'moviefilename': video.get_filename(),
}
query[cd] = cd_data
def run_query():
return self._xmlrpc_server.TryUploadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
movie_already_in_db = int(result['alreadyindb']) != 0
if movie_already_in_db:
return False
return True
def upload_subtitles(self, local_movie):
query = {
'baseinfo': {
'idmovieimdb': local_movie.get_imdb_id(),
'moviereleasename': local_movie.get_release_name(),
'movieaka': local_movie.get_movie_name(),
'sublanguageid': local_movie.get_language().xxx(),
'subauthorcomment': local_movie.get_comments(),
},
}
if local_movie.is_hearing_impaired() is not None:
query['hearingimpaired'] = local_movie.is_hearing_impaired()
if local_movie.is_high_definition() is not None:
query['highdefinition'] = local_movie.is_high_definition()
if local_movie.is_automatic_translation() is not None:
query['automatictranslation'] = local_movie.is_automatic_translation()
if local_movie.get_subtitle_author() is not None:
query['subtranslator'] = local_movie.get_subtitle_author()
if local_movie.is_foreign_only() is not None:
query['foreignpartsonly'] = local_movie.is_foreign_only()
for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()):
sub_bytes = subtitle.get_filepath().open(mode='rb').read()
sub_tx_data = b64encode(zlib.compress(sub_bytes))
cd = "cd{i}".format(i=i+1)
cd_data = {
'subhash': subtitle.get_md5_hash(),
'subfilename': subtitle.get_filename(),
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None,
'moviefps': str(video.get_fps()) if video.get_fps() else None,
'movieframes': str(video.get_framecount()) if video.get_framecount() else None,
'moviefilename': video.get_filename(),
'subcontent': sub_tx_data,
}
query[cd] = cd_data
def run_query():
return self._xmlrpc_server.UploadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
