class SDService(object):
"""
Contains the class that represents the OSDB RPC Server.
Encapsules all the XMLRPC methods.
Consult the OSDB API methods at http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC
If it fails to connect directly to the XMLRPC server, it will try to do so through a default proxy.
Default proxy uses a form to set which URL to open. We will try to change this in later stage.
"""
def __init__(self, server=None, proxy=None):
self.log = logging.getLogger("subdownloader.SDService.SDService")
self.log.debug(
"Creating Server with server = %s and proxy = %r" % (server, proxy))
self.timeout = 30
self.user_agent = USER_AGENT
self.language = ''
if server:
self.server = server
else:
self.server = DEFAULT_OSDB_SERVER
self.proxy = proxy
self.logged_as = None
self._xmlrpc_server = None
self._token = None
def connected(self):
return self._token is not None
def connect(self):
server = self.server
proxy = self.proxy
self.log.debug("connect()... to server %s with proxy %s" % (server, proxy))
connect_res = False
try:
self.log.debug(
"Connecting with parameters (%r, %r)" % (server, proxy))
connect = TimeoutFunction(self._connect)
connect_res = connect(server, proxy)
except TimeoutFunctionException as e:
self.log.error("Connection timed out. Maybe you need a proxy.")
raise
except:
self.log.exception("connect: Unexpected error")
raise
finally:
self.log.debug("connection connected %s" % connect_res)
return connect_res
def _connect(self, server, proxy):
try:
if proxy:
self.log.debug("Trying proxied connection... ({})".format(proxy))
self.proxied_transport = ProxiedTransport()
self.proxied_transport.set_proxy(proxy)
self._xmlrpc_server = ServerProxy(
server, transport=self.proxied_transport, allow_none=True)
# self.ServerInfo()
self.log.debug("...connected")
return True
elif test_connection(TEST_URL):
self.log.debug("Trying direct connection...")
self._xmlrpc_server = ServerProxy(
server, allow_none=True)
# self.ServerInfo()
self.log.debug("...connected")
return True
else:
self.log.debug("...failed")
self.log.error("Unable to connect. Try setting a proxy.")
return False
except ProtocolError as e:
self._connection_failed()
self.log.debug("error in HTTP/HTTPS transport layer")
raise
except Fault as e:
self.log.debug("error in xml-rpc server")
raise
except:
self.log.exception("Connection to the server failed/other error")
raise
def logged_in(self):
return self._token is not None
def login(self, username="", password=""):
try:
login = TimeoutFunction(self._login)
return login(username, password)
except TimeoutFunctionException:
self.log.error("login timed out")
except:
self.log.exception("login: other issue")
raise
def _login(self, username="", password=""):
"""Login to the Server using username/password,
empty parameters means an anonymously login
Returns True if login sucessful, and False if not.
"""
self.log.debug("----------------")
self.log.debug("Logging in (username: %s)..." % username)
def run_query():
return self._xmlrpc_server.LogIn(
username, password, self.language, self.user_agent)
info = self._safe_exec(run_query, None)
if info is None:
self._token = None
return False
self.log.debug("Login ended in %s with status: %s" %
(info['seconds'], info['status']))
if info['status'] == "200 OK":
self.log.debug("Session ID: %s" % info['token'])
self.log.debug("----------------")
self._token = info['token']
return True
else:
# force token reset
self.log.debug("----------------")
self._token = None
return False
def logout(self):
try:
logout = TimeoutFunction(self._logout)
result = logout()
self._token = None
return result
except TimeoutFunctionException:
self.log.error("logout timed out")
def _logout(self):
"""Logout from current session(token)
This functions doesn't return any boolean value, since it can 'fail' for anonymous logins
"""
self.log.debug("Logging out from session ID: %s" % self._token)
try:
info = self._xmlrpc_server.LogOut(self._token)
self.log.debug("Logout ended in %s with status: %s" %
(info['seconds'], info['status']))
except ProtocolError as e:
self.log.debug("error in HTTP/HTTPS transport layer")
raise
except Fault as e:
self.log.debug("error in xml-rpc server")
raise
except:
self.log.exception("Connection to the server failed/other error")
raise
finally:
# force token reset
self._token = None
STATUS_CODE_RE = re.compile('(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise ProviderConnectionError(None, 'No message')
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(None, 'Server returned status="{status}". Expected "200 OK".'.format(
status=data['status']))
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(code, message)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')
@classmethod
def name(cls):
return "opensubtitles"
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
try:
result = query()
return result
except (ProtocolError, xml.parsers.expat.ExpatError):
self._signal_connection_failed()
log.debug("Query failed", exc_info=sys.exc_info())
return default
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
def imdb_query(self, query):
if not self.connected():
return None
def run_query():
return self._xmlrpc_server.SearchMoviesOnIMDB(self._token, query)
result = self._safe_exec(run_query, None)
if result is None:
return None
provider_identities = []
for imdb_data in result['data']:
if not imdb_data:
continue
imdb_identity = ImdbIdentity(imdb_id=imdb_data['id'], imdb_rating=None)
video_identity = VideoIdentity(name=imdb_data['title'], year=None)
provider_identities.append(ProviderIdentities(
video_identity=video_identity, imdb_identity=imdb_identity,
provider=self))
return provider_identities
SEARCH_LIMIT = 500
def search_text(self, text, languages=None):
lang_str = self._languages_to_str(languages)
query = {
'sublanguageid': lang_str,
'query': str(text),
}
queries = [query]
def run_query():
return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT})
self._safe_exec(run_query, None)
def search_videos(self, videos, callback, languages=None):
if not self.connected():
return None
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(remote_language_raw,
xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitles_SubtitleFile(
filename=remote_filename,
file_size=remote_file_size ,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
age=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None)
identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception('Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(queries=queries))
log.error('Offending result is: {remote_sub}'.format(remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def _video_info_to_identification(self, video_info):
name = video_info['MovieName']
year = int(video_info['MovieYear'])
imdb_id = video_info['MovieImdbID']
video_identity = VideoIdentity(name=name, year=year)
imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None)
episode_identity = None
movie_kind = video_info['MovieKind']
if movie_kind == 'episode':
season = int(video_info['SeriesSeason'])
episode = int(video_info['SeriesEpisode'])
episode_identity = EpisodeIdentity(season=season, episode=episode)
elif movie_kind == 'movie':
pass
else:
log.warning('Unknown MoviesKind="{}"'.format(video_info['MovieKind']))
return ProviderIdentities(video_identity=video_identity, episode_identity=episode_identity,
imdb_identity=imdb_identity, provider=self)
def identify_videos(self, videos):
if not self.connected():
return
for part_videos in window_iterator(videos, 200):
hashes = [video.get_osdb_hash() for video in part_videos]
hash_video = {hash: video for hash, video in zip(hashes, part_videos)}
def run_query():
return self._xmlrpc_server.CheckMovieHash2(self._token, hashes)
result = self._safe_exec(run_query, None)
self.check_result(result)
for video_hash, video_info in result['data'].items():
identification = self._video_info_to_identification(video_info[0])
video = hash_video[video_hash]
video.add_identity(identification)
def download_subtitles(self, os_rsubs):
if not self.connected():
return None
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc_server.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({item['idsubtitlefile']: item['data'] for item in result['data']})
subtitles = [unzip_bytes(base64.b64decode(map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs]
return subtitles
def can_upload_subtitles(self, local_movie):
if not self.connected():
return False
query = {}
for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()):
# sub_bytes = open(subtitle.get_filepath(), mode='rb').read()
# sub_tx_data = b64encode(zlib.compress(sub_bytes))
cd = "cd{i}".format(i=i+1)
cd_data = {
'subhash': subtitle.get_md5_hash(),
'subfilename': subtitle.get_filename(),
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
'movietimems': str(video.get_time_ms()),
'moviefps': video.get_fps(),
'movieframes': str(video.get_framecount()),
'moviefilename': video.get_filename(),
}
query[cd] = cd_data
def run_query():
return self._xmlrpc_server.TryUploadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
movie_already_in_db = int(result['alreadyindb']) != 0
if movie_already_in_db:
return False
return True
def upload_subtitles(self, local_movie):
query = {
'baseinfo': {
'idmovieimdb': local_movie.get_imdb_id(),
'moviereleasename': local_movie.get_release_name(),
'movieaka': local_movie.get_movie_name(),
'sublanguageid': local_movie.get_language().xxx(),
'subauthorcomment': local_movie.get_comments(),
},
}
if local_movie.is_hearing_impaired() is not None:
query['hearingimpaired'] = local_movie.is_hearing_impaired()
if local_movie.is_high_definition() is not None:
query['highdefinition'] = local_movie.is_high_definition()
if local_movie.is_automatic_translation() is not None:
query['automatictranslation'] = local_movie.is_automatic_translation()
if local_movie.get_subtitle_author() is not None:
query['subtranslator'] = local_movie.get_subtitle_author()
if local_movie.is_foreign_only() is not None:
query['foreignpartsonly'] = local_movie.is_foreign_only()
for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()):
sub_bytes = subtitle.get_filepath().open(mode='rb').read()
sub_tx_data = b64encode(zlib.compress(sub_bytes))
cd = "cd{i}".format(i=i+1)
cd_data = {
'subhash': subtitle.get_md5_hash(),
'subfilename': subtitle.get_filename(),
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None,
'moviefps': str(video.get_fps()) if video.get_fps() else None,
'movieframes': str(video.get_framecount()) if video.get_framecount() else None,
'moviefilename': video.get_filename(),
'subcontent': sub_tx_data,
}
query[cd] = cd_data
def run_query():
return self._xmlrpc_server.UploadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
