class SubtitlesConncector:
RETURNED_OK_STATUS = '200 OK'
def __init__(self,
user_name='',
password='',
language='',
agent='OS Test User Agent'):
self.user_name = user_name
self.password = password
self.language = language
self.agent = agent
self._api_adres = 'http://api.opensubtitles.org/xml-rpc'
self.proxy_server = ServerProxy(self._api_adres)
self.token = self._get_token()
def _is_status_ok(self, status):
if status == SubtitlesConncector.RETURNED_OK_STATUS:
return True
return False
def _get_token(self):
data = self.proxy_server.LogIn(self.user_name, self.password,
self.language, self.agent)
status = StatusParsser(data=data).parse()
if self._is_status_ok(status):
return data['token']
else:
raise StatusError(status)
def _save_zip(self, data_to_zip, path, file_name):
b64img = base64.b64decode(data_to_zip.encode('ascii'))
b64imgfile = open(str(path + "/" + file_name + ".zip"), 'w')
b64imgfile.write(str(b64img))
def get_subtitles(self, subtitle, subtitles_dir=None):
if not subtitles_dir:
subtitles_dir = os.path.expanduser("~") + "/subtitles"
if not os.path.exists(subtitles_dir):
os.makedirs(subtitles_dir)
data = self.proxy_server.DownloadSubtitles(self.token,
[subtitle.idsubtitlefile])
try:
data = data['data'][0]['data']
except Exception:
raise GetSubtitlesError("Error: cant get data")
self._save_zip(data, subtitles_dir, subtitle.subfilename)
def search_subtitles(self, subtitles):
parsed_data_to_send = ParseSubtitlesToSend(subtitles).parse()
data = self.proxy_server.SearchSubtitles(self.token,
[parsed_data_to_send])
data = ParseIncomingData(data).parse()
collect = SubtitlesFindedCollection()
for d in data:
collect.append(
SubtitlesFindedProperties(sublangid=d['SubLanguageID'],
subfilename=d['SubFileName'],
idsubtitlefile=d['IDSubtitleFile']))
return collect
def query_open_subtitles(movie_filename, language):
uri = 'http://api.opensubtitles.org/xml-rpc'
server = ServerProxy(uri, verbose=0, allow_none=True, use_datetime=True)
login_info = server.LogIn('', '', 'en', 'ss v' + __version__)
token = login_info['token']
try:
guessit_query = obtain_guessit_query(movie_filename, language)
search_queries = [
guessit_query,
obtain_movie_hash_query(movie_filename, language),
]
response = server.SearchSubtitles(token, search_queries)
try:
search_results = response['data']
except KeyError: # noqa
raise KeyError('"data" key not found in response: %r' % response)
if search_results:
search_results = filter_bad_results(search_results, guessit_query)
return search_results
finally:
server.LogOut(token)
def get_subtitles(db, l_post_id, l_imdbid):
try:
server = ServerProxy('http://api.opensubtitles.org/xml-rpc')
token = server.LogIn('dh_moviebarcodes', 'dh_ws2015', 'eng', 'Moviebarcode Analyzer')['token']
imdb_id = int(l_imdbid[2:])
search_request = []
search_request.append({'imdbid': imdb_id, 'sublanguageid': 'eng'})
resp = server.SearchSubtitles(token, search_request)
subtitle_id = []
try:
sub = get_best_subtitle(resp['data'])
subtitle_id.append(sub['IDSubtitleFile'])
subtitle_data = server.DownloadSubtitles(token, subtitle_id)
except IndexError:
print("No subtitle available")
return ""
if subtitle_data['status'] == '200 OK':
compressed_data = subtitle_data['data'][0]['data']
decoded_subtitle = base64.b64decode(compressed_data)
# subtitle in byte format
decoded_subtitle = gzip.GzipFile(fileobj=io.BytesIO(decoded_subtitle)).read()
clean_subtitle = remove_invalid_characters(decoded_subtitle)
allLemmasAndMostFrequentLemmas = lemmatize(clean_subtitle)
print(allLemmasAndMostFrequentLemmas[0])
print(allLemmasAndMostFrequentLemmas[1])
update_value_in_db(db, l_post_id, "subtitlesLemmatisation", allLemmasAndMostFrequentLemmas[0])
update_value_in_db(db, l_post_id, "subtitlesMostFrequentWords", allLemmasAndMostFrequentLemmas[1])
except ValueError:
print("No subtitle available")
class OpenSubtitlesClient:
def __init__(self, username, password, language):
self.client = ServerProxy('http://api.opensubtitles.org/xml-rpc',
allow_none=True)
self.token = self.client.LogIn(username, password, language,
"TemporaryUserAgent")['token']
def get_list_subtitle_response(self, query, language):
response = self.client.SearchSubtitles(self.token,
[{
'query': query,
'sublanguageid': language
}], [{
'limit': 3
}])
return response['data']
def download_encrypted_subtitles(self, subtitle_file_id):
response = self.client.DownloadSubtitles(self.token,
[subtitle_file_id])
decompresses_subtitle_file = zlib.decompress(
base64.b64decode(response['data'][0]['data']), 16 + zlib.MAX_WBITS)
return decompresses_subtitle_file
def get_encryption_method(self, encrypted_subtitles):
return chardet.detect(encrypted_subtitles)['encoding']
def decode(self, encryption_method, data):
return data.decode(encryption_method)
class Auth:
def __init__(self):
self.config = Config.get_instance()
self.rpc = ServerProxy(
self.config.get("server_url"), allow_none=True
) # allow_none is mandatory to make some methods work
self.Token = None
self.user_agent = self.config.get("user_agent")
def get_rpc(self):
return self.rpc
def get_user_agent(self):
return self.user_agent
def login(self):
User = self.config.get("username")
Pass = self.config.get("password")
self.logindata = self.rpc.LogIn(User, Pass, "eng", self.user_agent)
if "200 OK" not in self.logindata["status"]:
return False
self.Token = self.logindata["token"]
return self.Token
def get_token(self):
# function to login the user
if self.Token is None:
return self.login()
return self.Token
def logout(self):
# function to logout
self.rpc.LogOut(self.Token)
class OpenSubtitlesApiWrapper():
def __init__(self):
try:
self.auth_token = "-1" ### state to check if initialized properly
self.server_connector = ServerProxy(SUBTITLE_SERVER,
allow_none=True)
auth_response = self.server_connector.LogIn(
USER_NAME, PASSWORD, "en", USER_AGENT_OPEN_SUTBTITLE)
if auth_response['status'] != HTTP_CODE_200:
raise Exception("Auth failed!!!")
self.auth_token = auth_response["token"]
except Exception as e:
LOGGER.exception(str(e))
def pull_subtitles(self, media_source):
movie_vs_id = {}
movie_ids = [] ### to preserve ordering with relevancy
media_file_name = os.path.basename(media_source)
media_dir = media_source.replace(media_file_name, "")
media_name = os.path.splitext(media_file_name)[0]
response = self.server_connector.SearchMoviesOnIMDB(
self.auth_token, media_name)
if len(response['data']) > 0:
for movie in response['data']:
movie_vs_id[movie['id']] = movie['title']
movie_ids.append(movie['id'])
#os.system("autosub " + media_source + " -o " + PROJECT_HOME + "/tmp/tmp.srt")
print(movie_vs_id[movie_ids[0]])
if input(
"Is this your media file for which you are searching subs y/n? "
) == "y":
media_id = movie_ids[0]
else:
count = 0
for movie_id in movie_ids:
print(str(count + 1) + ") " + movie_vs_id[movie_id])
count += 1
choice = int(
input("Please choose between the options [1-" + str(count) +
"] "))
media_id = movie_ids[choice - 1]
print("Pulling subtitles from opensubtitle for " +
movie_vs_id[media_id])
subtitles = self.server_connector.SearchSubtitles(
self.auth_token, [{
"sublanguageid": "eng",
"imdbid": media_id
}])['data']
print(subtitles)
counter = 1
for subtitle in subtitles:
download_link = subtitle['SubDownloadLink']
print("Downloading sub(s) from " + download_link)
downloaded_file = os.path.basename(download_link)
os.system("cd " + media_dir + "; wget " + download_link +
"; gunzip -c " + downloaded_file + " > " + media_name +
"." + str(counter) + ".srt; rm -rf '*.gz'")
counter += 1
class Downloader:
"""
Connects to the OpenSubtitles server and allows for file downloading.
"""
def __init__(self,
user_agent,
server='http://api.opensubtitles.org/xml-rpc',
language='eng'):
self.server = server
self.language = language
trans = Transport()
trans.user_agent = user_agent
self._rpc = ServerProxy(self.server, allow_none=True, transport=trans)
login_response = self._rpc.LogIn('', '', language, user_agent)
assert_status(login_response)
self._token = login_response.get('token')
def download(self, *ids):
"""
Downloads the subtitles with the given ids.
:param ids: The subtitles to download
:return: Result instances
:raises NotOKException
"""
bundles = sublists_of(ids,
20) # 20 files at once is an API restriction
for bundle in bundles:
download_response = self._rpc.DownloadSubtitles(
self._token, bundle)
assert_status(download_response)
download_data = download_response.get('data')
for item in download_data:
subtitle_id = item['idsubtitlefile']
subtitle_data = item['data']
decompressed = decompress(subtitle_data)
yield Result(subtitle_id, decompressed)
def close(self):
"""
Closes the connection.
"""
self._rpc.LogOut(self._token)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
class OpenSubtitles:
def __init__(self):
# noinspection SpellCheckingInspection
opensubtitles_ua = b'VkxzdWIgMC4xMC4y'
self.ua = base64.b64decode(opensubtitles_ua).decode()
self.opensubtitles_api_url = 'http://api.opensubtitles.org/xml-rpc'
self.opensubtitles_lang = 'en'
self.token = None
self.xmlrpc_transport = Transport()
self.xmlrpc_transport.user_agent = self.ua
self.xmlrpc = ServerProxy(self.opensubtitles_api_url,
allow_none=True,
transport=self.xmlrpc_transport)
def retrieve_subtitles(self, ids: List[str]) -> Dict[str, str]:
response = self.xmlrpc.DownloadSubtitles(self.token, ids)
status = response.get('status', '').split()[0]
if status == '200':
subtitles_content = {}
encoded_data = response.get('data')
for item in encoded_data:
subfile_id = item['idsubtitlefile']
try:
decoded_data = decompress(item['data'],
'utf_8_sig') or decompress(
item['data'], 'utf-8')
except UnicodeDecodeError:
decoded_data = decompress(item['data'], 'latin1')
if not decoded_data:
logger.error('An error occurred while decoding subtitle'
'file ID {}.'.format(subfile_id))
else:
subtitles_content[subfile_id] = decoded_data
return subtitles_content
else:
raise RuntimeWarning(
f'Unable to get subtitles, returned status is {status}')
def search_subtitles(self, param: List[dict]) -> List[Dict[str, str]]:
response = self.xmlrpc.SearchSubtitles(self.token, param)
status = response.get('status', '').split()[0]
if status == '200':
return response.get('data')
else:
raise RuntimeWarning(
f'Unable to search subtitles, returned status is {status}')
def login(self, username, password):
response = self.xmlrpc.LogIn(username, password,
self.opensubtitles_lang, self.ua)
status = response.get('status', '').split()[0]
if status == '200':
self.token = response.get('token')
else:
raise RuntimeWarning(
f'Unable to authenticate, returned status is {status}')
def log_in_opensubtitles(self, proxy: ServerProxy) -> str:
"""
Logs in the user to OpenSubtitles. This function should be called always when starting talking with server.
It returns token, which must be used in later communication. If user has no account, blank username and
password should be OK. As language - use ISO639 2 letter code.
:param proxy: ServerProxy.LogIn(username, password, language, useragent)
username: (string) Can be blank since anonymous users are allowed
password: (string) Can be blank since anonymous users are allowed
language: (string) Either HTTP ACCEPT-LANGUAGE header or ISO639 2
useragent: (string) Use your registered useragent, also provide version number - we need tracking
version numbers of your program. If your UA is not registered, you will get error 414 Unknown User Agent
:return: token or error message
Link to request useragent:
http://trac.opensubtitles.org/projects/opensubtitles/wiki/DevReadFirst
"""
try:
self.prompt_label.setText(
"Logging in to OpenSubtitles, please wait ...")
username = "******"
password = "******"
user_agent = "SubcrawlProjectUserAgent"
login = proxy.LogIn(username, password,
self.preference.language_iso3, user_agent)
except Fault:
self.prompt_label.setText(
"There was a fault while logging in to OpenSubtitles. Please try again."
)
return "error"
except ProtocolError:
self.prompt_label.setText(
"There was an error with the server. Please try again later.")
return "error"
except ConnectionResetError or ConnectionError or ConnectionAbortedError or ConnectionRefusedError:
self.prompt_label.setText("Please check your internet connection.")
return "error"
except expat.ExpatError:
# https://stackoverflow.com/questions/3664084/xml-parser-syntax-error
self.prompt_label.setText(
"The received payload is probably incorrect")
return "error"
except gaierror:
self.prompt_label.setText(
"Please check your internet connection and try again")
return "error"
except Exception as e:
self.prompt_label.setText(
"Be sure to send us this error: {}".format(str(e)))
return "error"
else:
if login["status"] == "200 OK":
return login["token"]
else:
return "error"
def login():
transport = Transport()
transport.user_agent = constants.USER_AGENT_OPENSUBS
xmlrpc = ServerProxy(constants.OPENSUBTITLES_URL,
allow_none=True,
transport=transport)
try:
data = xmlrpc.LogIn(constants.USERNAME, constants.PASSWORD,
constants.LANGUAGE, constants.USER_AGENT_OPENSUBS)
except:
logging.warning(
"Error occured while establishing connection to opensubtitles...")
return None, None
if '200' == data.get('status').split()[0]:
logging.info("Got token from opensubtitles")
return data.get('token'), xmlrpc
else:
logging.warning(
"Error occured while getting opensubtitles token. Returned status as "
+ data.get('status').split()[0])
return None
def query_open_subtitles(movie_filename, language):
uri = 'http://api.opensubtitles.org/xml-rpc'
server = ServerProxy(uri, verbose=0, allow_none=True, use_datetime=True)
login_info = server.LogIn('', '', 'en', 'OS Test User Agent')
token = login_info['token']
try:
guessit_query = obtain_guessit_query(movie_filename, language)
search_queries = [
guessit_query,
obtain_movie_hash_query(movie_filename, language),
]
response = server.SearchSubtitles(token, search_queries)
search_results = response['data']
if search_results:
search_results = filter_bad_results(search_results, guessit_query)
return search_results
finally:
server.LogOut(token)
time.sleep(2)
if opt_gui == 'cli' and opt_selection_mode != 'auto':
# Synchronous call
process_videoDispatched = subprocess.call(command)
else:
# Asynchronous call
process_videoDispatched = subprocess.Popen(command)
# ==== Search and download subtitles ===========================================
try:
# ==== Connection to OpenSubtitlesDownload
try:
session = osd_server.LogIn(
osd_username,
hashlib.md5(osd_password[0:32].encode('utf-8')).hexdigest(),
osd_language, 'opensubtitles-download 4.2')
except Exception:
# Retry once after a delay (could just be a momentary overloaded server?)
time.sleep(3)
try:
session = osd_server.LogIn(osd_username, osd_password,
osd_language,
'opensubtitles-download 4.2')
except Exception:
superPrint("error", "Connection error!", "Unable to reach OpenSubtitles.org servers!\n\nPlease check:\n" + \
"- Your Internet connection status\n" + \
"- www.opensubtitles.org availability\n" + \
"The subtitles search and download service is powered by <a href=\"https://opensubtitles.org\">opensubtitles.org</a>.\n" + \
"Be sure to donate if you appreciate the service provided!")
sys.exit(2)
class OpenSubtitles(SubtitleProvider):
URL = 'http://api.opensubtitles.org/xml-rpc'
def __init__(self):
SubtitleProvider.__init__(self)
self._xmlrpc = None
self._token = None
self._settings = OpenSubtitlesSettings()
def get_settings(self):
return self._settings
def set_settings(self, settings):
self._settings = settings
def connect(self):
log.debug('connect()')
if self.connected():
return
self._xmlrpc = ServerProxy(self.URL, allow_none=False)
def disconnect(self):
log.debug('disconnect()')
if self.logged_in():
self.logout()
if self.connected():
self._xmlrpc = None
def connected(self):
return self._xmlrpc is not None
def login(self):
log.debug('login()')
if self.logged_in():
return
if not self.connected():
raise ProviderNotConnectedError()
def login_query():
# FIXME: 'en' language ok??? or '' as in the original
return self._xmlrpc.LogIn(str(self._settings.username),
str(self._settings.password), 'en',
str(self._settings.get_user_agent()))
result = self._safe_exec(login_query, None)
self.check_result(result)
self._token = result['token']
def logout(self):
log.debug('logout()')
if self.logged_in():
def logout_query():
return self._xmlrpc.LogOut(self._token)
result = self._safe_exec(logout_query, None)
self.check_result(result)
self._token = None
def logged_in(self):
return self._token is not None
SEARCH_LIMIT = 500
def search_videos(self, videos, callback, languages=None):
log.debug('search_videos(#videos={})'.format(len(videos)))
if not self.logged_in():
raise ProviderNotConnectedError()
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(
window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc.SearchSubtitles(
self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(
remote_language_raw, xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(
rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitlesSubtitleFile(
filename=remote_filename,
file_size=remote_file_size,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
age=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
imdb_identity = ImdbIdentity(imdb_id=imdb_id,
imdb_rating=None)
identity = ProviderIdentities(imdb_identity=imdb_identity,
provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception(
'Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(
queries=queries))
log.error('Offending result is: {remote_sub}'.format(
remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def query_text(self, query):
return OpenSubtitlesTextQuery(query=query)
def download_subtitles(self, os_rsubs):
log.debug('download_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(
window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({
item['idsubtitlefile']: item['data']
for item in result['data']
})
subtitles = [
unzip_bytes(base64.b64decode(
map_id_data[os_rsub.get_id_online()])).read()
for os_rsub in os_rsubs
]
return subtitles
def ping(self):
log.debug('ping()')
if not self.logged_in():
raise ProviderNotConnectedError()
def run_query():
return self._xmlrpc.NoOperation(self._token)
result = self._safe_exec(run_query, None)
self.check_result(result)
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
@classmethod
def get_name(cls):
return 'opensubtitles'
@classmethod
def get_short_name(cls):
return 'os'
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
try:
result = query()
return result
except (ProtocolError, CannotSendRequest, SocketError):
self._signal_connection_failed()
log.warning('Query failed', exc_info=sys.exc_info())
return default
STATUS_CODE_RE = re.compile('(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise ProviderConnectionError(_('No message'))
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(
status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(
_('Server returned status="{status}". Expected "200 OK".').
format(status=data['status']), data['status'])
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(message, code)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')
class SubtitleDownload:
'''Traverses a directory and all subdirectories and downloads subtitles.
Relies on an undocumented OpenSubtitles feature where subtitles seems to be
sorted by filehash and popularity.
'''
api_url = 'http://api.opensubtitles.org/xml-rpc'
login_token = None
server = None
moviefiles = []
movie_exts = (".avi", ".mkv", ".mp4")
subb_exts = (".srt", ".sub", ".mpl")
def __init__(self, movie_path, lang = "en"):
print("OpenSubtitles Subtitle Downloader".center(78))
print("===================================".center(78))
self.server = ServerProxy(self.api_url, verbose=False)
self.lang_id = lang
# Traverse the directory tree and select all movie files
for root, _, files in os.walk(movie_path):
for file in files:
if self.is_movie(file):
file_path = os.path.join(root, file)
if not self.subtitles_already_present(file_path):
print("Found: " + file)
filehash = self.hashFile(file_path)
filesize = os.path.getsize(file_path)
self.moviefiles.append({'dir': root,
'file': file,
'hash': filehash,
'size': filesize,
'subtitleid': None})
try:
print("Login...")
self.login()
print("Searching for subtitles...")
self.search_subtitles()
print("Logout...")
self.logout()
except Error as e:
print("XML-RPC error:", e)
except UserWarning as uw:
print(uw)
def login(self):
'''Log in to OpenSubtitles'''
resp = self.server.LogIn('', '', 'en', 'OS Test User Agent')
self.check_status(resp)
self.login_token = resp['token']
def logout(self):
'''Log out from OpenSubtitles'''
resp = self.server.LogOut(self.login_token)
self.check_status(resp)
def search_subtitles(self):
'''Search OpenSubtitles for matching subtitles'''
search = []
for movie in self.moviefiles:
search.append({'sublanguageid': self.lang_id,
'moviehash': movie['hash'],
'moviebytesize': str(movie['size'])})
# fixes weird problem where subs aren't found when only searching
# for one movie.
if len(search) == 1:
search.append(search[0])
resp = self.server.SearchSubtitles(self.login_token, search)
self.check_status(resp)
if resp['data'] == False:
print("No subtitles found")
return
subtitles = []
for result in resp['data']:
if int(result['SubBad']) != 1:
subtitles.append({'subid': result['IDSubtitleFile'],
'hash': result['MovieHash']})
downloadable_subs = subtitles[:]
hash = None
for s in subtitles:
if hash == s['hash']:
downloadable_subs.remove(s)
hash = s['hash']
for ds in downloadable_subs:
sub = self.download_subtitles(ds)
for movie in self.moviefiles:
if movie['hash'] == ds['hash']:
print("Saving subtitle for: " + movie['file'])
filename = os.path.join(movie['dir'], os.path.splitext(movie['file'])[0] + ".srt")
file = open(filename, "wb")
file.write(sub)
file.close()
def download_subtitles(self, subtitle):
resp = self.server.DownloadSubtitles(self.login_token, [subtitle['subid']])
self.check_status(resp)
decoded = base64.standard_b64decode(resp['data'][0]['data'].encode('ascii'))
decompressed = zlib.decompress(decoded, 15 + 32)
return decompressed
def is_movie(self, file):
return os.path.splitext(file.lower())[1] in self.movie_exts
def subtitles_already_present(self, file):
file_base = os.path.splitext(file.lower())[0]
for ext in self.subb_exts:
if os.path.exists(file_base + ext):
return True
return False
def check_status(self, resp):
'''Check the return status of the request.
Anything other than "200 OK" raises a UserWarning
'''
if resp['status'].upper() != '200 OK':
raise UserWarning("Response error from " + self.api_url + ". Response status was: " + resp['status'])
def hashFile(self, name):
'''Calculates the hash value of a movie.
Copied from OpenSubtitles own examples:
http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
'''
try:
longlongformat = 'q' # long long
bytesize = struct.calcsize(longlongformat)
f = open(name, "rb")
filesize = os.path.getsize(name)
hash = filesize
if filesize < 65536 * 2:
return "SizeError"
for x in range(65536//bytesize):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
f.seek(max(0,filesize-65536),0)
for x in range(65536//bytesize):
buffer = f.read(bytesize)
(l_value,)= struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF
f.close()
returnedhash = "%016x" % hash
return returnedhash
except(IOError):
return "IOError"
if gui == 'cli' and opt_selection_mode == 'manual':
# Synchronous call
process_videoDispatched = subprocess.call(command_splitted)
else:
# Asynchronous call
process_videoDispatched = subprocess.Popen(command_splitted)
# Do not spawn too many instances at the same time
time.sleep(0.33)
# ==== Search and download subtitles
try:
try:
# Connection to opensubtitles.org server
session = server.LogIn('', '', 'en', 'opensubtitles-download 3.2')
except Exception:
# Retry once, it could be a momentary overloaded server?
time.sleep(3)
try:
# Connection to opensubtitles.org server
session = server.LogIn('', '', 'en', 'opensubtitles-download 3.2')
except Exception:
# Failed connection attempts?
superPrint(
"error", "Connection error!",
"Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your 200 downloads per 24h limit\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!"
)
sys.exit(1)
# Connection refused?
# Asynchronous dispatch
process_videoDispatched = subprocess.Popen(command_splitted)
# Do not spawn too many instances at the same time
time.sleep(0.33)
# The first video file will be processed by this instance
videoPathList.clear()
videoPathList.append(videoPath)
# ==== Search and download subtitles ===========================================
try:
# ==== Connection
try:
session = osd_server.LogIn(osd_username, osd_password, "en",
'opensubtitles-download 5.0')
except Exception:
# Retry once, it could be a momentary overloaded server?
time.sleep(3)
try:
session = osd_server.LogIn(osd_username, osd_password, "en",
'opensubtitles-download 5.0')
except Exception:
superPrint(
"error", "Connection error!",
"Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!"
)
sys.exit(2)
# Connection refused?
if session['status'] != '200 OK':
class SDService(object):
"""
Contains the class that represents the OSDB RPC Server.
Encapsules all the XMLRPC methods.
Consult the OSDB API methods at http://trac.opensubtitles.org/projects/opensubtitles/wiki/XMLRPC
If it fails to connect directly to the XMLRPC server, it will try to do so through a default proxy.
Default proxy uses a form to set which URL to open. We will try to change this in later stage.
"""
def __init__(self, server=None, proxy=None):
self.log = logging.getLogger("subdownloader.SDService.SDService")
self.log.debug(
"Creating Server with server = %s and proxy = %r" % (server, proxy))
self.timeout = 30
self.user_agent = USER_AGENT
self.language = ''
if server:
self.server = server
else:
self.server = DEFAULT_OSDB_SERVER
self.proxy = proxy
self.logged_as = None
self._xmlrpc_server = None
self._token = None
def connected(self):
return self._token is not None
def connect(self):
server = self.server
proxy = self.proxy
self.log.debug("connect()... to server %s with proxy %s" % (server, proxy))
connect_res = False
try:
self.log.debug(
"Connecting with parameters (%r, %r)" % (server, proxy))
connect = TimeoutFunction(self._connect)
connect_res = connect(server, proxy)
except TimeoutFunctionException as e:
self.log.error("Connection timed out. Maybe you need a proxy.")
raise
except:
self.log.exception("connect: Unexpected error")
raise
finally:
self.log.debug("connection connected %s" % connect_res)
return connect_res
def _connect(self, server, proxy):
try:
if proxy:
self.log.debug("Trying proxied connection... ({})".format(proxy))
self.proxied_transport = ProxiedTransport()
self.proxied_transport.set_proxy(proxy)
self._xmlrpc_server = ServerProxy(
server, transport=self.proxied_transport, allow_none=True)
# self.ServerInfo()
self.log.debug("...connected")
return True
elif test_connection(TEST_URL):
self.log.debug("Trying direct connection...")
self._xmlrpc_server = ServerProxy(
server, allow_none=True)
# self.ServerInfo()
self.log.debug("...connected")
return True
else:
self.log.debug("...failed")
self.log.error("Unable to connect. Try setting a proxy.")
return False
except ProtocolError as e:
self._connection_failed()
self.log.debug("error in HTTP/HTTPS transport layer")
raise
except Fault as e:
self.log.debug("error in xml-rpc server")
raise
except:
self.log.exception("Connection to the server failed/other error")
raise
def logged_in(self):
return self._token is not None
def login(self, username="", password=""):
try:
login = TimeoutFunction(self._login)
return login(username, password)
except TimeoutFunctionException:
self.log.error("login timed out")
except:
self.log.exception("login: other issue")
raise
def _login(self, username="", password=""):
"""Login to the Server using username/password,
empty parameters means an anonymously login
Returns True if login sucessful, and False if not.
"""
self.log.debug("----------------")
self.log.debug("Logging in (username: %s)..." % username)
def run_query():
return self._xmlrpc_server.LogIn(
username, password, self.language, self.user_agent)
info = self._safe_exec(run_query, None)
if info is None:
self._token = None
return False
self.log.debug("Login ended in %s with status: %s" %
(info['seconds'], info['status']))
if info['status'] == "200 OK":
self.log.debug("Session ID: %s" % info['token'])
self.log.debug("----------------")
self._token = info['token']
return True
else:
# force token reset
self.log.debug("----------------")
self._token = None
return False
def logout(self):
try:
logout = TimeoutFunction(self._logout)
result = logout()
self._token = None
return result
except TimeoutFunctionException:
self.log.error("logout timed out")
def _logout(self):
"""Logout from current session(token)
This functions doesn't return any boolean value, since it can 'fail' for anonymous logins
"""
self.log.debug("Logging out from session ID: %s" % self._token)
try:
info = self._xmlrpc_server.LogOut(self._token)
self.log.debug("Logout ended in %s with status: %s" %
(info['seconds'], info['status']))
except ProtocolError as e:
self.log.debug("error in HTTP/HTTPS transport layer")
raise
except Fault as e:
self.log.debug("error in xml-rpc server")
raise
except:
self.log.exception("Connection to the server failed/other error")
raise
finally:
# force token reset
self._token = None
STATUS_CODE_RE = re.compile('(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise ProviderConnectionError(None, 'No message')
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(None, 'Server returned status="{status}". Expected "200 OK".'.format(
status=data['status']))
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise ProviderConnectionError(code, message)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')
@classmethod
def name(cls):
return "opensubtitles"
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
try:
result = query()
return result
except (ProtocolError, xml.parsers.expat.ExpatError):
self._signal_connection_failed()
log.debug("Query failed", exc_info=sys.exc_info())
return default
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
def imdb_query(self, query):
if not self.connected():
return None
def run_query():
return self._xmlrpc_server.SearchMoviesOnIMDB(self._token, query)
result = self._safe_exec(run_query, None)
if result is None:
return None
provider_identities = []
for imdb_data in result['data']:
if not imdb_data:
continue
imdb_identity = ImdbIdentity(imdb_id=imdb_data['id'], imdb_rating=None)
video_identity = VideoIdentity(name=imdb_data['title'], year=None)
provider_identities.append(ProviderIdentities(
video_identity=video_identity, imdb_identity=imdb_identity,
provider=self))
return provider_identities
SEARCH_LIMIT = 500
def search_text(self, text, languages=None):
lang_str = self._languages_to_str(languages)
query = {
'sublanguageid': lang_str,
'query': str(text),
}
queries = [query]
def run_query():
return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT})
self._safe_exec(run_query, None)
def search_videos(self, videos, callback, languages=None):
if not self.connected():
return None
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc_server.SearchSubtitles(self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(remote_language_raw,
xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitles_SubtitleFile(
filename=remote_filename,
file_size=remote_file_size ,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
age=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None)
identity = ProviderIdentities(imdb_identity=imdb_identity, provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception('Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(queries=queries))
log.error('Offending result is: {remote_sub}'.format(remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def _video_info_to_identification(self, video_info):
name = video_info['MovieName']
year = int(video_info['MovieYear'])
imdb_id = video_info['MovieImdbID']
video_identity = VideoIdentity(name=name, year=year)
imdb_identity = ImdbIdentity(imdb_id=imdb_id, imdb_rating=None)
episode_identity = None
movie_kind = video_info['MovieKind']
if movie_kind == 'episode':
season = int(video_info['SeriesSeason'])
episode = int(video_info['SeriesEpisode'])
episode_identity = EpisodeIdentity(season=season, episode=episode)
elif movie_kind == 'movie':
pass
else:
log.warning('Unknown MoviesKind="{}"'.format(video_info['MovieKind']))
return ProviderIdentities(video_identity=video_identity, episode_identity=episode_identity,
imdb_identity=imdb_identity, provider=self)
def identify_videos(self, videos):
if not self.connected():
return
for part_videos in window_iterator(videos, 200):
hashes = [video.get_osdb_hash() for video in part_videos]
hash_video = {hash: video for hash, video in zip(hashes, part_videos)}
def run_query():
return self._xmlrpc_server.CheckMovieHash2(self._token, hashes)
result = self._safe_exec(run_query, None)
self.check_result(result)
for video_hash, video_info in result['data'].items():
identification = self._video_info_to_identification(video_info[0])
video = hash_video[video_hash]
video.add_identity(identification)
def download_subtitles(self, os_rsubs):
if not self.connected():
return None
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc_server.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({item['idsubtitlefile']: item['data'] for item in result['data']})
subtitles = [unzip_bytes(base64.b64decode(map_id_data[os_rsub.get_id_online()])).read() for os_rsub in os_rsubs]
return subtitles
def can_upload_subtitles(self, local_movie):
if not self.connected():
return False
query = {}
for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()):
# sub_bytes = open(subtitle.get_filepath(), mode='rb').read()
# sub_tx_data = b64encode(zlib.compress(sub_bytes))
cd = "cd{i}".format(i=i+1)
cd_data = {
'subhash': subtitle.get_md5_hash(),
'subfilename': subtitle.get_filename(),
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
'movietimems': str(video.get_time_ms()),
'moviefps': video.get_fps(),
'movieframes': str(video.get_framecount()),
'moviefilename': video.get_filename(),
}
query[cd] = cd_data
def run_query():
return self._xmlrpc_server.TryUploadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
movie_already_in_db = int(result['alreadyindb']) != 0
if movie_already_in_db:
return False
return True
def upload_subtitles(self, local_movie):
query = {
'baseinfo': {
'idmovieimdb': local_movie.get_imdb_id(),
'moviereleasename': local_movie.get_release_name(),
'movieaka': local_movie.get_movie_name(),
'sublanguageid': local_movie.get_language().xxx(),
'subauthorcomment': local_movie.get_comments(),
},
}
if local_movie.is_hearing_impaired() is not None:
query['hearingimpaired'] = local_movie.is_hearing_impaired()
if local_movie.is_high_definition() is not None:
query['highdefinition'] = local_movie.is_high_definition()
if local_movie.is_automatic_translation() is not None:
query['automatictranslation'] = local_movie.is_automatic_translation()
if local_movie.get_subtitle_author() is not None:
query['subtranslator'] = local_movie.get_subtitle_author()
if local_movie.is_foreign_only() is not None:
query['foreignpartsonly'] = local_movie.is_foreign_only()
for i, (video, subtitle) in enumerate(local_movie.iter_video_subtitle()):
sub_bytes = subtitle.get_filepath().open(mode='rb').read()
sub_tx_data = b64encode(zlib.compress(sub_bytes))
cd = "cd{i}".format(i=i+1)
cd_data = {
'subhash': subtitle.get_md5_hash(),
'subfilename': subtitle.get_filename(),
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
'movietimems': str(video.get_time_ms()) if video.get_time_ms() else None,
'moviefps': str(video.get_fps()) if video.get_fps() else None,
'movieframes': str(video.get_framecount()) if video.get_framecount() else None,
'moviefilename': video.get_filename(),
'subcontent': sub_tx_data,
}
query[cd] = cd_data
def run_query():
return self._xmlrpc_server.UploadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
class OpenSub(object):
def __init__(self, path):
self.TAGS = [
'bluray', 'cam', 'dvb', 'dvd', 'hd-dvd', 'hdtv', 'ppv', 'telecine',
'telesync', 'tv', 'vhs', 'vod', 'web-dl', 'webrip', 'workprint'
]
self.OPENSUBTITLES_SERVER = 'https://api.opensubtitles.org:443/xml-rpc'
self.USER_AGENT = 'TemporaryUserAgent' # needs to be changed from time to time to the recent temporary UA
self.xmlrpc = ServerProxy(self.OPENSUBTITLES_SERVER, allow_none=True)
self.language = 'eng'
self.token = None
self.mv_name = None
self.path = path
try:
with open("opensubtitles_userpass.txt", 'r') as f:
f = f.read()
user = f.split('\n')[0].split('=')[1]
password = f.split('\n')[1].split('=')[1]
self.user = user
self.passw = password
except FileNotFoundError:
print(
"Please fill OpenSubtitles credentials in opensubtitles_userpass.txt"
)
if self.login(self.user, self.passw):
print("[OpenSubtitles] Login Successful")
else:
print("[OpenSubtitles] Login Failed")
def _get_from_data_or_none(self, key):
'''Return the key recieved from data if the status is 200,
otherwise return None.
'''
status = self.data.get('status').split()[0]
return self.data.get(key) if '200' == status else None
def login(self, username, password):
'''Returns token is login is ok, otherwise None.
'''
self.data = self.xmlrpc.LogIn(username, password, self.language,
self.USER_AGENT)
token = self._get_from_data_or_none('token')
if token:
self.token = token
return token
def logout(self):
'''Returns True if logout is ok, otherwise None.
'''
data = self.xmlrpc.LogOut(self.token)
return '200' in data.get('status')
def search_subtitles(self, params):
'''Returns a list with the subtitles info.
'''
self.data = self.xmlrpc.SearchSubtitles(self.token, params)
return self._get_from_data_or_none('data')
def download_subtitles(self, payload):
"""
:return:
"""
print("Attempting to download subtitles...")
search_result = self.search_subtitles([payload])
if search_result:
dllink = self.analyse_result(search_result, payload)
gzfile = request.urlopen(dllink)
try:
with gzip.open(gzfile, 'rb') as f:
with open(self.path + "\\" + self.mv_name + '.srt',
'wb') as sub_file:
sub_file.write(f.read())
print("[OpenSubtitles] Subtitles downloaded for: " +
self.mv_name)
# Update stats
with open("metadata.json", "r") as jsonFile:
data = json.load(jsonFile)
data['num_of_movies_in_db'] = int(
data['num_of_movies_in_db']) + 1
data['latest_movie_added']['name'] = self.mv_name
data['latest_movie_added']['time'] = str(
datetime.datetime.now())
with open("metadata.json", "w") as jsonFile:
json.dump(data, jsonFile, indent=4)
# End Update
return os.path.join(self.path, self.mv_name + ".srt")
except PermissionError:
print(
"[OpenSubtitles] Permission Error: when creating subtitles for {}"
.format(self.mv_name))
def analyse_result(self, result, payload):
"""
:param result: Search result to find appropriate subtitles
:return: Download Link of best match for subtitles
"""
score = 0
dllink = None
for record in result:
if record.get("MovieName").lower() == payload['query'].lower(
) and record.get("SubLanguageID") == 'eng':
self.mv_name = record.get("MovieName")
dllink = record.get('SubDownloadLink')
break
if record.get('Score',
0) > score and record.get("SubLanguageID") == 'eng':
self.mv_name = record.get("MovieName")
score = record.get('Score', 0)
dllink = record.get('SubDownloadLink')
return dllink
def create_payload(self, mv_name):
"""
:param lang: Subtitle Language
:param mv_name: Movie name to search
:return: Payload containing data about file
"""
payload = {}
payload['query'] = mv_name
payload['SubLanguageID'] = self.language
payload['SubFormat'] = 'srt'
return payload
def search_and_download(self, mv_name):
movie_db = [
os.path.splitext(name)[0] for name in os.listdir(self.path)
]
for local in movie_db:
if mv_name.lower() == local.lower():
return os.path.join(self.path, mv_name + ".srt")
try:
return self.download_subtitles(self.create_payload(mv_name))
except:
print("[OpenSubtitles] Could not download subtitles for " +
mv_name)
def download_top_1000(self):
with open(os.path.join(os.getcwd(), "top1000.txt"), "r") as f:
f = f.read()
i = 1
for mv_name in f.split("\n"):
try:
self.search_and_download(mv_name)
except:
e = sys.exc_info()[0]
print("ERROR: " + str(e))
print("Error: subtitles not downloaded for: " + mv_name)
i = i + 1
if i % 39 == 0:
time.sleep(11)
class Agent():
""" opensubtitle 的用户代理"""
# __username=""
# __password=""
# __languae=""
# __osd_server=""
# __session=""
opt_selection_mode = 'default'
opt_search_mode = "filename"
opt_language_separator = '_'
def __init__(self, username, password, language="en"):
self.__username = username
self.__password = password
self.__languae = language
self.__osd_server = ServerProxy('http://api.opensubtitles.org/xml-rpc')
def LogIn(self):
try:
# ==== Connection
try:
self.__session = self.__osd_server.LogIn(
self.__username, self.__password, self.__languae,
'opensubtitles-download 4.0')
except Exception:
# Retry once, it could be a momentary overloaded server?
time.sleep(3)
try:
self.__session = self.__osd_server.LogIn(
self.__username, self.__password, self.__languae,
'opensubtitles-download 4.0')
except Exception:
print("error1")
#superPrint("error", "Connection error!", "Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!")
sys.exit(2)
# Connection refused?
if self.__session['status'] != '200 OK':
print("error2")
#superPrint("error", "Connection error!", "Opensubtitles.org servers refused the connection: " + session['status'] + ".\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!")
sys.exit(2)
print("login success!")
except:
print("login error3")
sys.exit(2)
def LogOut(self):
if self.__session and self.__session['token']:
self.__osd_server.LogOut(self.__session['token'])
print("logout success")
def __search(self, title):
""" 查询服务器，寻找潜在的字幕"""
searchList = []
searchList.append({'sublanguageid': 'eng', 'query': title})
try:
subtitlesList = self.__osd_server.SearchSubtitles(
self.__session['token'], searchList)
return subtitlesList
# time.sleep(1)
except Exception:
# Retry once, we are already connected, the server maybe momentary overloaded
time.sleep(3)
try:
subtitlesList = self.__osd_server.SearchSubtitles(
self.__session['token'], searchList)
return subtitlesList
except Exception:
print("Search error!")
#superPrint("error", "Search error!", "Unable to reach opensubtitles.org servers!\n<b>Search error</b>")
return -1
def __selectionAuto(self, subtitlesList, year, imdb_ID):
""" 从众多字幕中寻找合适的字幕 """
"""Automatic subtitles selection using filename match"""
# 需要用到的属性
# IDMovieImdb the most import
# MovieKind
# MovieYear
# SubDownloadLink
# SubFileName
# SubFormat
# 顺序选择即可。
# if len(subtitlesList['data'])==1:
# return 0
data = subtitlesList['data']
for i in range(len(data)):
this = data[i]
if this["IDMovieImdb"] != imdb_ID:
continue
# if this["SubFormat"]=="srt":
# return i
if this["MovieKind"] != "movie":
continue
if this["SubFormat"] == "srt" and abs(year -
int(this["MovieYear"])) < 2:
return i
return -1
def __down(self, subtitlesSelected, subtitlesList, subPath):
# 此处变量与下载有直接关系
subLangId = self.opt_language_separator + subtitlesList['data'][
subtitlesSelected]['ISO639']
subLangName = subtitlesList['data'][subtitlesSelected]['LanguageName']
subURL = subtitlesList['data'][subtitlesSelected]['SubDownloadLink']
# subPath 为存储路径
# subPath="1.txt"
#subPath = videoPath.rsplit('.', 1)[0] + '.' + subtitlesList['data'][subIndex]['SubFormat']
if sys.version_info >= (3, 0):
tmpFile1, headers = urllib.request.urlretrieve(subURL)
tmpFile2 = gzip.GzipFile(tmpFile1)
byteswritten = open(subPath, 'wb').write(tmpFile2.read())
if byteswritten > 0:
process_subtitlesDownload = 0
else:
process_subtitlesDownload = 1
else: # python 2
tmpFile1, headers = urllib.urlretrieve(subURL)
tmpFile2 = gzip.GzipFile(tmpFile1)
open(subPath, 'wb').write(tmpFile2.read())
process_subtitlesDownload = 0
# If an error occurs, say so
if process_subtitlesDownload != 0:
print("Subtitling error!")
#superPrint("error", "Subtitling error!", "An error occurred while downloading or writing <b>" + subtitlesList['data'][subIndex]['LanguageName'] + "</b> subtitles for <b>" + videoTitle + "</b>.")
self.__osd_server.LogOut(self.__session['token'])
sys.exit(2)
def __result(self, subtitlesList, year, imdb_ID, subPath):
""" 主体函数 """
# Parse the results of the XML-RPC query
if ('data' in subtitlesList) and (len(subtitlesList['data']) > 0):
# print("find it")
#找到数据
# Mark search as successful
subtitlesSelected = ''
# If there is only one subtitles (matched by file hash), auto-select it (except in CLI mode)
if (len(subtitlesList['data'])
== 1) and (subtitlesList['data'][0]['MatchedBy']
== 'moviehash'):
if opt_selection_mode != 'manual':
# subtitlesSelected = subtitlesList['data'][0]['SubFileName']
subtitlesSelected = 0
# Get video title
videoTitle = subtitlesList['data'][0]['MovieName']
# If there is more than one subtitles and opt_selection_mode != 'auto',
# then let the user decide which one will be downloaded
if subtitlesSelected == '':
subtitlesSelected = self.__selectionAuto(
subtitlesList, year, imdb_ID)
if subtitlesSelected == -1:
return -1
# If a subtitles has been selected at this point, download it!
if subtitlesSelected == -1:
# 记录未下载的电影名与ID
#print("find error!")
return -1
else:
# print("ready to down")
self.__down(subtitlesSelected, subtitlesList, subPath)
return 0
else:
# print("not find it!")
return -1
def work(self, title, year, imdb_ID, path):
""" 查找电影字幕并下载到指定路径中"""
subtitlesList = self.__search(title)
time.sleep(2)
if subtitlesList == -1:
return -1
return self.__result(subtitlesList, year, imdb_ID, path)
pass
def work_by_imda(self, year, imdb_ID):
list = self.__search(imdb_ID)
if list == -1:
return -1
return self.__selectionAuto(list, year, imdb_ID)
pass
pass
class OpenSubtitles(QtCore.QThread):
api_url = 'http://api.opensubtitles.org/xml-rpc'
login_token = None
server = None
def __init__(self, parent=None):
QtCore.QThread.__init__(self, parent)
self.stopping = False
def process(self, files_list, lang='English'):
self.moviefiles = files_list
self.imdbid_to_hash = {}
self.lang = LANGUAGES[lang][1]
if not self.stopping:
self.start()
def __del__(self):
if self.login_token:
self.logout()
def stopTask(self):
self.stopping = True
def run(self):
utils.communicator.updategui.emit('Querying OpenSubtitles.org...', 'info'
)
try:
if not self.login_token:
self.login()
self.search_subtitles()
except utils.NoInternetConnectionFound:
utils.communicator.updategui.emit('No active Internet connection found. Kindly check and try again.',
'error')
def login(self):
'''Log in to OpenSubtitles.org'''
self.server = ServerProxy(self.api_url, verbose=False)
utils.communicator.updategui.emit('Logging in to OpenSubtitles.org...', 'info'
)
try:
resp = self.server.LogIn('', '', 'en', 'PySubD v2.0')
self.check_status(resp)
self.login_token = resp['token']
except Exception as e:
if e.args[0] == 11004:
utils.communicator.updategui.emit("No Internet Connection Found", 'error')
else:
utils.communicator.updategui.emit(str(e), 'error')
def logout(self):
'''Log out from OpenSubtitles'''
utils.communicator.updategui.emit('Logging out...', 'info')
resp = self.server.LogOut(self.login_token)
self.check_status(resp)
def check_status(self, resp):
'''Check the return status of the request.
Anything other than "200 OK" raises a UserWarning
'''
if resp['status'].upper() != '200 OK':
raise utils.IncorrectResponseRecieved('Response error from %s. Response status: %s'
% (self.api_url, resp['status']))
def _query_opensubs(self, search):
results = []
while search[:500]:
if not self.stopping:
try:
tempresp = self.server.SearchSubtitles(self.login_token, search[:500])
except Exception as e:
if e.args[0] == 11004:
utils.communicator.updategui.emit("No Internet Connection Found", 'error')
else:
utils.communicator.updategui.emit(str(e), 'error')
return results
if tempresp['data']:
results.extend(tempresp['data'])
self.check_status(tempresp)
search = search[500:]
else:
return []
logger.debug('Results: %s' % results)
return results
def clean_results(self, results, imdb=False):
subtitles = {}
user_ranks = { 'administrator': 1,
'platinum member': 2,
'vip member': 3,
'gold member': 4,
'trusted': 5,
'silver member': 6,
'bronze member': 7,
'sub leecher': 8,
'': 9, }
for result in results:
if result['SubBad'] != '1':
movie_hash = result.get('MovieHash')
if not movie_hash:
movie_hash = self.imdbid_to_hash[int(result['IDMovieImdb'])]
subid = result['IDSubtitleFile']
downcount = int(result['SubDownloadsCnt'])
rating = float(result['SubRating'])
if rating and rating < 8:
# Ignore poorly rated subtitles, while not
# penalizing the ones that haven't yet been rated
continue
user_rank = user_ranks[result['UserRank']]
if imdb:
cleaned_release_name = utils.clean_name(result['MovieReleaseName'])
file_name = self.moviefiles[movie_hash]['file_name']
cleaned_file_name = utils.clean_name(file_name)
overlap = len(set.intersection(set(cleaned_release_name), set(cleaned_file_name)))
else:
overlap = 0
subtitles.setdefault(movie_hash, []).append({
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank,
'overlap' : overlap
})
return subtitles
def search_subtitles(self):
search = []
for video_file_details in self.moviefiles.itervalues():
video_file_details['sublanguageid'] = self.lang
search.append(video_file_details)
results = self._query_opensubs(search)
subtitles = self.clean_results(results)
for (hash, found_matches) in subtitles.iteritems():
subtitles[hash] = utils.multikeysort(found_matches,
['overlap', 'user_rank', '-rating', '-downcount'])[0]
for (hash, filedetails) in self.moviefiles.iteritems():
if not self.stopping:
if subtitles.get(hash):
utils.communicator.updategui.emit('Saving subtitles for %s'%filedetails['file_name'], 'success')
subtitle = \
self.download_subtitles([subtitles[hash]['subid'
]])
utils.communicator.downloaded_sub.emit()
utils.save_subs(subtitle, filedetails['save_subs_to'],
subtitles[hash])
else:
utils.communicator.no_sub_found.emit(filedetails['file_name'])
else:
return
def download_subtitles(self, subparam):
resp = self.server.DownloadSubtitles(self.login_token, subparam)
self.check_status(resp)
decoded = base64.standard_b64decode(resp['data'][0]['data'
].encode('ascii'))
decompressed = zlib.decompress(decoded, 15 + 32)
return decompressed
class OpenSubtitles(SubtitleProvider):
URL = 'http://api.opensubtitles.org/xml-rpc'
def __init__(self, settings=None):
SubtitleProvider.__init__(self)
self._xmlrpc = None
self._token = None
self._last_time = None
if settings is None:
settings = OpenSubtitlesSettings()
self._settings = settings
def get_settings(self):
return self._settings
def set_settings(self, settings):
if self.connected():
raise RuntimeError(
'Cannot set settings while connected') # FIXME: change error
self._settings = settings
def connect(self):
log.debug('connect()')
if self.connected():
return
self._xmlrpc = ServerProxy(self.URL, allow_none=False)
self._last_time = time.time()
def disconnect(self):
log.debug('disconnect()')
if self.logged_in():
self.logout()
if self.connected():
self._xmlrpc = None
def connected(self):
return self._xmlrpc is not None
def login(self):
log.debug('login()')
if self.logged_in():
return
if not self.connected():
self.connect()
def login_query():
# FIXME: 'en' language ok??? or '' as in the original
return self._xmlrpc.LogIn(str(self._settings.username),
str(self._settings.password), 'en',
str(self._settings.get_user_agent()))
result = self._safe_exec(login_query, None)
self.check_result(result)
self._token = result['token']
def logout(self):
log.debug('logout()')
if self.logged_in():
def logout_query():
return self._xmlrpc.LogOut(self._token)
# Do no check result of this call. Assume connection closed.
self._safe_exec(logout_query, None)
self._token = None
def logged_in(self):
return self._token is not None
def reestablish(self):
log.debug('reestablish()')
connected = self.connected()
logged_in = self.logged_in()
self.disconnect()
if connected:
self.connect()
if logged_in:
self.login()
_TIMEOUT_MS = 60000
def _ensure_connection(self):
now = time.time()
if now - time.time() > self._TIMEOUT_MS:
self.reestablish()
self._last_time = now
SEARCH_LIMIT = 500
def search_videos(self, videos, callback, languages=None):
log.debug('search_videos(#videos={})'.format(len(videos)))
if not self.logged_in():
raise ProviderNotConnectedError()
lang_str = self._languages_to_str(languages)
window_size = 5
callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)
remote_subtitles = []
for window_i, video_window in enumerate(
window_iterator(videos, window_size)):
callback.update(window_i)
if callback.canceled():
break
queries = []
hash_video = {}
for video in video_window:
query = {
'sublanguageid': lang_str,
'moviehash': video.get_osdb_hash(),
'moviebytesize': str(video.get_size()),
}
if video.get_osdb_hash() is None:
log.debug('osdb hash of "{}" is empty -> skip'.format(
video.get_filepath()))
self._signal_connection_failed(
) # FIXME: other name + general signaling
continue
queries.append(query)
hash_video[video.get_osdb_hash()] = video
def run_query():
return self._xmlrpc.SearchSubtitles(
self._token, queries, {'limit': self.SEARCH_LIMIT})
result = self._safe_exec(run_query, None)
self.check_result(result)
if result is None:
continue
for rsub_raw in result['data']:
try:
remote_filename = rsub_raw['SubFileName']
remote_file_size = int(rsub_raw['SubSize'])
remote_id = rsub_raw['IDSubtitleFile']
remote_md5_hash = rsub_raw['SubHash']
remote_download_link = rsub_raw['SubDownloadLink']
remote_link = rsub_raw['SubtitlesLink']
remote_uploader = rsub_raw['UserNickName'].strip()
remote_language_raw = rsub_raw['SubLanguageID']
try:
remote_language = Language.from_unknown(
remote_language_raw, xx=True, xxx=True)
except NotALanguageException:
remote_language = UnknownLanguage(remote_language_raw)
remote_rating = float(rsub_raw['SubRating'])
remote_date = datetime.datetime.strptime(
rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
remote_subtitle = OpenSubtitlesSubtitleFile(
filename=remote_filename,
file_size=remote_file_size,
md5_hash=remote_md5_hash,
id_online=remote_id,
download_link=remote_download_link,
link=remote_link,
uploader=remote_uploader,
language=remote_language,
rating=remote_rating,
date=remote_date,
)
movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
video = hash_video[movie_hash]
imdb_id = rsub_raw['IDMovieImdb']
try:
imdb_rating = float(rsub_raw['MovieImdbRating'])
except (ValueError, KeyError):
imdb_rating = None
imdb_identity = ImdbIdentity(imdb_id=imdb_id,
imdb_rating=imdb_rating)
video_name = rsub_raw['MovieName']
try:
video_year = int(rsub_raw['MovieYear'])
except (ValueError, KeyError):
video_year = None
video_identity = VideoIdentity(name=video_name,
year=video_year)
try:
series_season = int(rsub_raw['SeriesSeason'])
except (KeyError, ValueError):
series_season = None
try:
series_episode = int(rsub_raw['SeriesEpisode'])
except (KeyError, ValueError):
series_episode = None
series_identity = SeriesIdentity(season=series_season,
episode=series_episode)
identity = ProviderIdentities(
video_identity=video_identity,
imdb_identity=imdb_identity,
episode_identity=series_identity,
provider=self)
video.add_subtitle(remote_subtitle)
video.add_identity(identity)
remote_subtitles.append(remote_subtitle)
except (KeyError, ValueError):
log.exception(
'Error parsing result of SearchSubtitles(...)')
log.error('Offending query is: {queries}'.format(
queries=queries))
log.error('Offending result is: {remote_sub}'.format(
remote_sub=rsub_raw))
callback.finish()
return remote_subtitles
def query_text(self, query):
return OpenSubtitlesTextQuery(query=query)
def download_subtitles(self, os_rsubs):
log.debug('download_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
window_size = 20
map_id_data = {}
for window_i, os_rsub_window in enumerate(
window_iterator(os_rsubs, window_size)):
query = [subtitle.get_id_online() for subtitle in os_rsub_window]
def run_query():
return self._xmlrpc.DownloadSubtitles(self._token, query)
result = self._safe_exec(run_query, None)
self.check_result(result)
map_id_data.update({
item['idsubtitlefile']: item['data']
for item in result['data']
})
subtitles = [
unzip_bytes(base64.b64decode(
map_id_data[os_rsub.get_id_online()])).read()
for os_rsub in os_rsubs
]
return subtitles
def upload_subtitles(self, local_movie):
log.debug('upload_subtitles()')
if not self.logged_in():
raise ProviderNotConnectedError()
video_subtitles = list(local_movie.iter_video_subtitles())
if not video_subtitles:
return UploadResult(
type=UploadResult.Type.MISSINGDATA,
reason=_('Need at least one subtitle to upload'))
query_try = dict()
for sub_i, (video, subtitle) in enumerate(video_subtitles):
if not video:
return UploadResult(
type=UploadResult.Type.MISSINGDATA,
reason=_('Each subtitle needs an accompanying video'))
query_try['cd{}'.format(sub_i + 1)] = {
'subhash':
subtitle.get_md5_hash(),
'subfilename':
subtitle.get_filename(),
'moviehash':
video.get_osdb_hash(),
'moviebytesize':
str(video.get_size()),
'moviefps':
str(video.get_fps()) if video.get_fps() else None,
'movieframes':
str(video.get_framecount())
if video.get_framecount() else None,
'moviefilename':
video.get_filename(),
}
def run_query_try_upload():
return self._xmlrpc.TryUploadSubtitles(self._token, query_try)
try_result = self._safe_exec(run_query_try_upload, None)
self.check_result(try_result)
if int(try_result['alreadyindb']):
return UploadResult(type=UploadResult.Type.DUPLICATE,
reason=_('Subtitle is already in database'))
if local_movie.get_imdb_id() is None:
return UploadResult(type=UploadResult.Type.MISSINGDATA,
reason=_('Need IMDb id'))
upload_base_info = {
'idmovieimdb': local_movie.get_imdb_id(),
}
if local_movie.get_comments() is not None:
upload_base_info['subauthorcomment'] = local_movie.get_comments()
if not local_movie.get_language().is_generic():
upload_base_info['sublanguageid'] = local_movie.get_language().xxx(
)
if local_movie.get_release_name() is not None:
upload_base_info[
'moviereleasename'] = local_movie.get_release_name()
if local_movie.get_movie_name() is not None:
upload_base_info['movieaka'] = local_movie.get_movie_name()
if local_movie.is_hearing_impaired() is not None:
upload_base_info[
'hearingimpaired'] = local_movie.is_hearing_impaired()
if local_movie.is_high_definition() is not None:
upload_base_info[
'highdefinition'] = local_movie.is_high_definition()
if local_movie.is_automatic_translation() is not None:
upload_base_info[
'automatictranslation'] = local_movie.is_automatic_translation(
)
if local_movie.get_author() is not None:
upload_base_info['subtranslator'] = local_movie.get_author()
if local_movie.is_foreign_only() is not None:
upload_base_info['foreignpartsonly'] = local_movie.is_foreign_only(
)
query_upload = {
'baseinfo': upload_base_info,
}
for sub_i, (video, subtitle) in enumerate(video_subtitles):
sub_bytes = subtitle.get_filepath().open(mode='rb').read()
sub_tx_data = base64.b64encode(zlib.compress(sub_bytes)).decode()
query_upload['cd{}'.format(sub_i + 1)] = {
'subhash':
subtitle.get_md5_hash(),
'subfilename':
subtitle.get_filename(),
'moviehash':
video.get_osdb_hash(),
'moviebytesize':
str(video.get_size()),
'movietimems':
str(video.get_time_ms()) if video.get_time_ms() else None,
'moviefps':
str(video.get_fps()) if video.get_fps() else None,
'movieframes':
str(video.get_framecount())
if video.get_framecount() else None,
'moviefilename':
video.get_filename(),
'subcontent':
sub_tx_data,
}
def run_query_upload():
return self._xmlrpc.UploadSubtitles(self._token, query_upload)
result = self._safe_exec(run_query_upload, None)
self.check_result(result)
rsubs = []
for sub_data in result['data']:
filename = sub_data['SubFileName']
file_size = sub_data['SubSize']
md5_hash = sub_data['SubHash']
id_online = sub_data['IDSubMOvieFile']
download_link = sub_data['SubDownloadLink']
link = None
uploader = sub_data['UserNickName']
language = Language.from_xxx(sub_data['SubLanguageID'])
rating = float(sub_data['SubRating'])
add_date = datetime.datetime.strptime(sub_data['SubAddDate'],
'%Y-%m-%d %H:%M:%S')
sub = OpenSubtitlesSubtitleFile(filename=filename,
file_size=file_size,
md5_hash=md5_hash,
id_online=id_online,
download_link=download_link,
link=link,
uploader=uploader,
language=language,
rating=rating,
date=add_date)
rsubs.append(sub)
return UploadResult(type=UploadResult.Type.OK, rsubs=rsubs)
def imdb_search_title(self, title):
self._ensure_connection()
def run_query():
return self._xmlrpc.SearchMoviesOnIMDB(self._token, title.strip())
result = self._safe_exec(run_query, default=None)
self.check_result(result)
imdbs = []
re_title = re.compile(r'(?P<title>.*) \((?P<year>[0-9]+)\)')
for imdb_data in result['data']:
imdb_id = imdb_data['id']
if all(c in string.digits for c in imdb_id):
imdb_id = 'tt{}'.format(imdb_id)
m = re_title.match(imdb_data['title'])
if m:
imdb_title = m['title']
imdb_year = int(m['year'])
else:
imdb_title = imdb_data['title']
imdb_year = None
imdbs.append(
ImdbMovieMatch(imdb_id=imdb_id,
title=imdb_title,
year=imdb_year))
return imdbs
def ping(self):
log.debug('ping()')
if not self.logged_in():
raise ProviderNotConnectedError()
def run_query():
return self._xmlrpc.NoOperation(self._token)
result = self._safe_exec(run_query, None)
self.check_result(result)
def provider_info(self):
if self.connected():
def run_query():
return self._xmlrpc.ServerInfo()
result = self._safe_exec(run_query, None)
data = [
(_('XML-RPC version'), result['xmlrpc_version']),
(_('XML-RPC url'), result['xmlrpc_url']),
(_('Application'), result['application']),
(_('Contact'), result['contact']),
(_('Website url'), result['website_url']),
(_('Users online'), result['users_online_total']),
(_('Programs online'), result['users_online_program']),
(_('Users logged in'), result['users_loggedin']),
(_('Max users online'), result['users_max_alltime']),
(_('Users registered'), result['users_registered']),
(_('Subtitles downloaded'), result['subs_downloads']),
(_('Subtitles available'), result['subs_subtitle_files']),
(_('Number movies'), result['movies_total']),
(_('Number languages'), result['total_subtitles_languages']),
(_('Client IP'), result['download_limits']['client_ip']),
(_('24h global download limit'),
result['download_limits']['global_24h_download_limit']),
(_('24h client download limit'),
result['download_limits']['client_24h_download_limit']),
(_('24h client download count'),
result['download_limits']['client_24h_download_count']),
(_('Client download quota'),
result['download_limits']['client_download_quota']),
]
else:
data = []
return data
@staticmethod
def _languages_to_str(languages):
if languages:
lang_str = ','.join([language.xxx() for language in languages])
else:
lang_str = 'all'
return lang_str
@classmethod
def get_name(cls):
return 'opensubtitles'
@classmethod
def get_short_name(cls):
return 'os'
@classmethod
def get_icon(cls):
return ':/images/sites/opensubtitles.png'
def _signal_connection_failed(self):
# FIXME: set flag/... to signal users that the connection has failed
pass
def _safe_exec(self, query, default):
self._ensure_connection()
try:
result = query()
return result
except (ProtocolError, CannotSendRequest, SocketError,
ExpatError) as e:
self._signal_connection_failed()
log.debug('Query failed: {} {}'.format(type(e), e.args))
return default
STATUS_CODE_RE = re.compile(r'(\d+) (.+)')
@classmethod
def check_result(cls, data):
log.debug('check_result(<data>)')
if data is None:
log.warning('data is None ==> FAIL')
raise OpenSubtitlesProviderConnectionError(None, _('No message'))
log.debug('checking presence of "status" in result ...')
if 'status' not in data:
log.debug('... no "status" in result ==> assuming SUCCESS')
return
log.debug('... FOUND')
status = data['status']
log.debug('result["status"]="{status}"'.format(status=status))
log.debug('applying regex to status ...')
try:
code, message = cls.STATUS_CODE_RE.match(status).groups()
log.debug('... regex SUCCEEDED')
code = int(code)
except (AttributeError, ValueError):
log.debug('... regex FAILED')
log.warning('Got unexpected status="{status}" from server.'.format(
status=status))
log.debug('Checking for presence of "200" ...')
if '200' not in data['status']:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise OpenSubtitlesProviderConnectionError(
None,
_('Server returned status="{status}". Expected "200 OK".').
format(status=data['status']), data['status'])
log.debug('... SUCCESS')
code, message = 200, 'OK'
log.debug('Checking code={code} ...'.format(code=code))
if code != 200:
log.debug('... FAIL. Raising ProviderConnectionError.')
raise OpenSubtitlesProviderConnectionError(code, message)
log.debug('... SUCCESS.')
log.debug('check_result() finished (data is ok)')
class SubtitleDownload(QtCore.QThread):
'''Traverses a directory and all subdirectories and downloads subtitles.
Relies on an undocumented OpenSubtitles feature where subtitles seems to be
sorted by filehash and popularity.
'''
api_url = 'http://api.opensubtitles.org/xml-rpc'
login_token = None
server = None
def __init__(self, parent=None):
QtCore.QThread.__init__(self, parent)
def init(self, movie_paths):
self._movie_paths = movie_paths
self.start()
def __del__(self):
if self.login_token:
self.logout()
def stopTask(self):
self.stopping = True
def run(self):
self.stopping = False
self.moviefiles = {}
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Searching for video files...")
"""Check if the movie_path is a file or directory.
Calculate the file stats if its a file, else traverse the directory tree and calculate filestats for all movie files"""
for path in self._movie_paths:
if os.path.isfile(path):
self.calcfilestats(path, os.path.dirname(path))
else:
for root, _, files in os.walk(path):
for file in files:
if not self.stopping:
self.calcfilestats(file, root)
else:
return
try:
if self.moviefiles:
if not self.login_token:
try:
self.login()
except (noInternetConnection):
self.emit(
QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Sorry, No active Internet connection found. Re-Check and try again."
)
my_logger.debug(
"Sorry, No active Internet connection found. Re-Check and try again."
)
self.emit(
QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"),
self._movie_paths)
return
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Searching for subtitles...")
self.search_subtitles()
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Done...")
my_logger.debug("Done...")
else:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Sorry, no video files were found!")
self.emit(QtCore.SIGNAL("downloadComplete(PyQt_PyObject)"),
self._movie_paths)
except Error as e:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
("XML-RPC error:", e))
except UserWarning as uw:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), uw)
def calcfilestats(self, file, parentdir):
''' Calculates and adds the hash of a movie file to the moviefiles dictionary.
Also emits a signal to update the LCD counter showing the found files.'''
video_extns = [
'.avi', '.divx', '.mkv', '.mp4', '.ogg', '.rm', '.rmvb', '.vob',
'.x264', '.xvid'
]
if os.path.splitext(file)[1].lower() in video_extns:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Found: " + file)
my_logger.debug("Found: " + file)
self.emit(QtCore.SIGNAL("updateFound()"))
filehash = self.hashFile(os.path.join(parentdir, file))
filesize = os.path.getsize(os.path.join(parentdir, file))
self.moviefiles[filehash] = {
'dir': parentdir,
'file': file,
'size': filesize
}
def login(self):
'''Log in to OpenSubtitles'''
self.server = ServerProxy(self.api_url, verbose=False)
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging in...")
my_logger.debug("Logging in...")
try:
resp = self.server.LogIn('', '', 'en', 'PySubD v1.0')
self.check_status(resp)
self.login_token = resp['token']
except Error as e:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), e)
my_logger.debug(str(e))
def logout(self):
'''Log out from OpenSubtitles'''
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"), "Logging out...")
resp = self.server.LogOut(self.login_token)
self.check_status(resp)
def search_subtitles(self):
'''Search OpenSubtitles for matching subtitles using moviehash and filebytesize'''
search = []
for hash in self.moviefiles.keys():
search.append({
'sublanguageid': 'eng',
'moviehash': hash,
'moviebytesize': str(self.moviefiles[hash]['size'])
})
my_logger.debug("Length of search string: " + str(len(search)))
resp = {}
resp['data'] = []
while search[:500]:
if not self.stopping:
tempresp = self.server.SearchSubtitles(self.login_token,
search[:500])
if tempresp['data'] != False:
resp['data'].extend(tempresp['data'])
self.check_status(tempresp)
search = search[500:]
else:
return
#Check if we actually got some matching subtitles to download, else return
if not resp['data']:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Sorry, no subtitles were found!")
my_logger.debug("Sorry, no subtitles were found!")
return
#User Ranks on OpenSubtitles.org.These are decided based upon the number of subtitles uploaded by the member.
#A better rank is often an indication of a better subtitle quality.
user_ranks = {
'admin': 1,
'platinum member': 2,
'vip member': 3,
'gold member': 4,
'trusted': 5,
'silver member': 6,
'bronze member': 7,
'sub leecher': 8,
'': 9
}
# A dictionary to store the subtitle id's found corresponding to every file hash
subtitles = {}
for result in resp['data']:
#The subtitle must not be rated bad
if int(result['SubBad']) != 1:
hash = result['MovieHash']
subid = result['IDSubtitleFile']
downcount = result['SubDownloadsCnt']
rating = result['SubRating']
user_rank = user_ranks[result['UserRank']]
#First good matching subtitle found
if not subtitles.get(hash):
subtitles[hash] = {
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank
}
self.emit(QtCore.SIGNAL("updateAvailable()"))
#Another good quality subtitle found uploaded by a more reputed user
elif subtitles[hash]['user_rank'] > user_rank:
subtitles[hash] = {
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank
}
#Another good quality subtitle found with a perfect rating, uploaded by a more or equally reputed user
elif float(rating) == 10.0 and subtitles[hash][
'user_rank'] >= user_rank:
subtitles[hash] = {
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank
}
#Another good quality subtitle found with the better rating, higher download count and uploaded by a more or equally trusted user
elif float(subtitles[hash]['rating']) >= float(rating) \
and int(subtitles[hash]['downcount']) < int(downcount) and subtitles[hash]['user_rank'] >= user_rank:
subtitles[hash] = {
'subid': subid,
'downcount': downcount,
'rating': rating,
'user_rank': user_rank
}
my_logger.debug("Total number of subtitles found: " +
str(len(subtitles)))
notfound = []
for hash, filedetails in self.moviefiles.iteritems():
if not self.stopping:
if subtitles.get(hash):
try:
subtitle = self.download_subtitles(
[subtitles[hash]['subid']])
self.emit(
QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"Saving subtitle for: " + filedetails['file'])
my_logger.debug("Saving subtitle for: " + filedetails['file'] + " Hash : " + hash + " Rating: " +
subtitles[hash]['rating'] + " DwnCnt: " + subtitles[hash]['downcount'] + " UpldrRnk:" + \
str(subtitles[hash]['user_rank']))
self.emit(QtCore.SIGNAL("updateDownloaded()"))
filename = os.path.join(
filedetails['dir'],
os.path.splitext(filedetails['file'])[0] + ".srt")
file = open(filename, "wb")
file.write(subtitle)
file.close()
except IOError:
self.emit(
QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"IO Error in saving subs for " +
filedetails['file'])
else:
notfound.append(filedetails['file'])
else:
return
#Report all the files for which no subtitles were found in an alphabetically sorted order.
notfound.sort(key=str.lower)
for file in notfound:
self.emit(QtCore.SIGNAL("updategui(PyQt_PyObject)"),
"No subtitles found for: " + file)
my_logger.debug("No subtitles were found for: " + file)
def download_subtitles(self, subparam):
resp = self.server.DownloadSubtitles(self.login_token, subparam)
self.check_status(resp)
decoded = base64.standard_b64decode(
resp['data'][0]['data'].encode('ascii'))
decompressed = zlib.decompress(decoded, 15 + 32)
return decompressed
def check_status(self, resp):
'''Check the return status of the request.
Anything other than "200 OK" raises a UserWarning
'''
if resp['status'].upper() != '200 OK':
raise UserWarning("Response error from " + self.api_url +
". Response status was: " + resp['status'])
def hashFile(self, name):
''' Calculates the hash value of a movie.
Copied from OpenSubtitles own examples:
http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes
'''
try:
longlongformat = 'q' # long long
bytesize = struct.calcsize(longlongformat)
f = open(name, "rb")
filesize = os.path.getsize(name)
hash = filesize
if filesize < 65536 * 2:
return "SizeError: Minimum file size must be 120Kb"
for x in range(65536 // bytesize):
buffer = f.read(bytesize)
(l_value, ) = struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF # to remain as 64bit number
f.seek(max(0, filesize - 65536), 0)
for x in range(65536 // bytesize):
buffer = f.read(bytesize)
(l_value, ) = struct.unpack(longlongformat, buffer)
hash += l_value
hash = hash & 0xFFFFFFFFFFFFFFFF
f.close()
returnedhash = "%016x" % hash
return returnedhash
except (IOError):
return "IOError"
class OpenSubtitles(object):
def __init__(self):
self.xmlrpc = ServerProxy(Settings.OPENSUBTITLES_SERVER,
allow_none=True)
self.language = Settings.LANGUAGE
self.token = None
def _get_from_data_or_none(self, key):
"""
Проверка на аутентификацию
Args:
key (str): строка с параметром который нужно вернуть
Returns:
(str) : возвращаемое значение
"""
status = self.data.get('status').split()[0]
return self.data.get(key) if '200' == status else None
def login(self):
"""
Авторизация
Returns:
(str) : токен
"""
self.data = self.xmlrpc.LogIn(Settings.USER_NAME,
Settings.USER_PASSWORD, self.language,
Settings.USER_AGENT)
token = self._get_from_data_or_none('token')
if token:
self.token = token
return token
def logout(self):
"""
Логаут
Returns:
(str) : статус
"""
data = self.xmlrpc.LogOut(self.token)
return '200' in data.get('status')
def search_subtitles(self, params):
"""
Поиск субтитров
Args:
params(dict):
+ язык субтитров
+ id фильма на сайте *imdb.com*
Returns:
(dic) :
+ id субтитров
+ формат субтитров
+ id фильма
"""
self.data = self.xmlrpc.SearchSubtitles(self.token, params)
return self.data
def search_movies_on_imdb(self, params):
"""
Поиск фильмов
Args:
params(dict):
+ название фильма
Returns:
(dic) :
+ id фильма
+ название фильма
"""
self.data = self.xmlrpc.SearchMoviesOnIMDB(self.token, params)
return self.data
def download_subtitles(self, params):
"""
Загрузка субтитров
Args:
params(int): id субтитров
Returns:
(dic) :
+ id субтитров
+ строка с субтитрами
"""
self.data = self.xmlrpc.DownloadSubtitles(self.token, params)
return self.data
filePath = os.environ['NAUTILUS_SCRIPT_SELECTED_FILE_PATHS'].splitlines()[0]
print('File path:', filePath)
folders = filePath.split('/')
fileName = folders[len(folders)-1]
print('File name:', fileName)
folderPath = filePath[0:filePath.index(fileName)]
print('Folder path:', folderPath)
print('Searching the best subtitle for file', fileName, '...')
print('Retrieving loginToken...')
server = ServerProxy('https://api.opensubtitles.org/xml-rpc')
session = server.LogIn('Lodono', 'bocegajen', languageID, 'opensubtitles-download 4.1')
if session['status'] != '200 OK':
print('Login failed. Exiting...')
sys.exit()
loginToken = session['token']
print('Success:', loginToken)
print('First trying to query using file hash and size...')
tryHashAndSize = queryByHashAndSize(filePath, languageID, server, loginToken)
if tryHashAndSize['success']:
def on_task_download(self, task, config):
# filter all entries that have IMDB ID set
try:
entries = [e for e in task.accepted if e['imdb_id'] is not None]
except KeyError:
# No imdb urls on this task, skip it
# TODO: should do lookup via imdb_lookup plugin?
return
try:
s = ServerProxy("http://api.opensubtitles.org/xml-rpc")
res = s.LogIn("", "", "en", "FlexGet")
except Exception:
logger.warning('Error connecting to opensubtitles.org')
return
if res['status'] != '200 OK':
raise Exception("Login to opensubtitles.org XML-RPC interface failed")
config = self.prepare_config(config, task)
token = res['token']
# configuration
languages = config['languages']
min_sub_rating = config['min_sub_rating']
match_limit = config[
'match_limit'
] # no need to change this, but it should be configurable
# loop through the entries
for entry in entries:
imdbid = entry.get('imdb_id')
if not imdbid:
logger.debug('no match for {}', entry['title'])
continue
query = []
for language in languages:
query.append({'sublanguageid': language, 'imdbid': imdbid})
subtitles = s.SearchSubtitles(token, query)
subtitles = subtitles['data']
# nothing found -> continue
if not subtitles:
continue
# filter bad subs
subtitles = [x for x in subtitles if x['SubBad'] == '0']
# some quality required (0.0 == not reviewed)
subtitles = [
x
for x in subtitles
if float(x['SubRating']) >= min_sub_rating or float(x['SubRating']) == 0.0
]
filtered_subs = []
# find the best rated subs for each language
for language in languages:
langsubs = [x for x in subtitles if x['SubLanguageID'] == language]
# did we find any subs for this language?
if langsubs:
def seqmatch(subfile):
s = difflib.SequenceMatcher(lambda x: x in " ._", entry['title'], subfile)
# print "matching: ", entry['title'], subfile, s.ratio()
return s.ratio() > match_limit
# filter only those that have matching release names
langsubs = [x for x in subtitles if seqmatch(x['MovieReleaseName'])]
if langsubs:
# find the best one by SubRating
langsubs.sort(key=lambda x: float(x['SubRating']))
langsubs.reverse()
filtered_subs.append(langsubs[0])
# download
for sub in filtered_subs:
logger.debug(
'SUBS FOUND: {} {} {}',
sub['MovieReleaseName'],
sub['SubRating'],
sub['SubLanguageID'],
)
f = task.requests.get(sub['ZipDownloadLink'])
subfilename = re.match(
'^attachment; filename="(.*)"$', f.headers['content-disposition']
).group(1)
outfile = os.path.join(config['output'], subfilename)
fp = open(outfile, 'w')
fp.write(f.raw)
fp.close()
f.close()
s.LogOut(token)
# Do not spawn too many instances at once
time.sleep(0.33)
if opt_gui == 'cli' and opt_selection_mode != 'auto':
# Synchronous call
process_videoDispatched = subprocess.call(command)
else:
# Asynchronous call
process_videoDispatched = subprocess.Popen(command)
# ==== Search and download subtitles ===========================================
try:
# ==== Connection to OpenSubtitlesDownload
try:
session = osd_server.LogIn(osd_username, osd_password, osd_language, 'opensubtitles-download 4.1')
except Exception:
# Retry once after a delay (could just be a momentary overloaded server?)
time.sleep(3)
try:
session = osd_server.LogIn(osd_username, osd_password, osd_language, 'opensubtitles-download 4.1')
except Exception:
superPrint("error", "Connection error!", "Unable to reach opensubtitles.org servers!\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!")
sys.exit(2)
# Connection refused?
if session['status'] != '200 OK':
superPrint("error", "Connection error!", "Opensubtitles.org servers refused the connection: " + session['status'] + ".\n\nPlease check:\n- Your Internet connection status\n- www.opensubtitles.org availability\n- Your downloads limit (200 subtitles per 24h)\n\nThe subtitles search and download service is powered by opensubtitles.org. Be sure to donate if you appreciate the service provided!")
sys.exit(2)
# Count languages marked for this search
filePath = os.environ['NAUTILUS_SCRIPT_SELECTED_FILE_PATHS'].splitlines()[0]
print('File path:', filePath)
folders = filePath.split('/')
fileName = folders[len(folders) - 1]
print('File name:', fileName)
folderPath = filePath[0:filePath.index(fileName)]
print('Folder path:', folderPath)
print('Searching the best subtitle for file', fileName, '...')
print('Retrieving loginToken...')
server = ServerProxy('https://api.opensubtitles.org/xml-rpc')
session = server.LogIn('Lodono', 'bocegajen', languageID, 'TemporaryUserAgent')
if session['status'] != '200 OK':
print('Login failed. Exiting...')
sys.exit()
loginToken = session['token']
print('Success:', loginToken)
print('First trying to query using file hash and size...')
tryHashAndSize = queryByHashAndSize(filePath, languageID, server, loginToken)
if tryHashAndSize['success']:
