Ejemplo n.º 1
0
class OpenSubtitles(SubtitleProvider):
    URL = 'http://api.opensubtitles.org/xml-rpc'

    def __init__(self):
        SubtitleProvider.__init__(self)
        self._xmlrpc = None
        self._token = None

        self._settings = OpenSubtitlesSettings()

    def get_settings(self):
        return self._settings

    def set_settings(self, settings):
        self._settings = settings

    def connect(self):
        log.debug('connect()')
        if self.connected():
            return
        self._xmlrpc = ServerProxy(self.URL, allow_none=False)

    def disconnect(self):
        log.debug('disconnect()')
        if self.logged_in():
            self.logout()
        if self.connected():
            self._xmlrpc = None

    def connected(self):
        return self._xmlrpc is not None

    def login(self):
        log.debug('login()')
        if self.logged_in():
            return
        if not self.connected():
            raise ProviderNotConnectedError()

        def login_query():
            # FIXME: 'en' language ok??? or '' as in the original
            return self._xmlrpc.LogIn(str(self._settings.username),
                                      str(self._settings.password), 'en',
                                      str(self._settings.get_user_agent()))

        result = self._safe_exec(login_query, None)
        self.check_result(result)
        self._token = result['token']

    def logout(self):
        log.debug('logout()')
        if self.logged_in():

            def logout_query():
                return self._xmlrpc.LogOut(self._token)

            result = self._safe_exec(logout_query, None)
            self.check_result(result)
        self._token = None

    def logged_in(self):
        return self._token is not None

    SEARCH_LIMIT = 500

    def search_videos(self, videos, callback, languages=None):
        log.debug('search_videos(#videos={})'.format(len(videos)))
        if not self.logged_in():
            raise ProviderNotConnectedError()

        lang_str = self._languages_to_str(languages)

        window_size = 5
        callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)

        remote_subtitles = []
        for window_i, video_window in enumerate(
                window_iterator(videos, window_size)):
            callback.update(window_i)
            if callback.canceled():
                break

            queries = []
            hash_video = {}
            for video in video_window:
                query = {
                    'sublanguageid': lang_str,
                    'moviehash': video.get_osdb_hash(),
                    'moviebytesize': str(video.get_size()),
                }
                queries.append(query)
                hash_video[video.get_osdb_hash()] = video

            def run_query():
                return self._xmlrpc.SearchSubtitles(
                    self._token, queries, {'limit': self.SEARCH_LIMIT})

            result = self._safe_exec(run_query, None)
            self.check_result(result)
            if result is None:
                continue

            for rsub_raw in result['data']:
                try:
                    remote_filename = rsub_raw['SubFileName']
                    remote_file_size = int(rsub_raw['SubSize'])
                    remote_id = rsub_raw['IDSubtitleFile']
                    remote_md5_hash = rsub_raw['SubHash']
                    remote_download_link = rsub_raw['SubDownloadLink']
                    remote_link = rsub_raw['SubtitlesLink']
                    remote_uploader = rsub_raw['UserNickName'].strip()
                    remote_language_raw = rsub_raw['SubLanguageID']
                    try:
                        remote_language = Language.from_unknown(
                            remote_language_raw, xx=True, xxx=True)
                    except NotALanguageException:
                        remote_language = UnknownLanguage(remote_language_raw)
                    remote_rating = float(rsub_raw['SubRating'])
                    remote_date = datetime.datetime.strptime(
                        rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
                    remote_subtitle = OpenSubtitlesSubtitleFile(
                        filename=remote_filename,
                        file_size=remote_file_size,
                        md5_hash=remote_md5_hash,
                        id_online=remote_id,
                        download_link=remote_download_link,
                        link=remote_link,
                        uploader=remote_uploader,
                        language=remote_language,
                        rating=remote_rating,
                        age=remote_date,
                    )
                    movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
                    video = hash_video[movie_hash]

                    imdb_id = rsub_raw['IDMovieImdb']
                    imdb_identity = ImdbIdentity(imdb_id=imdb_id,
                                                 imdb_rating=None)
                    identity = ProviderIdentities(imdb_identity=imdb_identity,
                                                  provider=self)

                    video.add_subtitle(remote_subtitle)
                    video.add_identity(identity)

                    remote_subtitles.append(remote_subtitle)
                except (KeyError, ValueError):
                    log.exception(
                        'Error parsing result of SearchSubtitles(...)')
                    log.error('Offending query is: {queries}'.format(
                        queries=queries))
                    log.error('Offending result is: {remote_sub}'.format(
                        remote_sub=rsub_raw))

        callback.finish()
        return remote_subtitles

    def query_text(self, query):
        return OpenSubtitlesTextQuery(query=query)

    def download_subtitles(self, os_rsubs):
        log.debug('download_subtitles()')
        if not self.logged_in():
            raise ProviderNotConnectedError()

        window_size = 20
        map_id_data = {}
        for window_i, os_rsub_window in enumerate(
                window_iterator(os_rsubs, window_size)):
            query = [subtitle.get_id_online() for subtitle in os_rsub_window]

            def run_query():
                return self._xmlrpc.DownloadSubtitles(self._token, query)

            result = self._safe_exec(run_query, None)

            self.check_result(result)
            map_id_data.update({
                item['idsubtitlefile']: item['data']
                for item in result['data']
            })
        subtitles = [
            unzip_bytes(base64.b64decode(
                map_id_data[os_rsub.get_id_online()])).read()
            for os_rsub in os_rsubs
        ]
        return subtitles

    def ping(self):
        log.debug('ping()')
        if not self.logged_in():
            raise ProviderNotConnectedError()

        def run_query():
            return self._xmlrpc.NoOperation(self._token)

        result = self._safe_exec(run_query, None)
        self.check_result(result)

    @staticmethod
    def _languages_to_str(languages):
        if languages:
            lang_str = ','.join([language.xxx() for language in languages])
        else:
            lang_str = 'all'

        return lang_str

    @classmethod
    def get_name(cls):
        return 'opensubtitles'

    @classmethod
    def get_short_name(cls):
        return 'os'

    def _signal_connection_failed(self):
        # FIXME: set flag/... to signal users that the connection has failed
        pass

    def _safe_exec(self, query, default):
        try:
            result = query()
            return result
        except (ProtocolError, CannotSendRequest, SocketError):
            self._signal_connection_failed()
            log.warning('Query failed', exc_info=sys.exc_info())
            return default

    STATUS_CODE_RE = re.compile('(\d+) (.+)')

    @classmethod
    def check_result(cls, data):
        log.debug('check_result(<data>)')
        if data is None:
            log.warning('data is None ==> FAIL')
            raise ProviderConnectionError(_('No message'))
        log.debug('checking presence of "status" in result ...')
        if 'status' not in data:
            log.debug('... no "status" in result ==> assuming SUCCESS')
            return
        log.debug('... FOUND')
        status = data['status']
        log.debug('result["status"]="{status}"'.format(status=status))
        log.debug('applying regex to status ...')
        try:
            code, message = cls.STATUS_CODE_RE.match(status).groups()
            log.debug('... regex SUCCEEDED')
            code = int(code)
        except (AttributeError, ValueError):
            log.debug('... regex FAILED')
            log.warning('Got unexpected status="{status}" from server.'.format(
                status=status))
            log.debug('Checking for presence of "200" ...')
            if '200' not in data['status']:
                log.debug('... FAIL. Raising ProviderConnectionError.')
                raise ProviderConnectionError(
                    _('Server returned status="{status}". Expected "200 OK".').
                    format(status=data['status']), data['status'])
            log.debug('... SUCCESS')
            code, message = 200, 'OK'
        log.debug('Checking code={code} ...'.format(code=code))
        if code != 200:
            log.debug('... FAIL. Raising ProviderConnectionError.')
            raise ProviderConnectionError(message, code)
        log.debug('... SUCCESS.')
        log.debug('check_result() finished (data is ok)')
Ejemplo n.º 2
0
class OpenSubtitles(SubtitleProvider):
    URL = 'http://api.opensubtitles.org/xml-rpc'

    def __init__(self, settings=None):
        SubtitleProvider.__init__(self)
        self._xmlrpc = None
        self._token = None
        self._last_time = None

        if settings is None:
            settings = OpenSubtitlesSettings()
        self._settings = settings

    def get_settings(self):
        return self._settings

    def set_settings(self, settings):
        if self.connected():
            raise RuntimeError(
                'Cannot set settings while connected')  # FIXME: change error
        self._settings = settings

    def connect(self):
        log.debug('connect()')
        if self.connected():
            return
        self._xmlrpc = ServerProxy(self.URL, allow_none=False)
        self._last_time = time.time()

    def disconnect(self):
        log.debug('disconnect()')
        if self.logged_in():
            self.logout()
        if self.connected():
            self._xmlrpc = None

    def connected(self):
        return self._xmlrpc is not None

    def login(self):
        log.debug('login()')
        if self.logged_in():
            return
        if not self.connected():
            self.connect()

        def login_query():
            # FIXME: 'en' language ok??? or '' as in the original
            return self._xmlrpc.LogIn(str(self._settings.username),
                                      str(self._settings.password), 'en',
                                      str(self._settings.get_user_agent()))

        result = self._safe_exec(login_query, None)
        self.check_result(result)
        self._token = result['token']

    def logout(self):
        log.debug('logout()')
        if self.logged_in():

            def logout_query():
                return self._xmlrpc.LogOut(self._token)

            # Do no check result of this call. Assume connection closed.
            self._safe_exec(logout_query, None)
        self._token = None

    def logged_in(self):
        return self._token is not None

    def reestablish(self):
        log.debug('reestablish()')
        connected = self.connected()
        logged_in = self.logged_in()
        self.disconnect()
        if connected:
            self.connect()
        if logged_in:
            self.login()

    _TIMEOUT_MS = 60000

    def _ensure_connection(self):
        now = time.time()
        if now - time.time() > self._TIMEOUT_MS:
            self.reestablish()
            self._last_time = now

    SEARCH_LIMIT = 500

    def search_videos(self, videos, callback, languages=None):
        log.debug('search_videos(#videos={})'.format(len(videos)))
        if not self.logged_in():
            raise ProviderNotConnectedError()

        lang_str = self._languages_to_str(languages)

        window_size = 5
        callback.set_range(0, (len(videos) + (window_size - 1)) // window_size)

        remote_subtitles = []
        for window_i, video_window in enumerate(
                window_iterator(videos, window_size)):
            callback.update(window_i)
            if callback.canceled():
                break

            queries = []
            hash_video = {}
            for video in video_window:
                query = {
                    'sublanguageid': lang_str,
                    'moviehash': video.get_osdb_hash(),
                    'moviebytesize': str(video.get_size()),
                }
                if video.get_osdb_hash() is None:
                    log.debug('osdb hash of "{}" is empty -> skip'.format(
                        video.get_filepath()))
                    self._signal_connection_failed(
                    )  # FIXME: other name + general signaling
                    continue
                queries.append(query)
                hash_video[video.get_osdb_hash()] = video

            def run_query():
                return self._xmlrpc.SearchSubtitles(
                    self._token, queries, {'limit': self.SEARCH_LIMIT})

            result = self._safe_exec(run_query, None)
            self.check_result(result)
            if result is None:
                continue

            for rsub_raw in result['data']:
                try:
                    remote_filename = rsub_raw['SubFileName']
                    remote_file_size = int(rsub_raw['SubSize'])
                    remote_id = rsub_raw['IDSubtitleFile']
                    remote_md5_hash = rsub_raw['SubHash']
                    remote_download_link = rsub_raw['SubDownloadLink']
                    remote_link = rsub_raw['SubtitlesLink']
                    remote_uploader = rsub_raw['UserNickName'].strip()
                    remote_language_raw = rsub_raw['SubLanguageID']
                    try:
                        remote_language = Language.from_unknown(
                            remote_language_raw, xx=True, xxx=True)
                    except NotALanguageException:
                        remote_language = UnknownLanguage(remote_language_raw)
                    remote_rating = float(rsub_raw['SubRating'])
                    remote_date = datetime.datetime.strptime(
                        rsub_raw['SubAddDate'], '%Y-%m-%d %H:%M:%S')
                    remote_subtitle = OpenSubtitlesSubtitleFile(
                        filename=remote_filename,
                        file_size=remote_file_size,
                        md5_hash=remote_md5_hash,
                        id_online=remote_id,
                        download_link=remote_download_link,
                        link=remote_link,
                        uploader=remote_uploader,
                        language=remote_language,
                        rating=remote_rating,
                        date=remote_date,
                    )
                    movie_hash = '{:>016}'.format(rsub_raw['MovieHash'])
                    video = hash_video[movie_hash]

                    imdb_id = rsub_raw['IDMovieImdb']
                    try:
                        imdb_rating = float(rsub_raw['MovieImdbRating'])
                    except (ValueError, KeyError):
                        imdb_rating = None
                    imdb_identity = ImdbIdentity(imdb_id=imdb_id,
                                                 imdb_rating=imdb_rating)

                    video_name = rsub_raw['MovieName']
                    try:
                        video_year = int(rsub_raw['MovieYear'])
                    except (ValueError, KeyError):
                        video_year = None
                    video_identity = VideoIdentity(name=video_name,
                                                   year=video_year)

                    try:
                        series_season = int(rsub_raw['SeriesSeason'])
                    except (KeyError, ValueError):
                        series_season = None
                    try:
                        series_episode = int(rsub_raw['SeriesEpisode'])
                    except (KeyError, ValueError):
                        series_episode = None
                    series_identity = SeriesIdentity(season=series_season,
                                                     episode=series_episode)

                    identity = ProviderIdentities(
                        video_identity=video_identity,
                        imdb_identity=imdb_identity,
                        episode_identity=series_identity,
                        provider=self)
                    video.add_subtitle(remote_subtitle)
                    video.add_identity(identity)

                    remote_subtitles.append(remote_subtitle)
                except (KeyError, ValueError):
                    log.exception(
                        'Error parsing result of SearchSubtitles(...)')
                    log.error('Offending query is: {queries}'.format(
                        queries=queries))
                    log.error('Offending result is: {remote_sub}'.format(
                        remote_sub=rsub_raw))

        callback.finish()
        return remote_subtitles

    def query_text(self, query):
        return OpenSubtitlesTextQuery(query=query)

    def download_subtitles(self, os_rsubs):
        log.debug('download_subtitles()')
        if not self.logged_in():
            raise ProviderNotConnectedError()

        window_size = 20
        map_id_data = {}
        for window_i, os_rsub_window in enumerate(
                window_iterator(os_rsubs, window_size)):
            query = [subtitle.get_id_online() for subtitle in os_rsub_window]

            def run_query():
                return self._xmlrpc.DownloadSubtitles(self._token, query)

            result = self._safe_exec(run_query, None)

            self.check_result(result)
            map_id_data.update({
                item['idsubtitlefile']: item['data']
                for item in result['data']
            })
        subtitles = [
            unzip_bytes(base64.b64decode(
                map_id_data[os_rsub.get_id_online()])).read()
            for os_rsub in os_rsubs
        ]
        return subtitles

    def upload_subtitles(self, local_movie):
        log.debug('upload_subtitles()')
        if not self.logged_in():
            raise ProviderNotConnectedError()
        video_subtitles = list(local_movie.iter_video_subtitles())
        if not video_subtitles:
            return UploadResult(
                type=UploadResult.Type.MISSINGDATA,
                reason=_('Need at least one subtitle to upload'))

        query_try = dict()
        for sub_i, (video, subtitle) in enumerate(video_subtitles):
            if not video:
                return UploadResult(
                    type=UploadResult.Type.MISSINGDATA,
                    reason=_('Each subtitle needs an accompanying video'))
            query_try['cd{}'.format(sub_i + 1)] = {
                'subhash':
                subtitle.get_md5_hash(),
                'subfilename':
                subtitle.get_filename(),
                'moviehash':
                video.get_osdb_hash(),
                'moviebytesize':
                str(video.get_size()),
                'moviefps':
                str(video.get_fps()) if video.get_fps() else None,
                'movieframes':
                str(video.get_framecount())
                if video.get_framecount() else None,
                'moviefilename':
                video.get_filename(),
            }

        def run_query_try_upload():
            return self._xmlrpc.TryUploadSubtitles(self._token, query_try)

        try_result = self._safe_exec(run_query_try_upload, None)
        self.check_result(try_result)

        if int(try_result['alreadyindb']):
            return UploadResult(type=UploadResult.Type.DUPLICATE,
                                reason=_('Subtitle is already in database'))

        if local_movie.get_imdb_id() is None:
            return UploadResult(type=UploadResult.Type.MISSINGDATA,
                                reason=_('Need IMDb id'))
        upload_base_info = {
            'idmovieimdb': local_movie.get_imdb_id(),
        }

        if local_movie.get_comments() is not None:
            upload_base_info['subauthorcomment'] = local_movie.get_comments()
        if not local_movie.get_language().is_generic():
            upload_base_info['sublanguageid'] = local_movie.get_language().xxx(
            )
        if local_movie.get_release_name() is not None:
            upload_base_info[
                'moviereleasename'] = local_movie.get_release_name()
        if local_movie.get_movie_name() is not None:
            upload_base_info['movieaka'] = local_movie.get_movie_name()
        if local_movie.is_hearing_impaired() is not None:
            upload_base_info[
                'hearingimpaired'] = local_movie.is_hearing_impaired()
        if local_movie.is_high_definition() is not None:
            upload_base_info[
                'highdefinition'] = local_movie.is_high_definition()
        if local_movie.is_automatic_translation() is not None:
            upload_base_info[
                'automatictranslation'] = local_movie.is_automatic_translation(
                )
        if local_movie.get_author() is not None:
            upload_base_info['subtranslator'] = local_movie.get_author()
        if local_movie.is_foreign_only() is not None:
            upload_base_info['foreignpartsonly'] = local_movie.is_foreign_only(
            )

        query_upload = {
            'baseinfo': upload_base_info,
        }
        for sub_i, (video, subtitle) in enumerate(video_subtitles):
            sub_bytes = subtitle.get_filepath().open(mode='rb').read()
            sub_tx_data = base64.b64encode(zlib.compress(sub_bytes)).decode()

            query_upload['cd{}'.format(sub_i + 1)] = {
                'subhash':
                subtitle.get_md5_hash(),
                'subfilename':
                subtitle.get_filename(),
                'moviehash':
                video.get_osdb_hash(),
                'moviebytesize':
                str(video.get_size()),
                'movietimems':
                str(video.get_time_ms()) if video.get_time_ms() else None,
                'moviefps':
                str(video.get_fps()) if video.get_fps() else None,
                'movieframes':
                str(video.get_framecount())
                if video.get_framecount() else None,
                'moviefilename':
                video.get_filename(),
                'subcontent':
                sub_tx_data,
            }

        def run_query_upload():
            return self._xmlrpc.UploadSubtitles(self._token, query_upload)

        result = self._safe_exec(run_query_upload, None)
        self.check_result(result)

        rsubs = []

        for sub_data in result['data']:
            filename = sub_data['SubFileName']
            file_size = sub_data['SubSize']
            md5_hash = sub_data['SubHash']
            id_online = sub_data['IDSubMOvieFile']
            download_link = sub_data['SubDownloadLink']
            link = None
            uploader = sub_data['UserNickName']
            language = Language.from_xxx(sub_data['SubLanguageID'])
            rating = float(sub_data['SubRating'])
            add_date = datetime.datetime.strptime(sub_data['SubAddDate'],
                                                  '%Y-%m-%d %H:%M:%S')
            sub = OpenSubtitlesSubtitleFile(filename=filename,
                                            file_size=file_size,
                                            md5_hash=md5_hash,
                                            id_online=id_online,
                                            download_link=download_link,
                                            link=link,
                                            uploader=uploader,
                                            language=language,
                                            rating=rating,
                                            date=add_date)
            rsubs.append(sub)

        return UploadResult(type=UploadResult.Type.OK, rsubs=rsubs)

    def imdb_search_title(self, title):
        self._ensure_connection()

        def run_query():
            return self._xmlrpc.SearchMoviesOnIMDB(self._token, title.strip())

        result = self._safe_exec(run_query, default=None)
        self.check_result(result)

        imdbs = []
        re_title = re.compile(r'(?P<title>.*) \((?P<year>[0-9]+)\)')
        for imdb_data in result['data']:
            imdb_id = imdb_data['id']
            if all(c in string.digits for c in imdb_id):
                imdb_id = 'tt{}'.format(imdb_id)
            m = re_title.match(imdb_data['title'])
            if m:
                imdb_title = m['title']
                imdb_year = int(m['year'])
            else:
                imdb_title = imdb_data['title']
                imdb_year = None
            imdbs.append(
                ImdbMovieMatch(imdb_id=imdb_id,
                               title=imdb_title,
                               year=imdb_year))
        return imdbs

    def ping(self):
        log.debug('ping()')
        if not self.logged_in():
            raise ProviderNotConnectedError()

        def run_query():
            return self._xmlrpc.NoOperation(self._token)

        result = self._safe_exec(run_query, None)
        self.check_result(result)

    def provider_info(self):
        if self.connected():

            def run_query():
                return self._xmlrpc.ServerInfo()

            result = self._safe_exec(run_query, None)
            data = [
                (_('XML-RPC version'), result['xmlrpc_version']),
                (_('XML-RPC url'), result['xmlrpc_url']),
                (_('Application'), result['application']),
                (_('Contact'), result['contact']),
                (_('Website url'), result['website_url']),
                (_('Users online'), result['users_online_total']),
                (_('Programs online'), result['users_online_program']),
                (_('Users logged in'), result['users_loggedin']),
                (_('Max users online'), result['users_max_alltime']),
                (_('Users registered'), result['users_registered']),
                (_('Subtitles downloaded'), result['subs_downloads']),
                (_('Subtitles available'), result['subs_subtitle_files']),
                (_('Number movies'), result['movies_total']),
                (_('Number languages'), result['total_subtitles_languages']),
                (_('Client IP'), result['download_limits']['client_ip']),
                (_('24h global download limit'),
                 result['download_limits']['global_24h_download_limit']),
                (_('24h client download limit'),
                 result['download_limits']['client_24h_download_limit']),
                (_('24h client download count'),
                 result['download_limits']['client_24h_download_count']),
                (_('Client download quota'),
                 result['download_limits']['client_download_quota']),
            ]
        else:
            data = []
        return data

    @staticmethod
    def _languages_to_str(languages):
        if languages:
            lang_str = ','.join([language.xxx() for language in languages])
        else:
            lang_str = 'all'

        return lang_str

    @classmethod
    def get_name(cls):
        return 'opensubtitles'

    @classmethod
    def get_short_name(cls):
        return 'os'

    @classmethod
    def get_icon(cls):
        return ':/images/sites/opensubtitles.png'

    def _signal_connection_failed(self):
        # FIXME: set flag/... to signal users that the connection has failed
        pass

    def _safe_exec(self, query, default):
        self._ensure_connection()
        try:
            result = query()
            return result
        except (ProtocolError, CannotSendRequest, SocketError,
                ExpatError) as e:
            self._signal_connection_failed()
            log.debug('Query failed: {} {}'.format(type(e), e.args))
            return default

    STATUS_CODE_RE = re.compile(r'(\d+) (.+)')

    @classmethod
    def check_result(cls, data):
        log.debug('check_result(<data>)')
        if data is None:
            log.warning('data is None ==> FAIL')
            raise OpenSubtitlesProviderConnectionError(None, _('No message'))
        log.debug('checking presence of "status" in result ...')
        if 'status' not in data:
            log.debug('... no "status" in result ==> assuming SUCCESS')
            return
        log.debug('... FOUND')
        status = data['status']
        log.debug('result["status"]="{status}"'.format(status=status))
        log.debug('applying regex to status ...')
        try:
            code, message = cls.STATUS_CODE_RE.match(status).groups()
            log.debug('... regex SUCCEEDED')
            code = int(code)
        except (AttributeError, ValueError):
            log.debug('... regex FAILED')
            log.warning('Got unexpected status="{status}" from server.'.format(
                status=status))
            log.debug('Checking for presence of "200" ...')
            if '200' not in data['status']:
                log.debug('... FAIL. Raising ProviderConnectionError.')
                raise OpenSubtitlesProviderConnectionError(
                    None,
                    _('Server returned status="{status}". Expected "200 OK".').
                    format(status=data['status']), data['status'])
            log.debug('... SUCCESS')
            code, message = 200, 'OK'
        log.debug('Checking code={code} ...'.format(code=code))
        if code != 200:
            log.debug('... FAIL. Raising ProviderConnectionError.')
            raise OpenSubtitlesProviderConnectionError(code, message)
        log.debug('... SUCCESS.')
        log.debug('check_result() finished (data is ok)')