def login(self): try: r = self.session.post(self.server_url + 'login', json={"username": self.username, "password": self.password}, allow_redirects=False, timeout=10) except (ConnectionError, Timeout, ReadTimeout): raise ServiceUnavailable('Unknown Error, empty response: %s: %r' % (r.status_code, r)) else: if r.status_code == 200: try: self.token = r.json()['token'] except ValueError: raise ProviderError('Invalid JSON returned by provider') else: self.session.headers.update({'Authorization': 'Beaker ' + self.token}) region.set("oscom_token", self.token) return True elif r.status_code == 401: raise AuthenticationError('Login failed: {}'.format(r.reason)) elif r.status_code == 429: raise TooManyRequests() else: raise ProviderError('Bad status code: {}'.format(r.status_code)) finally: return False
def search_titles(self, title): title_id = None parameters = {'query': title.lower()} logging.debug('Searching using this title: {}'.format(title)) results = self.session.get(self.server_url + 'features', params=parameters, timeout=30) if results.status_code == 401: logging.debug( 'Authentification failed: clearing cache and attempting to login.' ) region.delete("oscom_token") self.login() results = self.session.get(self.server_url + 'features', params=parameters, timeout=30) if results.status_code == 429: raise TooManyRequests() elif results.status_code == 503: raise ProviderError(results.reason) elif results.status_code == 429: raise TooManyRequests() elif results.status_code == 503: raise ProviderError(results.reason) # deserialize results try: results_dict = results.json()['data'] except ValueError: raise ProviderError('Invalid JSON returned by provider') else: # loop over results for result in results_dict: if 'title' in result['attributes']: if isinstance(self.video, Episode): if fix_tv_naming(title).lower() == result['attributes']['title'].lower() and \ (not self.video.year or self.video.year == int(result['attributes']['year'])): title_id = result['id'] break else: if fix_movie_naming(title).lower() == result['attributes']['title'].lower() and \ (not self.video.year or self.video.year == int(result['attributes']['year'])): title_id = result['id'] break else: continue if title_id: logging.debug('Found this title ID: {}'.format(title_id)) return self.sanitize_external_ids(title_id) finally: if not title_id: logger.debug('No match found for {}'.format(title))
def download_subtitle(self, subtitle): r = self.session.get(subtitle.download_link, timeout=10) r.raise_for_status() # open the archive archive_stream = io.BytesIO(r.content) if is_rarfile(archive_stream): logger.debug('Archive identified as rar') archive = RarFile(archive_stream) elif is_zipfile(archive_stream): logger.debug('Archive identified as zip') archive = ZipFile(archive_stream) else: raise ProviderError('Unidentified archive type') # extract subtitle's content subs_in_archive = [] for name in archive.namelist(): for ext in (".srt", ".sub", ".ssa", ".ass"): if name.endswith(ext): subs_in_archive.append(name) # select the correct subtitle file matching_sub = None if len(subs_in_archive) == 1: matching_sub = subs_in_archive[0] else: for sub_name in subs_in_archive: guess = guessit(sub_name) # consider subtitle valid if: # - episode and season match # - format matches (if it was matched before) # - release group matches (and we asked for one and it was matched, or it was not matched) if guess["episode"] == subtitle.episode and guess[ "season"] == subtitle.season: format_matches = "format" not in subtitle.matches or \ ("format" in subtitle.matches and guess["format"].lower() in subtitle.releases.lower()) release_group_matches = True if subtitle.asked_for_release_group: release_group_matches = "release_group" not in subtitle.matches or \ ("release_group" in subtitle.matches and guess["release_group"].lower() == subtitle.asked_for_release_group.lower()) if release_group_matches and format_matches: matching_sub = sub_name break if not matching_sub: raise ProviderError("None of expected subtitle found in archive") subtitle.content = fix_line_ending(archive.read(matching_sub))
def download_subtitle(self, subtitle): if self.token is NO_VALUE: logger.debug("No cached token, we'll try to login again.") self.login() if self.token is NO_VALUE: logger.debug( "Unable to obtain an authentication token right now, we'll try again later." ) raise ProviderError("Unable to obtain an authentication token") logger.info('Downloading subtitle %r', subtitle) headers = { 'Accept': 'application/json', 'Content-Type': 'application/json', 'Authorization': 'Beaker ' + self.token } res = self.session.post(self.server_url + 'download', json={ 'file_id': subtitle.file_id, 'sub_format': 'srt' }, headers=headers, timeout=30) if res.status_code == 429: raise TooManyRequests() elif res.status_code == 406: raise DownloadLimitExceeded("Daily download limit reached") elif res.status_code == 503: raise ProviderError(res.reason) else: try: subtitle.download_link = res.json()['link'] except ValueError: raise ProviderError('Invalid JSON returned by provider') else: r = self.session.get(subtitle.download_link, timeout=30) if res.status_code == 429: raise TooManyRequests() elif res.status_code == 406: raise DownloadLimitExceeded("Daily download limit reached") elif res.status_code == 503: raise ProviderError(res.reason) subtitle_content = r.content if subtitle_content: subtitle.content = fix_line_ending(subtitle_content) else: logger.debug('Could not download subtitle from {}'.format( subtitle.download_link))
def download_subtitle(self, subtitle): r = self.session.get(subtitle.download_link, timeout=10) r.raise_for_status() # open the archive archive_stream = io.BytesIO(r.content) if is_rarfile(archive_stream): logger.debug('Archive identified as rar') archive = RarFile(archive_stream) elif is_zipfile(archive_stream): logger.debug('Archive identified as zip') archive = ZipFile(archive_stream) else: subtitle.content = r.content if subtitle.is_valid(): return subtitle.content = None raise ProviderError('Unidentified archive type') subs_in_archive = archive.namelist() # if Serbian lat and cyr versions are packed together, try to find right version if len(subs_in_archive) > 1 and (subtitle.language == 'sr' or subtitle.language == 'sr-Cyrl'): self.get_subtitle_from_bundled_archive(subtitle, subs_in_archive, archive) else: # use default method for everything else subtitle.content = self.get_subtitle_from_archive( subtitle, archive)
def download_subtitle(self, subtitle): logger.info('Downloading subtitle %r', subtitle) headers = {'Accept': 'application/json', 'Content-Type': 'application/json'} res = self.session.post(self.server_url + 'download', json={'file_id': subtitle.file_id, 'sub_format': 'srt'}, headers=headers, timeout=10) res.raise_for_status() if res.status_code == 429: raise TooManyRequests() elif res.status_code == 406: raise DownloadLimitExceeded("Daily download limit reached") else: try: subtitle.download_link = res.json()['link'] except ValueError: raise ProviderError('Invalid JSON returned by provider') else: r = self.session.get(subtitle.download_link, timeout=10) r.raise_for_status() if res.status_code == 429: raise TooManyRequests() elif res.status_code == 406: raise DownloadLimitExceeded("Daily download limit reached") subtitle_content = r.content if subtitle_content: subtitle.content = fix_line_ending(subtitle_content) else: logger.debug('Could not download subtitle from {}'.format(subtitle.download_link))
def download_subtitle(self, subtitle): r = self.session.get(subtitle.download_link, headers={'Referer': self.api_url}, timeout=10) r.raise_for_status() # open the archive archive_stream = io.BytesIO(r.content) if is_rarfile(archive_stream): logger.debug('Archive identified as rar') archive = RarFile(archive_stream) elif is_zipfile(archive_stream): logger.debug('Archive identified as zip') archive = ZipFile(archive_stream) else: subtitle.content = r.content if subtitle.is_valid(): return subtitle.content = None raise ProviderError('Unidentified archive type') if subtitle.is_episode: subtitle.content = self._get_subtitle_from_archive( subtitle, archive) else: subtitle.content = self.get_subtitle_from_archive( subtitle, archive)
def _get_show_ids(self): """Get the ``dict`` of show ids per series by querying the `series.php` page. :return: show id per series, lower case and without quotes. :rtype: dict """ # get the show page logger.info('Getting show ids') r = self.session.get(self.series_url, timeout=10) r.raise_for_status() if r.status_code != 200: logger.error('Error getting show ids') raise ProviderError('Error getting show ids') soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) # populate the show ids show_ids = {} for show in soup.select('td > a[href^="/show/"]'): show_ids[sanitize(show.get_text())] = int(show['href'][6:]) logger.debug('Found %d show ids', len(show_ids)) return show_ids
def download_subtitle(self, subtitle): # download url = self.server_url + 'subtitle/download/{}/{}/'.format(subtitle.language.alpha2, subtitle.subtitle_id) params = {'v': subtitle.subtitle_version, 'key': subtitle.subtitle_key} r = self.session.get(url, params=params, headers={'Referer': subtitle.page_link}, timeout=10) r.raise_for_status() # open the zip try: with zipfile.ZipFile(io.BytesIO(r.content)) as zf: # remove some filenames from the namelist namelist = [n for n in zf.namelist() if not n.endswith('.txt')] if len(namelist) > 1: raise ProviderError('More than one file to unzip') subtitle.content = fix_line_ending(zf.read(namelist[0])) except zipfile.BadZipfile: # if no zip file was retrieved, daily downloads limit has exceeded raise ProviderError('Daily limit exceeded')
def _search_url_titles(self, series, season, episode, year=None): """Search the URL titles by kind for the given `title`, `season` and `episode`. :param str series: series to search for. :param int season: season to search for. :param int episode: episode to search for. :param int year: year to search for. :return: the episode URL. :rtype: str """ # make the search logger.info('Searching episode url for %s, season %d, episode %d', series, season, episode) episode_url = None search = '{} {}x{}'.format(series, season, episode) r = self.session.get(self.search_url, headers={'Referer': self.server_url}, params={'q': search}, timeout=10) r.raise_for_status() if r.status_code != 200: logger.warning('Error getting episode url') raise ProviderError('%s: Error getting episode url', self.__class__.__name__.upper()) results = json.loads(r.text) for result in results: title = sanitize(result['name']) # attempt series with year if sanitize('{} ({})'.format(series, year)) in title: for episode_data in result['episodes']: if season == episode_data[ 'season'] and episode == episode_data['number']: episode_url = self.server_url + 'episodes/{}'.format( episode_data['id']) logger.info('Episode url found with year %s', episode_url) return episode_url # attempt series without year elif sanitize(series) in title: for episode_data in result['episodes']: if season == episode_data[ 'season'] and episode == episode_data['number']: episode_url = self.server_url + 'episodes/{}'.format( episode_data['id']) logger.info('Episode url found without year %s', episode_url) return episode_url return episode_url
def search_titles(self, title): title_id = None imdb_id = None if isinstance(self.video, Episode) and self.video.series_imdb_id: imdb_id = self.video.series_imdb_id elif isinstance(self.video, Movie) and self.video.imdb_id: imdb_id = self.video.imdb_id if imdb_id: parameters = {'imdb_id': imdb_id} logging.debug('Searching using this IMDB id: {}'.format(imdb_id)) else: parameters = {'query': title} logging.debug('Searching using this title: {}'.format(title)) results = self.session.get(self.server_url + 'features', params=parameters, timeout=10) results.raise_for_status() if results.status_code == 401: logging.debug('Authentification failed: clearing cache and attempting to login.') region.delete("oscom_token") self.login() results = self.session.get(self.server_url + 'features', params=parameters, timeout=10) results.raise_for_status() if results.status_code == 429: raise TooManyRequests() elif results.status_code == 429: raise TooManyRequests() # deserialize results try: results_dict = results.json()['data'] except ValueError: raise ProviderError('Invalid JSON returned by provider') else: # loop over results for result in results_dict: if title.lower() == result['attributes']['title'].lower() and \ (not self.video.year or self.video.year == int(result['attributes']['year'])): title_id = result['id'] break if title_id: logging.debug('Found this title ID: {}'.format(title_id)) return title_id finally: if not title_id: logger.debug('No match found for {}'.format(title))
def download_subtitle(self, subtitle): # download url = 'http://zip.{}/{}.zip'.format(self.server_url, subtitle.subtitle_id) r = self.session.get(url, headers={'Referer': subtitle.page_link}, timeout=10) r.raise_for_status() # open the zip with zipfile.ZipFile(io.BytesIO(r.content)) as zf: # remove some filenames from the namelist namelist = [n for n in zf.namelist() if os.path.splitext(n)[1] in ['.srt', '.sub']] if len(namelist) > 1: raise ProviderError('More than one file to unzip') subtitle.content = fix_line_ending(zf.read(namelist[0]))
def _search_url_titles(self, series, season, episode, year=None): """Search the URL titles by kind for the given `title`, `season` and `episode`. :param str series: series to search for. :param int season: season to search for. :param int episode: episode to search for. :param int year: year to search for. :return: the episode URL. :rtype: str """ # make the search logger.info("Searching episode url for %s, season %d, episode %d", series, season, episode) episode_url = None search = "{} {}x{}".format(series, season, episode) r = self.session.get(self.search_url, headers={"Referer": self.server_url}, params={"q": search}, timeout=10) r.raise_for_status() if r.status_code != 200: logger.error("Error getting episode url") raise ProviderError("Error getting episode url") results = json.loads(r.text) for result in results: title = sanitize(result["name"]) # attempt series with year if sanitize("{} ({})".format(series, year)) in title: for episode_data in result["episodes"]: if season == episode_data[ "season"] and episode == episode_data["number"]: episode_url = self.server_url + "episodes/{}".format( episode_data["id"]) return episode_url # attempt series without year elif sanitize(series) in title: for episode_data in result["episodes"]: if season == episode_data[ "season"] and episode == episode_data["number"]: episode_url = self.server_url + "episodes/{}".format( episode_data["id"]) return episode_url return episode_url
def fetch_page(self, url, ref=None): logger.debug(f"Titulky.com: Fetching url: {url}") res = self.session.get( url, timeout=self.timeout, headers={'Referer': ref if ref else self.server_url}) if res.status_code != 200: raise HTTPError(f"Fetch failed with status code {res.status_code}") if not res.text: raise ProviderError("No response returned from the provider") return res.text
def download_subtitle(self, subtitle): r = self.session.get(subtitle.download_link, timeout=10) r.raise_for_status() # open the archive archive_stream = io.BytesIO(r.content) if is_rarfile(archive_stream): logger.debug('Archive identified as rar') archive = RarFile(archive_stream) elif is_zipfile(archive_stream): logger.debug('Archive identified as zip') archive = ZipFile(archive_stream) else: raise ProviderError('Unidentified archive type') subtitle.content = self.get_subtitle_from_archive(subtitle, archive)
def download_subtitle(self, subtitle): if self.token is NO_VALUE: logger.debug("No cached token, we'll try to login again.") self.login() if self.token is NO_VALUE: logger.debug( "Unable to obtain an authentication token right now, we'll try again later." ) raise ProviderError("Unable to obtain an authentication token") logger.info('Downloading subtitle %r', subtitle) headers = { 'Accept': 'application/json', 'Content-Type': 'application/json', 'Authorization': 'Beaker ' + self.token } res = self.retry(lambda: checked(lambda: self.session.post( self.server_url + 'download', json={ 'file_id': subtitle.file_id, 'sub_format': 'srt' }, headers=headers, timeout=30), validate_json=True, json_key_name='link'), amount=retry_amount) download_data = res.json() subtitle.download_link = download_data['link'] r = self.retry(lambda: checked(lambda: self.session.get( subtitle.download_link, timeout=30), validate_content=True), amount=retry_amount) if not r: logger.debug( f'Could not download subtitle from {subtitle.download_link}') subtitle.content = None return else: subtitle_content = r.content subtitle.content = fix_line_ending(subtitle_content)
def download_subtitle(self, subtitle): # download the subtitle logger.info('Downloading subtitle %r', subtitle.sub_id) params = { 'mac': subtitle.sub_id } r = self.session.get(self.server_url + '/subtitle/download', params=params, timeout=30) r.raise_for_status() # open the zip with zipfile.ZipFile(io.BytesIO(r.content)) as zf: # remove some filenames from the namelist namelist = [n for n in zf.namelist() if os.path.splitext(n)[1] in ['.srt', '.sub']] if len(namelist) > 1: raise ProviderError('More than one file to unzip') subtitle.content = fix_line_ending(zf.read(namelist[0]))
def download_subtitle(self, subtitle): if isinstance(subtitle, NekurSubtitle): # download the subtitle r = self.session.get(subtitle.download_link, timeout=10) r.raise_for_status() # open the archive archive_stream = io.BytesIO(r.content) if is_rarfile(archive_stream): archive = RarFile(archive_stream) elif is_zipfile(archive_stream): archive = ZipFile(archive_stream) else: subtitle.content = r.content if subtitle.is_valid(): return subtitle.content = None raise ProviderError('Unidentified archive type') subtitle.content = self.get_subtitle_from_archive(subtitle, archive)
def download_subtitle(self, subtitle): # download url = self.server_url + 'get/download/{}/'.format( subtitle.language.alpha2) params = { 'id': subtitle.subtitle_id, 'v': subtitle.releases[0], 'key': subtitle.subtitle_key } r = self.session.get(url, params=params, headers={'Referer': subtitle.page_link}, timeout=10) r.raise_for_status() # open the zip with zipfile.ZipFile(io.BytesIO(r.content)) as zf: # remove some filenames from the namelist namelist = [n for n in zf.namelist() if not n.endswith('.txt')] if len(namelist) > 1: raise ProviderError('More than one file to unzip') subtitle.content = fix_line_ending(zf.read(namelist[0]))
def query(self, language, video_names, type, keyword=None, year=None, season=None, episode=None, imdb_id=None): ## Build the search URL params = {} # Keyword if keyword: params['Fulltext'] = keyword # Video type if type == 'episode': params['Serial'] = 'S' else: params['Serial'] = 'F' # Season / Episode if season: params['Sezona'] = season if episode: params['Epizoda'] = episode # IMDB ID if imdb_id: params['IMDB'] = imdb_id[2:] # Remove the tt from the imdb id # Year if year: params['Rok'] = year # Language if language == Language('ces'): params['Jazyk'] = 'CZ' elif language == Language('slk'): params['Jazyk'] = 'SK' elif language == None: params['Jazyk'] = '' else: return [] # Status if self.approved_only: logger.debug(f"Titulky.com: Searching only for approved subtitles") params['ASchvalene'] = '1' else: params['ASchvalene'] = '' search_url = self.build_search_url(params) ## Search results page parsing html_src = self.fetch_page(search_url) search_page_soup = ParserBeautifulSoup(html_src, ['lxml', 'html.parser']) # If there is a message containing "Žádny odpovídající záznam", it means that there are no results # If that's the case, return an empty list error_message = search_page_soup.select('.panel-body > strong') if len( error_message ) > 0 and 'Žádný odpovídající záznam' in error_message[0].get_text( strip=True): logger.info("Titulky.com: No results found") return [] # Get the table containing the search results table = search_page_soup.find('table', class_='table') if not table: logger.debug("Titulky.com: Could not find table") raise ParseResponseError( "Could not find table. Did the HTML source change?") # Get table body containing rows of subtitles table_body = table.find('tbody') if not table_body: logger.debug("Titulky.com: Could not find table body") raise ParseResponseError( "Could not find table body. Did the HTML source change?") ## Loop over all subtitles on the first page and put them in a list subtitles = [] rows = table_body.find_all('tr') if not self.multithreading: # Process the rows sequentially logger.info("Titulky.com: processing results in sequence") for i, row in enumerate(rows): sub_info = self.process_row(row, video_names, search_url) # If subtitle info was returned, then everything was okay # and we can instationate it and add it to the list if sub_info: logger.debug( f"Titulky.com: Sucessfully retrieved subtitle info, row: {i}" ) # If we found the subtitle by IMDB ID, no need to get it from details page sub_imdb_id = imdb_id or sub_info['imdb_id'] subtitle_instance = self.subtitle_class( sub_info['id'], sub_imdb_id, sub_info['language'], sub_info['names'], season, episode, sub_info['year'], sub_info['releases'], sub_info['fps'], sub_info['uploader'], sub_info['approved'], sub_info['details_link'], sub_info['download_link'], skip_wrong_fps=self.skip_wrong_fps, asked_for_episode=(type == 'episode')) subtitles.append(subtitle_instance) else: # No subtitle info was returned, i. e. something unexpected # happend during subtitle details page fetching and processing. logger.debug( f"Titulky.com: No subtitle info retrieved, row: {i}") else: # Process the rows in paralell logger.info( f"Titulky.com: processing results in parelell, {self.max_threads} rows at a time." ) threads = [None] * len(rows) threads_data = [None] * len(rows) # Process rows in parallel, self.max_threads at a time. cycles = math.ceil(len(rows) / self.max_threads) for i in range(cycles): # Batch number i starting_index = i * self.max_threads # Inclusive ending_index = starting_index + self.max_threads # Non-inclusive # Create threads for all rows in this batch for j in range(starting_index, ending_index): # Check if j-th row exists if j < len(rows): # Row number j logger.debug( f"Titulky.com: Creating thread {j} (batch: {i})") # Create a thread for row j and start it threads[j] = Thread( target=self.process_row, args=[rows[j], video_names, search_url], kwargs={ 'thread_id': j, 'threads_data': threads_data }) threads[j].start() # Wait for all created threads to finish before moving to another batch of rows for j in range(starting_index, ending_index): # Check if j-th row exists if j < len(rows): threads[j].join() # Process the resulting data from all threads for i in range(len(threads_data)): thread_data = threads_data[i] # If the thread returned didn't return anything, but expected a dict object if not thread_data: raise ProviderError( f"No data returned from thread ID: {i}") # If an exception was raised in a thread, raise it again here if 'exception' in thread_data and thread_data['exception']: logger.debug( f"Titulky.com: An error occured while processing a row in the thread ID {i}" ) raise thread_data['exception'] # If the thread returned a subtitle info, great, instantiate it and add it to the list if 'sub_info' in thread_data and thread_data['sub_info']: # Instantiate the subtitle object logger.debug( f"Titulky.com: Sucessfully retrieved subtitle info, thread ID: {i}" ) sub_info = thread_data['sub_info'] # If we found the subtitle by IMDB ID, no need to get it from details page sub_imdb_id = imdb_id or sub_info['imdb_id'] subtitle_instance = self.subtitle_class( sub_info['id'], sub_imdb_id, sub_info['language'], sub_info['names'], season, episode, sub_info['year'], sub_info['releases'], sub_info['fps'], sub_info['uploader'], sub_info['approved'], sub_info['details_link'], sub_info['download_link'], skip_wrong_fps=self.skip_wrong_fps, asked_for_episode=(type == 'episode')) subtitles.append(subtitle_instance) else: # The thread returned data, but it didn't contain a subtitle info, i. e. something unexpected # happend during subtitle details page fetching and processing. logger.debug( f"Titulky.com: No subtitle info retrieved, thread ID: {i}" ) # Clean up search_page_soup.decompose() search_page_soup = None logger.debug(f"Titulky.com: Found subtitles: {subtitles}") return subtitles
def get_archives(self, title_id, language_code): """Get the archive list from a given `title_id` and `language_code`. :param int title_id: title id. :param int language_code: language code. :return: the archives. :rtype: list of :class:`LegendasTVArchive` """ logger.info('Getting archives for title %d and language %d', title_id, language_code) archives = [] page = 0 while True: # get the archive page url = self.server_url + 'legenda/busca/-/{language}/-/{page}/{title}'.format( language=language_code, page=page, title=title_id) r = self.session.get(url) r.raise_for_status() # parse the results soup = ParserBeautifulSoup(r.content, ['lxml', 'html.parser']) for archive_soup in soup.select('div.list_element > article > div > div.f_left'): # create archive archive = LegendasTVArchive(archive_soup.a['href'].split('/')[2], archive_soup.a.text, 'pack' in archive_soup.parent['class'], 'destaque' in archive_soup.parent['class'], self.server_url + archive_soup.a['href'][1:]) # extract text containing downloads, rating and timestamp data_text = archive_soup.find('p', class_='data').text # match downloads archive.downloads = int(downloads_re.search(data_text).group('downloads')) # match rating match = rating_re.search(data_text) if match: archive.rating = int(match.group('rating')) # match timestamp and validate it time_data = {k: int(v) for k, v in timestamp_re.search(data_text).groupdict().items()} archive.timestamp = pytz.timezone('America/Sao_Paulo').localize(datetime(**time_data)) if archive.timestamp > datetime.utcnow().replace(tzinfo=pytz.utc): raise ProviderError('Archive timestamp is in the future') # add archive logger.info('Found archive for title %d and language %d at page %s: %s', title_id, language_code, page, archive) archives.append(archive) # stop on last page if soup.find('a', attrs={'class': 'load_more'}, string='carregar mais') is None: break # increment page count page += 1 logger.debug('Found %d archives', len(archives)) return archives
def checked(fn, raise_api_limit=False, validate_token=False, validate_json=False, json_key_name=None, validate_content=False): """Run :fn: and check the response status before returning it. :param fn: the function to make an API call to OpenSubtitles.com. :param raise_api_limit: if True we wait a little bit longer before running the call again. :param validate_token: test if token is valid and return 401 if not. :param validate_json: test if response is valid json. :param json_key_name: test if returned json contain a specific key. :param validate_content: test if response have a content (used with download). :return: the response. """ response = None try: try: response = fn() except APIThrottled: if not raise_api_limit: logger.info( "API request limit hit, waiting and trying again once.") time.sleep(2) return checked(fn, raise_api_limit=True) raise except (ConnectionError, Timeout, ReadTimeout): raise ServiceUnavailable( f'Unknown Error, empty response: {response.status_code}: {response}' ) except Exception: logging.exception('Unhandled exception raised.') raise ProviderError('Unhandled exception raised. Check log.') else: status_code = response.status_code except Exception: status_code = None else: if status_code == 401: if validate_token: return 401 else: raise AuthenticationError(f'Login failed: {response.reason}') elif status_code == 403: raise ProviderError("Bazarr API key seems to be in problem") elif status_code == 406: raise DownloadLimitExceeded("Daily download limit reached") elif status_code == 410: raise ProviderError("Download as expired") elif status_code == 429: raise TooManyRequests() elif status_code == 502: # this one should deal with Bad Gateway issue on their side. raise APIThrottled() elif 500 <= status_code <= 599: raise ProviderError(response.reason) if status_code != 200: raise ProviderError(f'Bad status code: {response.status_code}') if validate_json: try: json_test = response.json() except JSONDecodeError: raise ProviderError('Invalid JSON returned by provider') else: if json_key_name not in json_test: raise ProviderError( f'Invalid JSON returned by provider: no {json_key_name} key in returned json.' ) if validate_content: if not hasattr(response, 'content'): logging.error('Download link returned no content attribute.') return False elif not response.content: logging.error( f'This download link returned empty content: {response.url}' ) return False return response
def query(self, languages, video): self.video = video if self.use_hash: hash = self.video.hashes.get('opensubtitlescom') logging.debug('Searching using this hash: {}'.format(hash)) else: hash = None if isinstance(self.video, Episode): title = self.video.series else: title = self.video.title title_id = self.search_titles(title) if not title_id: return [] lang_strings = [str(lang) for lang in languages] langs = ','.join(lang_strings) logging.debug('Searching for this languages: {}'.format(lang_strings)) # query the server if isinstance(self.video, Episode): res = self.session.get(self.server_url + 'subtitles', params={'parent_feature_id': title_id, 'languages': langs, 'episode_number': self.video.episode, 'season_number': self.video.season, 'moviehash': hash}, timeout=10) else: res = self.session.get(self.server_url + 'subtitles', params={'id': title_id, 'languages': langs, 'moviehash': hash}, timeout=10) res.raise_for_status() if res.status_code == 429: raise TooManyRequests() subtitles = [] try: result = res.json() except ValueError: raise ProviderError('Invalid JSON returned by provider') else: logging.debug('Query returned {} subtitles'.format(len(result['data']))) if len(result['data']): for item in result['data']: if 'season_number' in item['attributes']['feature_details']: season_number = item['attributes']['feature_details']['season_number'] else: season_number = None if 'episode_number' in item['attributes']['feature_details']: episode_number = item['attributes']['feature_details']['episode_number'] else: episode_number = None if 'moviehash_match' in item['attributes']: moviehash_match = item['attributes']['moviehash_match'] else: moviehash_match = False if len(item['attributes']['files']): subtitle = OpenSubtitlesComSubtitle( language=Language.fromietf(item['attributes']['language']), hearing_impaired=item['attributes']['hearing_impaired'], page_link=item['attributes']['url'], file_id=item['attributes']['files'][0]['file_id'], releases=item['attributes']['release'], uploader=item['attributes']['uploader']['name'], title=item['attributes']['feature_details']['movie_name'], year=item['attributes']['feature_details']['year'], season=season_number, episode=episode_number, hash_matched=moviehash_match ) subtitle.get_matches(self.video) subtitles.append(subtitle) return subtitles
def query(self, languages, video): self.video = video if self.use_hash: file_hash = self.video.hashes.get('opensubtitlescom') logging.debug('Searching using this hash: {}'.format(hash)) else: file_hash = None if isinstance(self.video, Episode): title = self.video.series else: title = self.video.title imdb_id = None if isinstance(self.video, Episode) and self.video.series_imdb_id: imdb_id = self.sanitize_external_ids(self.video.series_imdb_id) elif isinstance(self.video, Movie) and self.video.imdb_id: imdb_id = self.sanitize_external_ids(self.video.imdb_id) title_id = None if not imdb_id: title_id = self.search_titles(title) if not title_id: return [] lang_strings = [str(lang.basename) for lang in languages] only_foreign = all([lang.forced for lang in languages]) also_foreign = any([lang.forced for lang in languages]) if only_foreign: forced = 'only' elif also_foreign: forced = 'include' else: forced = 'exclude' langs = ','.join(lang_strings) logging.debug('Searching for this languages: {}'.format(lang_strings)) # query the server if isinstance(self.video, Episode): res = self.session.get( self.server_url + 'subtitles', params=(('episode_number', self.video.episode), ('foreign_parts_only', forced), ('languages', langs.lower()), ('moviehash', file_hash), ('parent_feature_id', title_id) if title_id else ('imdb_id', imdb_id), ('season_number', self.video.season), ('query', os.path.basename(self.video.name))), timeout=30) else: res = self.session.get( self.server_url + 'subtitles', params=(('foreign_parts_only', forced), ('id', title_id) if title_id else ('imdb_id', imdb_id), ('languages', langs.lower()), ('moviehash', file_hash), ('query', os.path.basename(self.video.name))), timeout=30) if res.status_code == 429: raise TooManyRequests() elif res.status_code == 503: raise ProviderError(res.reason) subtitles = [] try: result = res.json() if 'data' not in result: raise ValueError except ValueError: raise ProviderError('Invalid JSON returned by provider') else: logging.debug('Query returned {} subtitles'.format( len(result['data']))) if len(result['data']): for item in result['data']: if 'season_number' in item['attributes'][ 'feature_details']: season_number = item['attributes']['feature_details'][ 'season_number'] else: season_number = None if 'episode_number' in item['attributes'][ 'feature_details']: episode_number = item['attributes']['feature_details'][ 'episode_number'] else: episode_number = None if 'moviehash_match' in item['attributes']: moviehash_match = item['attributes']['moviehash_match'] else: moviehash_match = False if len(item['attributes']['files']): subtitle = OpenSubtitlesComSubtitle( language=Language.fromietf( item['attributes']['language']), forced=item['attributes']['foreign_parts_only'], hearing_impaired=item['attributes'] ['hearing_impaired'], page_link=item['attributes']['url'], file_id=item['attributes']['files'][0]['file_id'], releases=item['attributes']['release'], uploader=item['attributes']['uploader']['name'], title=item['attributes']['feature_details'] ['movie_name'], year=item['attributes']['feature_details']['year'], season=season_number, episode=episode_number, hash_matched=moviehash_match) subtitle.get_matches(self.video) subtitles.append(subtitle) return subtitles