def search(self, search_strings, age=0, ep_obj=None, **kwargs): results = [] if not self.login(): return results freeleech = '&free=on' if self.freeleech else '' for mode in search_strings: sickrage.app.log.debug("Search Mode: %s" % mode) for search_string in search_strings[mode]: if mode != 'RSS': sickrage.app.log.debug("Search string: %s " % search_string) # URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile search_url = self.urls['search'] % (self.categories, freeleech, search_string) search_url += ';o=seeders' if mode != 'RSS' else '' if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1]) try: data = self.session.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug("No data returned from provider") return results
def search(self, search_strings, age=0, ep_obj=None): results = [] search_params = { 'out': 'json', 'filter': 2101, 'showmagnets': 'on', 'num': 50 } for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode != 'RSS': sickrage.app.log.debug("Search string: {0}".format (search_string)) search_params['s'] = search_string search_url = self.urls['base_url'] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url set, please check your settings") return results search_url = self.custom_url try: data = sickrage.app.wsession.get(search_url, params=search_params).json() results += self.parse(data, mode) except Exception: sickrage.app.log.debug("No data returned from provider") return results
def cleanup_provider_cache(self): for item in self.all('providers'): if int(item["quality"]) == Quality.UNKNOWN: self.delete(item) elif not validate_url(item["url"]) and not item["url"].startswith("magnet") \ or is_ip_private(item["url"].split(r'//')[-1].split(r'/')[0]): self.delete(item)
def search(self, search_strings, age=0, ep_obj=None, **kwargs): # pylint: disable=too-many-locals results = [] for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode == 'Season': search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string) if mode != 'RSS': sickrage.app.log.debug("Search string: {}".format(search_string)) search_query = re.sub(r'\W', '-', search_string) search_url = urljoin(self.urls['search'], "{search_query}.html".format(search_query=search_query)) else: search_url = self.urls['rss'] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1]) try: data = self.session.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug("No data returned from provider") return results
def test_result_values(self): """Check that the provider returns results in proper format""" results = self.provider.search(self.search_strings('Episode'), ep_obj=self.ep) for result in results: self.assertIsInstance(result, dict) self.assertEqual(sorted(result.keys()), ['leechers', 'link', 'seeders', 'size', 'title']) self.assertIsInstance(result['title'], six.text_type) self.assertIsInstance(result['link'], six.text_type) self.assertIsInstance(result['seeders'], six.integer_types) self.assertIsInstance(result['leechers'], six.integer_types) self.assertIsInstance(result['size'], six.integer_types) self.assertTrue(len(result['title'])) self.assertTrue(len(result['link'])) self.assertTrue(result['seeders'] >= 0) self.assertTrue(result['leechers'] >= 0) self.assertTrue(result['size'] >= -1) if result['link'].startswith('magnet'): self.assertTrue(magnet_regex.match(result['link'])) else: self.assertTrue(validate_url(result['link'])) self.assertIsInstance(self.provider._get_size(result), six.integer_types) self.assertTrue(all(self.provider._get_title_and_url(result))) self.assertTrue(self.provider._get_size(result))
def test_result_values(self): """Check that the provider returns results in proper format""" results = self.provider.search(self.search_strings('Episode'), ep_obj=self.ep) for result in results: self.assertIsInstance(result, dict) self.assertEqual( sorted(result.keys()), ['leechers', 'link', 'seeders', 'size', 'title']) self.assertIsInstance(result['title'], six.text_type) self.assertIsInstance(result['link'], six.text_type) self.assertIsInstance(result['seeders'], six.integer_types) self.assertIsInstance(result['leechers'], six.integer_types) self.assertIsInstance(result['size'], six.integer_types) self.assertTrue(len(result['title'])) self.assertTrue(len(result['link'])) self.assertTrue(result['seeders'] >= 0) self.assertTrue(result['leechers'] >= 0) self.assertTrue(result['size'] >= -1) if result['link'].startswith('magnet'): self.assertTrue(magnet_regex.match(result['link'])) else: self.assertTrue(validate_url(result['link'])) self.assertIsInstance(self.provider._get_size(result), six.integer_types) self.assertTrue(all(self.provider._get_title_and_url(result))) self.assertTrue(self.provider._get_size(result))
def search(self, search_strings, age=0, ep_obj=None, **kwargs): results = [] for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: search_url = (self.urls["search"], self.urls["rss"])[mode == "RSS"] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning( "Invalid custom url: {0}".format(self.custom_url)) return results search_url = urljoin( self.custom_url, search_url.split(self.urls['base_url'])[1]) if mode != "RSS": search_url = search_url % search_string sickrage.app.log.debug( "Search string: {}".format(search_string)) try: data = self.session.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug("No data returned from provider") return results
def search(self, search_strings, age=0, ep_obj=None, **kwargs): results = [] for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode != 'RSS': sickrage.app.log.debug("Search string: {}".format(search_string)) search_url = urljoin(self.urls['search'], "{search_query}".format(search_query=search_string)) else: search_url = self.urls['rss'] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1]) try: data = self.session.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug("No data returned from provider") return results
def parse(self, data, mode, **kwargs): """ Parse search results from data :param data: response data :param mode: search mode :return: search results """ results = [] with bs4_parser(data) as html: table_body = html.find('tbody') # Continue only if at least one release is found if not table_body: sickrage.app.log.debug('Data returned from provider does not contain any torrents') return results for row in table_body('tr'): cells = row('td') if len(cells) < 4: continue try: title = download_url = None info_cell = cells[0].a if info_cell: title = info_cell.get_text() download_url = self._get_download_link(urljoin(self.urls['base_url'], info_cell.get('href'))) if not all([title, download_url]): continue title = '{name} {codec}'.format(name=title, codec='x264') if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url)) return results download_url = urljoin(self.custom_url, download_url.split(self.urls['base_url'])[1]) seeders = try_int(cells[2].get_text(strip=True)) leechers = try_int(cells[3].get_text(strip=True)) torrent_size = cells[1].get_text() size = convert_size(torrent_size, -1) results += [{ 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers }] if mode != 'RSS': sickrage.app.log.debug("Found result: {}".format(title)) except Exception: sickrage.app.log.error("Failed parsing provider") return results
def search(self, search_strings, age=0, ep_obj=None): results = [] search_url = self.urls["search"] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {0}".format(self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1]) # Search Params search_params = { 'category': 'anime' if ep_obj and ep_obj.show and ep_obj.show.anime else 'tv', 'apiKey': self.api_key, } for mode in search_strings: sickrage.app.log.debug('Search mode: {}'.format(mode)) for search_string in search_strings[mode]: search_params['q'] = search_string if mode != 'RSS': sickrage.app.log.debug('Search string: {}'.format(search_string)) try: data = sickrage.app.wsession.get(search_url, params=search_params).json() results += self.parse(data, mode) except Exception: sickrage.app.log.debug('No data returned from provider') return results
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs): results = [] if not self.login(): return results freeleech = '&free=on' if self.freeleech else '' for mode in search_strings: sickrage.app.log.debug("Search Mode: %s" % mode) for search_string in search_strings[mode]: if mode != 'RSS': sickrage.app.log.debug("Search string: %s " % search_string) # URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile search_url = self.urls['search'] % (self.categories, freeleech, search_string) search_url += ';o=seeders' if mode != 'RSS' else '' if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1]) resp = self.session.get(search_url) if not resp or not resp.text: sickrage.app.log.debug("No data returned from provider") continue results += self.parse(resp.text, mode) return results
def addCacheEntry(self, name, url, seeders, leechers, size): # check for existing entry in cache if len([x for x in sickrage.app.cache_db.get_many('providers', self.providerID) if x['url'] == url]): return # ignore invalid urls if not validate_url(url) and not url.startswith('magnet') \ or is_ip_private(url.split(r'//')[-1].split(r'/')[0]): return try: # parse release name parse_result = NameParser(validate_show=sickrage.app.config.enable_rss_cache_valid_shows).parse(name) if parse_result.series_name and parse_result.quality != Quality.UNKNOWN: season = parse_result.season_number if parse_result.season_number else 1 episodes = parse_result.episode_numbers if season and episodes: # store episodes as a seperated string episodeText = "|" + "|".join(map(str, episodes)) + "|" # get quality of release quality = parse_result.quality # get release group release_group = parse_result.release_group # get version version = parse_result.version dbData = { '_t': 'providers', 'provider': self.providerID, 'name': name, 'season': season, 'episodes': episodeText, 'indexerid': parse_result.indexerid, 'url': url, 'time': int(time.mktime(datetime.datetime.today().timetuple())), 'quality': quality, 'release_group': release_group, 'version': version, 'seeders': seeders, 'leechers': leechers, 'size': size } # add to internal database sickrage.app.cache_db.insert(dbData) # add to external provider cache database if sickrage.app.config.enable_api_providers_cache and not self.provider.private: try: sickrage.app.api.add_provider_cache_result(dbData) except Exception: pass sickrage.app.log.debug("SEARCH RESULT:[%s] ADDED TO CACHE!", name) except (InvalidShowException, InvalidNameException): pass
def search(self, search_strings, age=0, series_id=None, series_provider_id=None, season=None, episode=None, **kwargs): results = [] for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: search_url = (self.urls["search"], self.urls["rss"])[mode == "RSS"] if self.custom_settings['custom_url']: if not validate_url(self.custom_settings['custom_url']): sickrage.app.log.warning("Invalid custom url: {0}".format(self.custom_settings['custom_url'])) return results search_url = urljoin(self.custom_settings['custom_url'], search_url.split(self.urls['base_url'])[1]) if mode != "RSS": search_url = search_url % search_string sickrage.app.log.debug("Search string: {}".format(search_string)) resp = self.session.get(search_url) if not resp or not resp.text: sickrage.app.log.debug("No data returned from provider") continue results += self.parse(resp.text, mode) return results
def search(self, search_strings, age=0, ep_obj=None, **kwargs): # pylint: disable=too-many-locals results = [] for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode == 'Season': search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string) if mode != 'RSS': sickrage.app.log.debug("Search string: {0}".format (search_string)) search_string = search_string.replace('.', '-').replace(' ', '-') search_url = urljoin(self.urls['search'], "{search_string}.html".format(search_string=search_string)) else: search_url = self.urls['rss'] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {0}".format(self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1]) try: data = self.session.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug("No data returned from provider") return results
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs): results = [] for mode in search_strings: sickrage.app.log.debug("Search Mode: {}".format(mode)) for search_string in search_strings[mode]: if mode != "RSS": sickrage.app.log.debug( "Search string: {0}".format(search_string)) search_url = (self.urls["base_url"], self.urls["rss"])[mode == "RSS"] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning( "Invalid custom url: {}".format(self.custom_url)) return results search_url = urljoin( self.custom_url, search_url.split(self.urls['base_url'])[1]) if mode != "RSS": search_params = { 'query': search_string, 'sort': ('seeders', 'created')[mode == 'RSS'], 'type': 'video', 'tag': 'hd', 'category': 'show' } else: search_params = { 'category': 'show', 'type': 'video', 'sort': 'created' } resp = self.session.get(search_url, params=search_params) if not resp or not resp.text: sickrage.app.log.debug("No data returned from provider") continue results += self.parse(resp.text, mode) return results
def search(self, search_strings, age=0, series_id=None, series_provider_id=None, season=None, episode=None, **kwargs): results = [] search_url = self.urls["search"] if self.custom_settings['custom_url']: if not validate_url(self.custom_settings['custom_url']): sickrage.app.log.warning("Invalid custom url: {0}".format( self.custom_settings['custom_url'])) return results search_url = urljoin(self.custom_settings['custom_url'], search_url.split(self.urls['base_url'])[1]) show_object = find_show(series_id, series_provider_id) # Search Params search_params = { 'category': ("tv", "anime")[bool(show_object.anime)], 'apiKey': self.custom_settings['api_key'], } for mode in search_strings: sickrage.app.log.debug('Search mode: {}'.format(mode)) for search_string in search_strings[mode]: search_params['q'] = search_string if mode != 'RSS': sickrage.app.log.debug( 'Search string: {}'.format(search_string)) resp = self.session.get(search_url, params=search_params) if not resp or not resp.content: sickrage.app.log.debug('No data returned from provider') continue try: data = resp.json() except ValueError: sickrage.app.log.debug('No data returned from provider') continue results += self.parse(data, mode) return results
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs): results = [] """ sorting ss: relevance ed: seeds desc ea: seeds asc pd: peers desc pa: peers asc sd: big > small sa: small > big ad: added desc (latest) aa: added asc (oldest) """ for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode != "RSS": sickrage.app.log.debug( "Search string: {0}".format(search_string)) search_url = self.urls["search"] % ( ("ed", "ad")[mode == "RSS"], 1, search_string) if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning( "Invalid custom url: {0}".format(self.custom_url)) return results search_url = urljoin( self.custom_url, search_url.split(self.urls['base_url'])[1]) try: data = self.session.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug( "URL did not return results/data, if the results are on the site maybe try " "a custom url, or a different one") return results
def find_propers(self, show_id, season, episode): results = [] for term in self.proper_strings: search_strngs = self._get_episode_search_strings(show_id, season, episode, add_string=term) for item in self.search(search_strngs[0], show_id=show_id, season=season, episode=episode): result = self.getResult(season, [episode]) result.name, result.url = self._get_title_and_url(item) if not validate_url(result.url) and not result.url.startswith('magnet'): continue result.seeders, result.leechers = self._get_result_stats(item) result.size = self._get_size(item) result.date = datetime.datetime.today() results.append(result) return results
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs): results = [] for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode == 'Season': search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string) if mode != 'RSS': sickrage.app.log.debug( "Search string: {}".format(search_string)) search_query = re.sub(r'\W', '-', search_string) search_url = urljoin( self.urls['search'], "{search_query}.html".format( search_query=search_query)) else: search_url = self.urls['rss'] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning( "Invalid custom url: {}".format(self.custom_url)) return results search_url = urljoin( self.custom_url, search_url.split(self.urls['base_url'])[1]) try: data = self.session.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug("No data returned from provider") return results
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs): results = [] search_params = { 'out': 'json', 'filter': 2101, 'showmagnets': 'on', 'num': 50 } for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode != 'RSS': sickrage.app.log.debug("Search string: {0}".format (search_string)) search_params['s'] = search_string search_url = self.urls['base_url'] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url set, please check your settings") return results search_url = self.custom_url resp = self.session.get(search_url, params=search_params) if not resp or not resp.content: sickrage.app.log.debug("No data returned from provider") continue try: data = resp.json() except ValueError: sickrage.app.log.debug("No data returned from provider") continue results += self.parse(data, mode) return results
def search(self, search_strings, age=0, ep_obj=None): results = [] """ sorting ss: relevance ed: seeds desc ea: seeds asc pd: peers desc pa: peers asc sd: big > small sa: small > big ad: added desc (latest) aa: added asc (oldest) """ for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode != "RSS": sickrage.app.log.debug("Search string: {0}".format (search_string)) search_url = urljoin(self.urls["search"], "{sorting}/{page}/{search_string}".format(sorting=("ed", "ad")[mode == "RSS"], page=1, search_string=search_string)) if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {0}".format(self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1]) try: data = sickrage.app.wsession.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug("URL did not return results/data, if the results are on the site " "maybe try a custom url, or a different one") return results
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs): results = [] for mode in search_strings: sickrage.app.log.debug("Search Mode: {0}".format(mode)) for search_string in search_strings[mode]: if mode != 'RSS': sickrage.app.log.debug( "Search string: {}".format(search_string)) search_url = urljoin( self.urls['search'], "{search_query}".format(search_query=search_string)) else: search_url = self.urls['rss'] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning( "Invalid custom url: {}".format(self.custom_url)) return results search_url = urljoin( self.custom_url, search_url.split(self.urls['base_url'])[1]) resp = self.session.get(search_url) if not resp or not resp.text: sickrage.app.log.debug("No data returned from provider") continue results += self.parse(resp.text, mode) return results
def search(self, search_strings, age=0, ep_obj=None): results = [] if not self.login(): return results freeleech = '&free=on' if self.freeleech else '' for mode in search_strings: sickrage.app.log.debug("Search Mode: %s" % mode) for search_string in search_strings[mode]: if mode != 'RSS': sickrage.app.log.debug("Search string: %s " % search_string) # URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile search_url = self.urls['search'] % (self.categories, freeleech, search_string) search_url += ';o=seeders' if mode != 'RSS' else '' if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning( "Invalid custom url: {}".format(self.custom_url)) return results search_url = urljoin( self.custom_url, search_url.split(self.urls['base_url'])[1]) try: data = sickrage.app.wsession.get(search_url).text results += self.parse(data, mode) except Exception: sickrage.app.log.debug("No data returned from provider") return results
def search(self, search_strings, age=0, ep_obj=None): results = [] search_url = self.urls["search"] if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning("Invalid custom url: {0}".format( self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1]) # Search Params search_params = { 'category': 'anime' if ep_obj and ep_obj.show and ep_obj.show.anime else 'tv', 'apiKey': self.api_key, } for mode in search_strings: sickrage.app.log.debug('Search mode: {}'.format(mode)) for search_string in search_strings[mode]: search_params['q'] = search_string if mode != 'RSS': sickrage.app.log.debug( 'Search string: {}'.format(search_string)) try: data = sickrage.app.wsession.get( search_url, params=search_params).json() results += self.parse(data, mode) except Exception: sickrage.app.log.debug('No data returned from provider') return results
def parse(self, data, mode, **kwargs): """ Parse search results from data :param data: response data :param mode: search mode :return: search results """ results = [] with bs4_parser(data) as html: table_body = html.find('tbody') # Continue only if at least one release is found if not table_body: sickrage.app.log.debug( 'Data returned from provider does not contain any torrents' ) return results for row in table_body('tr'): cells = row('td') if len(cells) < 4: continue try: info_cell = cells[0].a title = info_cell.get_text() download_url = self._get_download_link( urljoin(self.urls['base_url'], info_cell.get('href'))) if not all([title, download_url]): continue title = '{name} {codec}'.format(name=title, codec='x264') if self.custom_url: if not validate_url(self.custom_url): sickrage.app.log.warning( "Invalid custom url: {}".format( self.custom_url)) return results download_url = urljoin( self.custom_url, download_url.split(self.urls['base_url'])[1]) seeders = try_int(cells[2].get_text(strip=True)) leechers = try_int(cells[3].get_text(strip=True)) torrent_size = cells[1].get_text() size = convert_size(torrent_size, -1, ['O', 'KO', 'MO', 'GO', 'TO', 'PO']) results += [{ 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers }] if mode != 'RSS': sickrage.app.log.debug( "Found result: {}".format(title)) except Exception: sickrage.app.log.error("Failed parsing provider") return results
def search_cache(self, show_id, season, episode, manualSearch=False, downCurQuality=False): cache_results = {} dbData = [] # get data from external database if sickrage.app.config.enable_api_providers_cache and not self.provider.private: try: dbData += sickrage.app.api.provider_cache.get(self.providerID, show_id, season, episode)['data'] except Exception: pass # get data from internal database session = sickrage.app.cache_db.session() dbData += [x.as_dict() for x in session.query(CacheDB.Provider).filter_by(provider=self.providerID, series_id=show_id, season=season).filter( CacheDB.Provider.episodes.contains("|{}|".format(episode)))] for curResult in dbData: show_object = find_show(int(curResult["series_id"])) if not show_object: continue result = self.provider.get_result() # ignore invalid and private IP address urls if not validate_url(curResult["url"]): if not curResult["url"].startswith('magnet'): continue elif is_ip_private(curResult["url"].split(r'//')[-1].split(r'/')[0]): continue # ignored/required words, and non-tv junk if not show_names.filter_bad_releases(curResult["name"]): continue # get the show object, or if it's not one of our shows then ignore it result.show_id = int(curResult["series_id"]) # skip if provider is anime only and show is not anime if self.provider.anime_only and not show_object.is_anime: sickrage.app.log.debug("" + str(show_object.name) + " is not an anime, skiping") continue # get season and ep data (ignoring multi-eps for now) curSeason = int(curResult["season"]) if curSeason == -1: continue result.season = curSeason result.episodes = [int(curEp) for curEp in filter(None, curResult["episodes"].split("|"))] result.quality = int(curResult["quality"]) result.release_group = curResult["release_group"] result.version = curResult["version"] # make sure we want the episode wantEp = False for result_episode in result.episodes: if show_object.want_episode(result.season, result_episode, result.quality, manualSearch, downCurQuality): wantEp = True if not wantEp: sickrage.app.log.info("Skipping " + curResult["name"] + " because we don't want an episode that's " + Quality.qualityStrings[result.quality]) continue # build a result object result.name = curResult["name"] result.url = curResult["url"] sickrage.app.log.info("Found cached {} result {}".format(result.type, result.name)) result.seeders = curResult.get("seeders", -1) result.leechers = curResult.get("leechers", -1) result.size = curResult.get("size", -1) result.content = None # add it to the list if episode not in cache_results: cache_results[int(episode)] = [result] else: cache_results[int(episode)] += [result] # datetime stamp this search so cache gets cleared self.last_search = datetime.datetime.today() return cache_results
def add_cache_entry(self, name, url, seeders, leechers, size): session = sickrage.app.cache_db.session() # check for existing entry in cache if session.query(CacheDB.Provider).filter_by(url=url).count(): return # ignore invalid and private IP address urls if not validate_url(url): if not url.startswith('magnet'): return elif is_ip_private(url.split(r'//')[-1].split(r'/')[0]): return try: # parse release name parse_result = NameParser(validate_show=True).parse(name) if parse_result.series_name and parse_result.quality != Quality.UNKNOWN: season = parse_result.season_number if parse_result.season_number else 1 episodes = parse_result.episode_numbers if season and episodes: # store episodes as a seperated string episodeText = "|" + "|".join(map(str, episodes)) + "|" # get quality of release quality = parse_result.quality # get release group release_group = parse_result.release_group # get version version = parse_result.version dbData = { 'provider': self.providerID, 'name': name, 'season': season, 'episodes': episodeText, 'series_id': parse_result.indexer_id, 'url': url, 'time': int(time.mktime(datetime.datetime.today().timetuple())), 'quality': quality, 'release_group': release_group, 'version': version, 'seeders': try_int(seeders), 'leechers': try_int(leechers), 'size': try_int(size, -1) } # add to internal database try: session.add(CacheDB.Provider(**dbData)) session.commit() sickrage.app.log.debug("SEARCH RESULT:[{}] ADDED TO CACHE!".format(name)) except IntegrityError: pass # add to external provider cache database if sickrage.app.config.enable_api_providers_cache and not self.provider.private: try: sickrage.app.io_loop.run_in_executor(None, functools.partial(sickrage.app.api.provider_cache.add, data=dbData)) except Exception as e: pass except (InvalidShowException, InvalidNameException): pass
def parse(self, data, mode, **kwargs): results = [] with bs4_parser(data) as html: if not self._check_auth_from_data(html): return results try: self.torznab = 'xmlns:torznab' in html.rss.attrs except AttributeError: self.torznab = False if not html('item'): sickrage.app.log.debug('No results returned from provider. Check chosen Newznab ' 'search categories in provider settings and/or usenet ' 'retention') return results for item in html('item'): try: title = item.title.get_text(strip=True) download_url = None if item.link: url = item.link.get_text(strip=True) if validate_url(url) or url.startswith('magnet'): download_url = url if not download_url: url = item.link.next.strip() if validate_url(url) or url.startswith('magnet'): download_url = url if not download_url and item.enclosure: url = item.enclosure.get('url', '').strip() if validate_url(url) or url.startswith('magnet'): download_url = url if not (title and download_url): continue seeders = leechers = -1 if 'gingadaddy' in self.urls['base_url']: size_regex = re.search(r'\d*.?\d* [KMGT]B', str(item.description)) item_size = size_regex.group() if size_regex else -1 else: item_size = item.size.get_text(strip=True) if item.size else -1 newznab_attrs = item(re.compile('newznab:attr')) torznab_attrs = item(re.compile('torznab:attr')) for attr in newznab_attrs + torznab_attrs: item_size = attr['value'] if attr['name'] == 'size' else item_size seeders = try_int(attr['value']) if attr['name'] == 'seeders' else seeders peers = try_int(attr['value']) if attr['name'] == 'peers' else None leechers = peers - seeders if peers else leechers if not item_size or (self.torznab and (seeders is -1 or leechers is -1)): continue size = convert_size(item_size, -1) results += [ {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers} ] if mode != 'RSS': sickrage.app.log.debug('Found result: {}'.format(title)) except (AttributeError, TypeError, KeyError, ValueError, IndexError): sickrage.app.log.error('Failed parsing provider') return results
def find_search_results(self, show_id, season, episode, search_mode, manualSearch=False, downCurQuality=False, cacheOnly=False, session=None): provider_results = {} item_list = [] if not self._check_auth: return provider_results show_object = find_show(show_id, session=session) episode_object = show_object.get_episode(season, episode) # search cache for episode result provider_results = self.cache.search_cache(show_id, season, episode, manualSearch, downCurQuality) # check if this is a cache only search if cacheOnly: return provider_results search_strings = [] if search_mode == 'sponly': # get season search results search_strings = self._get_season_search_strings(show_id, season, episode) elif search_mode == 'eponly': # get single episode search results search_strings = self._get_episode_search_strings(show_id, season, episode) for curString in search_strings: try: item_list += self.search(curString, show_id=show_id, season=season, episode=episode) except SAXParseException: continue # sort list by quality if item_list: # categorize the items into lists by quality items = defaultdict(list) for item in item_list: items[self.get_quality(item, anime=show_object.is_anime)].append(item) # temporarily remove the list of items with unknown quality unknown_items = items.pop(Quality.UNKNOWN, []) # make a generator to sort the remaining items by descending quality items_list = (items[quality] for quality in sorted(items, reverse=True)) # unpack all of the quality lists into a single sorted list items_list = list(itertools.chain(*items_list)) # extend the list with the unknown qualities, now sorted at the bottom of the list items_list.extend(unknown_items) # filter results for item in item_list: provider_result = self.getResult() provider_result.name, provider_result.url = self._get_title_and_url(item) # ignore invalid urls if not validate_url(provider_result.url) and not provider_result.url.startswith('magnet'): continue try: parse_result = NameParser(show_id=show_id).parse(provider_result.name) except (InvalidNameException, InvalidShowException) as e: sickrage.app.log.debug("{}".format(e)) continue provider_result.show_id = parse_result.indexer_id provider_result.quality = parse_result.quality provider_result.release_group = parse_result.release_group provider_result.version = parse_result.version provider_result.size = self._get_size(item) provider_result.seeders, provider_result.leechers = self._get_result_stats(item) sickrage.app.log.debug("Adding item from search to cache: {}".format(provider_result.name)) self.cache.add_cache_entry(provider_result.name, provider_result.url, provider_result.seeders, provider_result.leechers, provider_result.size) if not provider_result.show_id: continue provider_result_show_obj = find_show(provider_result.show_id, session=session) if not provider_result_show_obj: continue if not parse_result.is_air_by_date and (provider_result_show_obj.air_by_date or provider_result_show_obj.sports): sickrage.app.log.debug("This is supposed to be a date search but the result {} didn't parse as one, skipping it".format(provider_result.name)) continue if search_mode == 'sponly': if len(parse_result.episode_numbers): sickrage.app.log.debug("This is supposed to be a season pack search but the result {} is not " "a valid season pack, skipping it".format(provider_result.name)) continue elif parse_result.season_number != (episode_object.season, episode_object.scene_season)[show_object.is_scene]: sickrage.app.log.debug("This season result {} is for a season we are not searching for, skipping it".format(provider_result.name)) continue else: if not all([parse_result.season_number is not None, parse_result.episode_numbers, parse_result.season_number == (episode_object.season, episode_object.scene_season)[show_object.is_scene], (episode_object.episode, episode_object.scene_episode)[show_object.is_scene] in parse_result.episode_numbers]): sickrage.app.log.debug("The result {} doesn't seem to be a valid episode " "that we are trying to snatch, ignoring".format(provider_result.name)) continue provider_result.season = int(parse_result.season_number) provider_result.episodes = list(map(int, parse_result.episode_numbers)) # make sure we want the episode for episode_number in provider_result.episodes.copy(): if not provider_result_show_obj.want_episode(provider_result.season, episode_number, provider_result.quality, manualSearch, downCurQuality): sickrage.app.log.info("RESULT:[{}] QUALITY:[{}] IGNORED!".format(provider_result.name, Quality.qualityStrings[provider_result.quality])) if episode_number in provider_result.episodes: provider_result.episodes.remove(episode_number) # detects if season pack and if not checks if we wanted any of the episodes if len(provider_result.episodes) != len(parse_result.episode_numbers): continue sickrage.app.log.debug( "FOUND RESULT:[{}] QUALITY:[{}] URL:[{}]".format(provider_result.name, Quality.qualityStrings[provider_result.quality], provider_result.url) ) if len(provider_result.episodes) == 1: episode_number = provider_result.episodes[0] sickrage.app.log.debug("Single episode result.") elif len(provider_result.episodes) > 1: episode_number = MULTI_EP_RESULT sickrage.app.log.debug("Separating multi-episode result to check for later - result contains episodes: " + str(parse_result.episode_numbers)) else: episode_number = SEASON_RESULT sickrage.app.log.debug("Separating full season result to check for later") if episode_number not in provider_results: provider_results[int(episode_number)] = [provider_result] else: provider_results[int(episode_number)] += [provider_result] return provider_results
def search_cache(self, ep_obj, manualSearch=False, downCurQuality=False): season = ep_obj.scene_season if ep_obj.show.scene else ep_obj.season episode = ep_obj.scene_episode if ep_obj.show.scene else ep_obj.episode neededEps = {} dbData = [] # get data from external database if sickrage.app.config.enable_api_providers_cache and not self.provider.private: try: dbData += ProviderCacheAPI().get(self.providerID, ep_obj.show.indexerid, season, episode)['data'] except Exception: pass # get data from internal database dbData += [x for x in sickrage.app.cache_db.get_many('providers', self.providerID)] # for each cache entry for curResult in (x for x in dbData if x.get('indexerid') == ep_obj.show.indexerid and x.get('season') == season and "|{}|".format( episode) in x.get('episodes', [])): result = self.provider.getResult() # ignore invalid and private IP address urls if not validate_url(curResult["url"]): if not curResult["url"].startswith('magnet'): continue elif is_ip_private(curResult["url"].split(r'//')[-1].split(r'/')[0]): continue # ignored/required words, and non-tv junk if not show_names.filterBadReleases(curResult["name"]): continue # get the show object, or if it's not one of our shows then ignore it result.show = findCertainShow(int(curResult["indexerid"])) if not result.show: continue # skip if provider is anime only and show is not anime if self.provider.anime_only and not result.show.is_anime: sickrage.app.log.debug("" + str(result.show.name) + " is not an anime, skiping") continue # get season and ep data (ignoring multi-eps for now) curSeason = int(curResult["season"]) if curSeason == -1: continue try: result.episodes = [result.show.get_episode(curSeason, int(curEp)) for curEp in filter(None, curResult["episodes"].split("|"))] except EpisodeNotFoundException: continue result.quality = int(curResult["quality"]) result.release_group = curResult["release_group"] result.version = curResult["version"] # make sure we want the episode wantEp = False for curEp in result.episodes: if result.show.want_episode(curEp.season, curEp.episode, result.quality, manualSearch, downCurQuality): wantEp = True if not wantEp: sickrage.app.log.info("Skipping " + curResult["name"] + " because we don't want an episode that's " + Quality.qualityStrings[result.quality]) continue # build a result object result.name = curResult["name"] result.url = curResult["url"] sickrage.app.log.info("Found result " + result.name + " at " + result.url) result.seeders = curResult.get("seeders", -1) result.leechers = curResult.get("leechers", -1) result.size = curResult.get("size", -1) result.content = None # add it to the list if ep_obj.episode not in neededEps: neededEps[ep_obj.episode] = [result] else: neededEps[ep_obj.episode] += [result] # datetime stamp this search so cache gets cleared self.last_search = datetime.datetime.today() return neededEps
def search_cache(self, ep_obj, manualSearch=False, downCurQuality=False): neededEps = {} dbData = [] # get data from external database if sickrage.app.config.enable_api_providers_cache and not self.provider.private: try: dbData += ProviderCacheAPI().get(self.providerID, ep_obj.show.indexerid, ep_obj.season, ep_obj.episode)['data'] except Exception: pass # get data from internal database dbData += [ x for x in sickrage.app.cache_db.get_many('providers', self.providerID) ] # for each cache entry for curResult in (x for x in dbData if x['indexerid'] == ep_obj.show.indexerid and x['season'] == ep_obj.season and "|" + str(ep_obj.episode) + "|" in x['episodes']): result = self.provider.getResult() # ignore invalid urls if not validate_url(curResult["url"]) and not curResult["url"].startswith('magnet') \ or is_ip_private(curResult["url"].split(r'//')[-1].split(r'/')[0]): continue # ignored/required words, and non-tv junk if not show_names.filterBadReleases(curResult["name"]): continue # get the show object, or if it's not one of our shows then ignore it result.show = findCertainShow(int(curResult["indexerid"])) if not result.show: continue # skip if provider is anime only and show is not anime if self.provider.anime_only and not result.show.is_anime: sickrage.app.log.debug("" + str(result.show.name) + " is not an anime, skiping") continue # get season and ep data (ignoring multi-eps for now) curSeason = int(curResult["season"]) if curSeason == -1: continue try: result.episodes = [ result.show.getEpisode(curSeason, int(curEp)) for curEp in filter(None, curResult["episodes"].split("|")) ] except EpisodeNotFoundException: continue result.quality = int(curResult["quality"]) result.release_group = curResult["release_group"] result.version = curResult["version"] # make sure we want the episode wantEp = False for curEp in result.episodes: if result.show.wantEpisode(curEp.season, curEp.episode, result.quality, manualSearch, downCurQuality): wantEp = True if not wantEp: sickrage.app.log.info( "Skipping " + curResult["name"] + " because we don't want an episode that's " + Quality.qualityStrings[result.quality]) continue # build a result object result.name = curResult["name"] result.url = curResult["url"] sickrage.app.log.info("Found result " + result.name + " at " + result.url) result.seeders = curResult.get("seeders", -1) result.leechers = curResult.get("leechers", -1) result.size = curResult.get("size", -1) result.content = None # add it to the list if ep_obj.episode not in neededEps: neededEps[ep_obj.episode] = [result] else: neededEps[ep_obj.episode] += [result] # datetime stamp this search so cache gets cleared self.last_search = datetime.datetime.today() return neededEps
def addCacheEntry(self, name, url, seeders, leechers, size): # check for existing entry in cache if len([x for x in sickrage.app.cache_db.get_many('providers', self.providerID) if x.get('url') == url]): return # ignore invalid and private IP address urls if not validate_url(url): if not url.startswith('magnet'): return elif is_ip_private(url.split(r'//')[-1].split(r'/')[0]): return try: # parse release name parse_result = NameParser(validate_show=True).parse(name) if parse_result.series_name and parse_result.quality != Quality.UNKNOWN: season = parse_result.season_number if parse_result.season_number else 1 episodes = parse_result.episode_numbers if season and episodes: # store episodes as a seperated string episodeText = "|" + "|".join(map(str, episodes)) + "|" # get quality of release quality = parse_result.quality # get release group release_group = parse_result.release_group # get version version = parse_result.version dbData = { '_t': 'providers', 'provider': self.providerID, 'name': name, 'season': season, 'episodes': episodeText, 'indexerid': parse_result.indexerid, 'url': url, 'time': int(time.mktime(datetime.datetime.today().timetuple())), 'quality': quality, 'release_group': release_group, 'version': version, 'seeders': try_int(seeders), 'leechers': try_int(leechers), 'size': try_int(size, -1) } # add to internal database sickrage.app.cache_db.insert(dbData) # add to external provider cache database if sickrage.app.config.enable_api_providers_cache and not self.provider.private: try: sickrage.app.event_queue.fire_event(ProviderCacheAPI().add, data=dbData) except Exception: pass sickrage.app.log.debug("SEARCH RESULT:[%s] ADDED TO CACHE!", name) except (InvalidShowException, InvalidNameException): pass