def search(self, search_strings, age=0, ep_obj=None, **kwargs):
results = []
if not self.login():
return results
freeleech = '&free=on' if self.freeleech else ''
for mode in search_strings:
sickrage.app.log.debug("Search Mode: %s" % mode)
for search_string in search_strings[mode]:
if mode != 'RSS':
sickrage.app.log.debug("Search string: %s " % search_string)
# URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile
search_url = self.urls['search'] % (self.categories, freeleech, search_string)
search_url += ';o=seeders' if mode != 'RSS' else ''
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url))
return results
search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1])
try:
data = self.session.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("No data returned from provider")
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
search_params = {
'out': 'json',
'filter': 2101,
'showmagnets': 'on',
'num': 50
}
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode != 'RSS':
sickrage.app.log.debug("Search string: {0}".format
(search_string))
search_params['s'] = search_string
search_url = self.urls['base_url']
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url set, please check your settings")
return results
search_url = self.custom_url
try:
data = sickrage.app.wsession.get(search_url, params=search_params).json()
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("No data returned from provider")
return results
def cleanup_provider_cache(self):
for item in self.all('providers'):
if int(item["quality"]) == Quality.UNKNOWN:
self.delete(item)
elif not validate_url(item["url"]) and not item["url"].startswith("magnet") \
or is_ip_private(item["url"].split(r'//')[-1].split(r'/')[0]):
self.delete(item)
def search(self, search_strings, age=0, ep_obj=None, **kwargs): # pylint: disable=too-many-locals
results = []
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode == 'Season':
search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string)
if mode != 'RSS':
sickrage.app.log.debug("Search string: {}".format(search_string))
search_query = re.sub(r'\W', '-', search_string)
search_url = urljoin(self.urls['search'], "{search_query}.html".format(search_query=search_query))
else:
search_url = self.urls['rss']
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url))
return results
search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1])
try:
data = self.session.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("No data returned from provider")
return results
def test_result_values(self):
"""Check that the provider returns results in proper format"""
results = self.provider.search(self.search_strings('Episode'), ep_obj=self.ep)
for result in results:
self.assertIsInstance(result, dict)
self.assertEqual(sorted(result.keys()), ['leechers', 'link', 'seeders', 'size', 'title'])
self.assertIsInstance(result['title'], six.text_type)
self.assertIsInstance(result['link'], six.text_type)
self.assertIsInstance(result['seeders'], six.integer_types)
self.assertIsInstance(result['leechers'], six.integer_types)
self.assertIsInstance(result['size'], six.integer_types)
self.assertTrue(len(result['title']))
self.assertTrue(len(result['link']))
self.assertTrue(result['seeders'] >= 0)
self.assertTrue(result['leechers'] >= 0)
self.assertTrue(result['size'] >= -1)
if result['link'].startswith('magnet'):
self.assertTrue(magnet_regex.match(result['link']))
else:
self.assertTrue(validate_url(result['link']))
self.assertIsInstance(self.provider._get_size(result), six.integer_types)
self.assertTrue(all(self.provider._get_title_and_url(result)))
self.assertTrue(self.provider._get_size(result))
def test_result_values(self):
"""Check that the provider returns results in proper format"""
results = self.provider.search(self.search_strings('Episode'),
ep_obj=self.ep)
for result in results:
self.assertIsInstance(result, dict)
self.assertEqual(
sorted(result.keys()),
['leechers', 'link', 'seeders', 'size', 'title'])
self.assertIsInstance(result['title'], six.text_type)
self.assertIsInstance(result['link'], six.text_type)
self.assertIsInstance(result['seeders'], six.integer_types)
self.assertIsInstance(result['leechers'], six.integer_types)
self.assertIsInstance(result['size'], six.integer_types)
self.assertTrue(len(result['title']))
self.assertTrue(len(result['link']))
self.assertTrue(result['seeders'] >= 0)
self.assertTrue(result['leechers'] >= 0)
self.assertTrue(result['size'] >= -1)
if result['link'].startswith('magnet'):
self.assertTrue(magnet_regex.match(result['link']))
else:
self.assertTrue(validate_url(result['link']))
self.assertIsInstance(self.provider._get_size(result),
six.integer_types)
self.assertTrue(all(self.provider._get_title_and_url(result)))
self.assertTrue(self.provider._get_size(result))
def search(self, search_strings, age=0, ep_obj=None, **kwargs):
results = []
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
search_url = (self.urls["search"],
self.urls["rss"])[mode == "RSS"]
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning(
"Invalid custom url: {0}".format(self.custom_url))
return results
search_url = urljoin(
self.custom_url,
search_url.split(self.urls['base_url'])[1])
if mode != "RSS":
search_url = search_url % search_string
sickrage.app.log.debug(
"Search string: {}".format(search_string))
try:
data = self.session.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("No data returned from provider")
return results
def cleanup_provider_cache(self):
for item in self.all('providers'):
if int(item["quality"]) == Quality.UNKNOWN:
self.delete(item)
elif not validate_url(item["url"]) and not item["url"].startswith("magnet") \
or is_ip_private(item["url"].split(r'//')[-1].split(r'/')[0]):
self.delete(item)
def search(self, search_strings, age=0, ep_obj=None, **kwargs):
results = []
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode != 'RSS':
sickrage.app.log.debug("Search string: {}".format(search_string))
search_url = urljoin(self.urls['search'], "{search_query}".format(search_query=search_string))
else:
search_url = self.urls['rss']
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url))
return results
search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1])
try:
data = self.session.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("No data returned from provider")
return results
def parse(self, data, mode, **kwargs):
"""
Parse search results from data
:param data: response data
:param mode: search mode
:return: search results
"""
results = []
with bs4_parser(data) as html:
table_body = html.find('tbody')
# Continue only if at least one release is found
if not table_body:
sickrage.app.log.debug('Data returned from provider does not contain any torrents')
return results
for row in table_body('tr'):
cells = row('td')
if len(cells) < 4:
continue
try:
title = download_url = None
info_cell = cells[0].a
if info_cell:
title = info_cell.get_text()
download_url = self._get_download_link(urljoin(self.urls['base_url'], info_cell.get('href')))
if not all([title, download_url]):
continue
title = '{name} {codec}'.format(name=title, codec='x264')
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url))
return results
download_url = urljoin(self.custom_url, download_url.split(self.urls['base_url'])[1])
seeders = try_int(cells[2].get_text(strip=True))
leechers = try_int(cells[3].get_text(strip=True))
torrent_size = cells[1].get_text()
size = convert_size(torrent_size, -1)
results += [{
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers
}]
if mode != 'RSS':
sickrage.app.log.debug("Found result: {}".format(title))
except Exception:
sickrage.app.log.error("Failed parsing provider")
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
search_url = self.urls["search"]
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {0}".format(self.custom_url))
return results
search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1])
# Search Params
search_params = {
'category': 'anime' if ep_obj and ep_obj.show and ep_obj.show.anime else 'tv',
'apiKey': self.api_key,
}
for mode in search_strings:
sickrage.app.log.debug('Search mode: {}'.format(mode))
for search_string in search_strings[mode]:
search_params['q'] = search_string
if mode != 'RSS':
sickrage.app.log.debug('Search string: {}'.format(search_string))
try:
data = sickrage.app.wsession.get(search_url, params=search_params).json()
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug('No data returned from provider')
return results
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs):
results = []
if not self.login():
return results
freeleech = '&free=on' if self.freeleech else ''
for mode in search_strings:
sickrage.app.log.debug("Search Mode: %s" % mode)
for search_string in search_strings[mode]:
if mode != 'RSS':
sickrage.app.log.debug("Search string: %s " % search_string)
# URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile
search_url = self.urls['search'] % (self.categories, freeleech, search_string)
search_url += ';o=seeders' if mode != 'RSS' else ''
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {}".format(self.custom_url))
return results
search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1])
resp = self.session.get(search_url)
if not resp or not resp.text:
sickrage.app.log.debug("No data returned from provider")
continue
results += self.parse(resp.text, mode)
return results
def addCacheEntry(self, name, url, seeders, leechers, size):
# check for existing entry in cache
if len([x for x in sickrage.app.cache_db.get_many('providers', self.providerID) if x['url'] == url]):
return
# ignore invalid urls
if not validate_url(url) and not url.startswith('magnet') \
or is_ip_private(url.split(r'//')[-1].split(r'/')[0]):
return
try:
# parse release name
parse_result = NameParser(validate_show=sickrage.app.config.enable_rss_cache_valid_shows).parse(name)
if parse_result.series_name and parse_result.quality != Quality.UNKNOWN:
season = parse_result.season_number if parse_result.season_number else 1
episodes = parse_result.episode_numbers
if season and episodes:
# store episodes as a seperated string
episodeText = "|" + "|".join(map(str, episodes)) + "|"
# get quality of release
quality = parse_result.quality
# get release group
release_group = parse_result.release_group
# get version
version = parse_result.version
dbData = {
'_t': 'providers',
'provider': self.providerID,
'name': name,
'season': season,
'episodes': episodeText,
'indexerid': parse_result.indexerid,
'url': url,
'time': int(time.mktime(datetime.datetime.today().timetuple())),
'quality': quality,
'release_group': release_group,
'version': version,
'seeders': seeders,
'leechers': leechers,
'size': size
}
# add to internal database
sickrage.app.cache_db.insert(dbData)
# add to external provider cache database
if sickrage.app.config.enable_api_providers_cache and not self.provider.private:
try:
sickrage.app.api.add_provider_cache_result(dbData)
except Exception:
pass
sickrage.app.log.debug("SEARCH RESULT:[%s] ADDED TO CACHE!", name)
except (InvalidShowException, InvalidNameException):
pass
def search(self, search_strings, age=0, series_id=None, series_provider_id=None, season=None, episode=None, **kwargs):
results = []
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
search_url = (self.urls["search"], self.urls["rss"])[mode == "RSS"]
if self.custom_settings['custom_url']:
if not validate_url(self.custom_settings['custom_url']):
sickrage.app.log.warning("Invalid custom url: {0}".format(self.custom_settings['custom_url']))
return results
search_url = urljoin(self.custom_settings['custom_url'], search_url.split(self.urls['base_url'])[1])
if mode != "RSS":
search_url = search_url % search_string
sickrage.app.log.debug("Search string: {}".format(search_string))
resp = self.session.get(search_url)
if not resp or not resp.text:
sickrage.app.log.debug("No data returned from provider")
continue
results += self.parse(resp.text, mode)
return results
def search(self, search_strings, age=0, ep_obj=None, **kwargs): # pylint: disable=too-many-locals
results = []
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode == 'Season':
search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string)
if mode != 'RSS':
sickrage.app.log.debug("Search string: {0}".format
(search_string))
search_string = search_string.replace('.', '-').replace(' ', '-')
search_url = urljoin(self.urls['search'],
"{search_string}.html".format(search_string=search_string))
else:
search_url = self.urls['rss']
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {0}".format(self.custom_url))
return results
search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1])
try:
data = self.session.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("No data returned from provider")
return results
def search(self,
search_strings,
age=0,
show_id=None,
season=None,
episode=None,
**kwargs):
results = []
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {}".format(mode))
for search_string in search_strings[mode]:
if mode != "RSS":
sickrage.app.log.debug(
"Search string: {0}".format(search_string))
search_url = (self.urls["base_url"],
self.urls["rss"])[mode == "RSS"]
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning(
"Invalid custom url: {}".format(self.custom_url))
return results
search_url = urljoin(
self.custom_url,
search_url.split(self.urls['base_url'])[1])
if mode != "RSS":
search_params = {
'query': search_string,
'sort': ('seeders', 'created')[mode == 'RSS'],
'type': 'video',
'tag': 'hd',
'category': 'show'
}
else:
search_params = {
'category': 'show',
'type': 'video',
'sort': 'created'
}
resp = self.session.get(search_url, params=search_params)
if not resp or not resp.text:
sickrage.app.log.debug("No data returned from provider")
continue
results += self.parse(resp.text, mode)
return results
def search(self,
search_strings,
age=0,
series_id=None,
series_provider_id=None,
season=None,
episode=None,
**kwargs):
results = []
search_url = self.urls["search"]
if self.custom_settings['custom_url']:
if not validate_url(self.custom_settings['custom_url']):
sickrage.app.log.warning("Invalid custom url: {0}".format(
self.custom_settings['custom_url']))
return results
search_url = urljoin(self.custom_settings['custom_url'],
search_url.split(self.urls['base_url'])[1])
show_object = find_show(series_id, series_provider_id)
# Search Params
search_params = {
'category': ("tv", "anime")[bool(show_object.anime)],
'apiKey': self.custom_settings['api_key'],
}
for mode in search_strings:
sickrage.app.log.debug('Search mode: {}'.format(mode))
for search_string in search_strings[mode]:
search_params['q'] = search_string
if mode != 'RSS':
sickrage.app.log.debug(
'Search string: {}'.format(search_string))
resp = self.session.get(search_url, params=search_params)
if not resp or not resp.content:
sickrage.app.log.debug('No data returned from provider')
continue
try:
data = resp.json()
except ValueError:
sickrage.app.log.debug('No data returned from provider')
continue
results += self.parse(data, mode)
return results
def search(self,
search_strings,
age=0,
show_id=None,
season=None,
episode=None,
**kwargs):
results = []
"""
sorting
ss: relevance
ed: seeds desc
ea: seeds asc
pd: peers desc
pa: peers asc
sd: big > small
sa: small > big
ad: added desc (latest)
aa: added asc (oldest)
"""
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode != "RSS":
sickrage.app.log.debug(
"Search string: {0}".format(search_string))
search_url = self.urls["search"] % (
("ed", "ad")[mode == "RSS"], 1, search_string)
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning(
"Invalid custom url: {0}".format(self.custom_url))
return results
search_url = urljoin(
self.custom_url,
search_url.split(self.urls['base_url'])[1])
try:
data = self.session.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug(
"URL did not return results/data, if the results are on the site maybe try "
"a custom url, or a different one")
return results
def find_propers(self, show_id, season, episode):
results = []
for term in self.proper_strings:
search_strngs = self._get_episode_search_strings(show_id, season, episode, add_string=term)
for item in self.search(search_strngs[0], show_id=show_id, season=season, episode=episode):
result = self.getResult(season, [episode])
result.name, result.url = self._get_title_and_url(item)
if not validate_url(result.url) and not result.url.startswith('magnet'):
continue
result.seeders, result.leechers = self._get_result_stats(item)
result.size = self._get_size(item)
result.date = datetime.datetime.today()
results.append(result)
return results
def search(self,
search_strings,
age=0,
show_id=None,
season=None,
episode=None,
**kwargs):
results = []
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode == 'Season':
search_string = re.sub(r'(.*)S0?', r'\1Saison ',
search_string)
if mode != 'RSS':
sickrage.app.log.debug(
"Search string: {}".format(search_string))
search_query = re.sub(r'\W', '-', search_string)
search_url = urljoin(
self.urls['search'], "{search_query}.html".format(
search_query=search_query))
else:
search_url = self.urls['rss']
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning(
"Invalid custom url: {}".format(self.custom_url))
return results
search_url = urljoin(
self.custom_url,
search_url.split(self.urls['base_url'])[1])
try:
data = self.session.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("No data returned from provider")
return results
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs):
results = []
search_params = {
'out': 'json',
'filter': 2101,
'showmagnets': 'on',
'num': 50
}
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode != 'RSS':
sickrage.app.log.debug("Search string: {0}".format
(search_string))
search_params['s'] = search_string
search_url = self.urls['base_url']
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url set, please check your settings")
return results
search_url = self.custom_url
resp = self.session.get(search_url, params=search_params)
if not resp or not resp.content:
sickrage.app.log.debug("No data returned from provider")
continue
try:
data = resp.json()
except ValueError:
sickrage.app.log.debug("No data returned from provider")
continue
results += self.parse(data, mode)
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
"""
sorting
ss: relevance
ed: seeds desc
ea: seeds asc
pd: peers desc
pa: peers asc
sd: big > small
sa: small > big
ad: added desc (latest)
aa: added asc (oldest)
"""
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode != "RSS":
sickrage.app.log.debug("Search string: {0}".format
(search_string))
search_url = urljoin(self.urls["search"],
"{sorting}/{page}/{search_string}".format(sorting=("ed", "ad")[mode == "RSS"],
page=1, search_string=search_string))
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {0}".format(self.custom_url))
return results
search_url = urljoin(self.custom_url, search_url.split(self.urls['base_url'])[1])
try:
data = sickrage.app.wsession.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("URL did not return results/data, if the results are on the site "
"maybe try a custom url, or a different one")
return results
def search(self,
search_strings,
age=0,
show_id=None,
season=None,
episode=None,
**kwargs):
results = []
for mode in search_strings:
sickrage.app.log.debug("Search Mode: {0}".format(mode))
for search_string in search_strings[mode]:
if mode != 'RSS':
sickrage.app.log.debug(
"Search string: {}".format(search_string))
search_url = urljoin(
self.urls['search'],
"{search_query}".format(search_query=search_string))
else:
search_url = self.urls['rss']
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning(
"Invalid custom url: {}".format(self.custom_url))
return results
search_url = urljoin(
self.custom_url,
search_url.split(self.urls['base_url'])[1])
resp = self.session.get(search_url)
if not resp or not resp.text:
sickrage.app.log.debug("No data returned from provider")
continue
results += self.parse(resp.text, mode)
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if not self.login():
return results
freeleech = '&free=on' if self.freeleech else ''
for mode in search_strings:
sickrage.app.log.debug("Search Mode: %s" % mode)
for search_string in search_strings[mode]:
if mode != 'RSS':
sickrage.app.log.debug("Search string: %s " %
search_string)
# URL with 50 tv-show results, or max 150 if adjusted in IPTorrents profile
search_url = self.urls['search'] % (self.categories, freeleech,
search_string)
search_url += ';o=seeders' if mode != 'RSS' else ''
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning(
"Invalid custom url: {}".format(self.custom_url))
return results
search_url = urljoin(
self.custom_url,
search_url.split(self.urls['base_url'])[1])
try:
data = sickrage.app.wsession.get(search_url).text
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug("No data returned from provider")
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
search_url = self.urls["search"]
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning("Invalid custom url: {0}".format(
self.custom_url))
return results
search_url = urljoin(self.custom_url,
search_url.split(self.urls['base_url'])[1])
# Search Params
search_params = {
'category':
'anime' if ep_obj and ep_obj.show and ep_obj.show.anime else 'tv',
'apiKey':
self.api_key,
}
for mode in search_strings:
sickrage.app.log.debug('Search mode: {}'.format(mode))
for search_string in search_strings[mode]:
search_params['q'] = search_string
if mode != 'RSS':
sickrage.app.log.debug(
'Search string: {}'.format(search_string))
try:
data = sickrage.app.wsession.get(
search_url, params=search_params).json()
results += self.parse(data, mode)
except Exception:
sickrage.app.log.debug('No data returned from provider')
return results
def parse(self, data, mode, **kwargs):
"""
Parse search results from data
:param data: response data
:param mode: search mode
:return: search results
"""
results = []
with bs4_parser(data) as html:
table_body = html.find('tbody')
# Continue only if at least one release is found
if not table_body:
sickrage.app.log.debug(
'Data returned from provider does not contain any torrents'
)
return results
for row in table_body('tr'):
cells = row('td')
if len(cells) < 4:
continue
try:
info_cell = cells[0].a
title = info_cell.get_text()
download_url = self._get_download_link(
urljoin(self.urls['base_url'], info_cell.get('href')))
if not all([title, download_url]):
continue
title = '{name} {codec}'.format(name=title, codec='x264')
if self.custom_url:
if not validate_url(self.custom_url):
sickrage.app.log.warning(
"Invalid custom url: {}".format(
self.custom_url))
return results
download_url = urljoin(
self.custom_url,
download_url.split(self.urls['base_url'])[1])
seeders = try_int(cells[2].get_text(strip=True))
leechers = try_int(cells[3].get_text(strip=True))
torrent_size = cells[1].get_text()
size = convert_size(torrent_size, -1,
['O', 'KO', 'MO', 'GO', 'TO', 'PO'])
results += [{
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers
}]
if mode != 'RSS':
sickrage.app.log.debug(
"Found result: {}".format(title))
except Exception:
sickrage.app.log.error("Failed parsing provider")
return results
def search_cache(self, show_id, season, episode, manualSearch=False, downCurQuality=False):
cache_results = {}
dbData = []
# get data from external database
if sickrage.app.config.enable_api_providers_cache and not self.provider.private:
try:
dbData += sickrage.app.api.provider_cache.get(self.providerID, show_id, season, episode)['data']
except Exception:
pass
# get data from internal database
session = sickrage.app.cache_db.session()
dbData += [x.as_dict() for x in session.query(CacheDB.Provider).filter_by(provider=self.providerID, series_id=show_id, season=season).filter(
CacheDB.Provider.episodes.contains("|{}|".format(episode)))]
for curResult in dbData:
show_object = find_show(int(curResult["series_id"]))
if not show_object:
continue
result = self.provider.get_result()
# ignore invalid and private IP address urls
if not validate_url(curResult["url"]):
if not curResult["url"].startswith('magnet'):
continue
elif is_ip_private(curResult["url"].split(r'//')[-1].split(r'/')[0]):
continue
# ignored/required words, and non-tv junk
if not show_names.filter_bad_releases(curResult["name"]):
continue
# get the show object, or if it's not one of our shows then ignore it
result.show_id = int(curResult["series_id"])
# skip if provider is anime only and show is not anime
if self.provider.anime_only and not show_object.is_anime:
sickrage.app.log.debug("" + str(show_object.name) + " is not an anime, skiping")
continue
# get season and ep data (ignoring multi-eps for now)
curSeason = int(curResult["season"])
if curSeason == -1:
continue
result.season = curSeason
result.episodes = [int(curEp) for curEp in filter(None, curResult["episodes"].split("|"))]
result.quality = int(curResult["quality"])
result.release_group = curResult["release_group"]
result.version = curResult["version"]
# make sure we want the episode
wantEp = False
for result_episode in result.episodes:
if show_object.want_episode(result.season, result_episode, result.quality, manualSearch, downCurQuality):
wantEp = True
if not wantEp:
sickrage.app.log.info("Skipping " + curResult["name"] + " because we don't want an episode that's " + Quality.qualityStrings[result.quality])
continue
# build a result object
result.name = curResult["name"]
result.url = curResult["url"]
sickrage.app.log.info("Found cached {} result {}".format(result.type, result.name))
result.seeders = curResult.get("seeders", -1)
result.leechers = curResult.get("leechers", -1)
result.size = curResult.get("size", -1)
result.content = None
# add it to the list
if episode not in cache_results:
cache_results[int(episode)] = [result]
else:
cache_results[int(episode)] += [result]
# datetime stamp this search so cache gets cleared
self.last_search = datetime.datetime.today()
return cache_results
def add_cache_entry(self, name, url, seeders, leechers, size):
session = sickrage.app.cache_db.session()
# check for existing entry in cache
if session.query(CacheDB.Provider).filter_by(url=url).count():
return
# ignore invalid and private IP address urls
if not validate_url(url):
if not url.startswith('magnet'):
return
elif is_ip_private(url.split(r'//')[-1].split(r'/')[0]):
return
try:
# parse release name
parse_result = NameParser(validate_show=True).parse(name)
if parse_result.series_name and parse_result.quality != Quality.UNKNOWN:
season = parse_result.season_number if parse_result.season_number else 1
episodes = parse_result.episode_numbers
if season and episodes:
# store episodes as a seperated string
episodeText = "|" + "|".join(map(str, episodes)) + "|"
# get quality of release
quality = parse_result.quality
# get release group
release_group = parse_result.release_group
# get version
version = parse_result.version
dbData = {
'provider': self.providerID,
'name': name,
'season': season,
'episodes': episodeText,
'series_id': parse_result.indexer_id,
'url': url,
'time': int(time.mktime(datetime.datetime.today().timetuple())),
'quality': quality,
'release_group': release_group,
'version': version,
'seeders': try_int(seeders),
'leechers': try_int(leechers),
'size': try_int(size, -1)
}
# add to internal database
try:
session.add(CacheDB.Provider(**dbData))
session.commit()
sickrage.app.log.debug("SEARCH RESULT:[{}] ADDED TO CACHE!".format(name))
except IntegrityError:
pass
# add to external provider cache database
if sickrage.app.config.enable_api_providers_cache and not self.provider.private:
try:
sickrage.app.io_loop.run_in_executor(None, functools.partial(sickrage.app.api.provider_cache.add, data=dbData))
except Exception as e:
pass
except (InvalidShowException, InvalidNameException):
pass
def parse(self, data, mode, **kwargs):
results = []
with bs4_parser(data) as html:
if not self._check_auth_from_data(html):
return results
try:
self.torznab = 'xmlns:torznab' in html.rss.attrs
except AttributeError:
self.torznab = False
if not html('item'):
sickrage.app.log.debug('No results returned from provider. Check chosen Newznab '
'search categories in provider settings and/or usenet '
'retention')
return results
for item in html('item'):
try:
title = item.title.get_text(strip=True)
download_url = None
if item.link:
url = item.link.get_text(strip=True)
if validate_url(url) or url.startswith('magnet'):
download_url = url
if not download_url:
url = item.link.next.strip()
if validate_url(url) or url.startswith('magnet'):
download_url = url
if not download_url and item.enclosure:
url = item.enclosure.get('url', '').strip()
if validate_url(url) or url.startswith('magnet'):
download_url = url
if not (title and download_url):
continue
seeders = leechers = -1
if 'gingadaddy' in self.urls['base_url']:
size_regex = re.search(r'\d*.?\d* [KMGT]B', str(item.description))
item_size = size_regex.group() if size_regex else -1
else:
item_size = item.size.get_text(strip=True) if item.size else -1
newznab_attrs = item(re.compile('newznab:attr'))
torznab_attrs = item(re.compile('torznab:attr'))
for attr in newznab_attrs + torznab_attrs:
item_size = attr['value'] if attr['name'] == 'size' else item_size
seeders = try_int(attr['value']) if attr['name'] == 'seeders' else seeders
peers = try_int(attr['value']) if attr['name'] == 'peers' else None
leechers = peers - seeders if peers else leechers
if not item_size or (self.torznab and (seeders is -1 or leechers is -1)):
continue
size = convert_size(item_size, -1)
results += [
{'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers}
]
if mode != 'RSS':
sickrage.app.log.debug('Found result: {}'.format(title))
except (AttributeError, TypeError, KeyError, ValueError, IndexError):
sickrage.app.log.error('Failed parsing provider')
return results
def find_search_results(self, show_id, season, episode, search_mode, manualSearch=False, downCurQuality=False, cacheOnly=False, session=None):
provider_results = {}
item_list = []
if not self._check_auth:
return provider_results
show_object = find_show(show_id, session=session)
episode_object = show_object.get_episode(season, episode)
# search cache for episode result
provider_results = self.cache.search_cache(show_id, season, episode, manualSearch, downCurQuality)
# check if this is a cache only search
if cacheOnly:
return provider_results
search_strings = []
if search_mode == 'sponly':
# get season search results
search_strings = self._get_season_search_strings(show_id, season, episode)
elif search_mode == 'eponly':
# get single episode search results
search_strings = self._get_episode_search_strings(show_id, season, episode)
for curString in search_strings:
try:
item_list += self.search(curString, show_id=show_id, season=season, episode=episode)
except SAXParseException:
continue
# sort list by quality
if item_list:
# categorize the items into lists by quality
items = defaultdict(list)
for item in item_list:
items[self.get_quality(item, anime=show_object.is_anime)].append(item)
# temporarily remove the list of items with unknown quality
unknown_items = items.pop(Quality.UNKNOWN, [])
# make a generator to sort the remaining items by descending quality
items_list = (items[quality] for quality in sorted(items, reverse=True))
# unpack all of the quality lists into a single sorted list
items_list = list(itertools.chain(*items_list))
# extend the list with the unknown qualities, now sorted at the bottom of the list
items_list.extend(unknown_items)
# filter results
for item in item_list:
provider_result = self.getResult()
provider_result.name, provider_result.url = self._get_title_and_url(item)
# ignore invalid urls
if not validate_url(provider_result.url) and not provider_result.url.startswith('magnet'):
continue
try:
parse_result = NameParser(show_id=show_id).parse(provider_result.name)
except (InvalidNameException, InvalidShowException) as e:
sickrage.app.log.debug("{}".format(e))
continue
provider_result.show_id = parse_result.indexer_id
provider_result.quality = parse_result.quality
provider_result.release_group = parse_result.release_group
provider_result.version = parse_result.version
provider_result.size = self._get_size(item)
provider_result.seeders, provider_result.leechers = self._get_result_stats(item)
sickrage.app.log.debug("Adding item from search to cache: {}".format(provider_result.name))
self.cache.add_cache_entry(provider_result.name, provider_result.url, provider_result.seeders, provider_result.leechers, provider_result.size)
if not provider_result.show_id:
continue
provider_result_show_obj = find_show(provider_result.show_id, session=session)
if not provider_result_show_obj:
continue
if not parse_result.is_air_by_date and (provider_result_show_obj.air_by_date or provider_result_show_obj.sports):
sickrage.app.log.debug("This is supposed to be a date search but the result {} didn't parse as one, skipping it".format(provider_result.name))
continue
if search_mode == 'sponly':
if len(parse_result.episode_numbers):
sickrage.app.log.debug("This is supposed to be a season pack search but the result {} is not "
"a valid season pack, skipping it".format(provider_result.name))
continue
elif parse_result.season_number != (episode_object.season, episode_object.scene_season)[show_object.is_scene]:
sickrage.app.log.debug("This season result {} is for a season we are not searching for, skipping it".format(provider_result.name))
continue
else:
if not all([parse_result.season_number is not None, parse_result.episode_numbers,
parse_result.season_number == (episode_object.season, episode_object.scene_season)[show_object.is_scene],
(episode_object.episode, episode_object.scene_episode)[show_object.is_scene] in parse_result.episode_numbers]):
sickrage.app.log.debug("The result {} doesn't seem to be a valid episode "
"that we are trying to snatch, ignoring".format(provider_result.name))
continue
provider_result.season = int(parse_result.season_number)
provider_result.episodes = list(map(int, parse_result.episode_numbers))
# make sure we want the episode
for episode_number in provider_result.episodes.copy():
if not provider_result_show_obj.want_episode(provider_result.season, episode_number, provider_result.quality, manualSearch, downCurQuality):
sickrage.app.log.info("RESULT:[{}] QUALITY:[{}] IGNORED!".format(provider_result.name, Quality.qualityStrings[provider_result.quality]))
if episode_number in provider_result.episodes:
provider_result.episodes.remove(episode_number)
# detects if season pack and if not checks if we wanted any of the episodes
if len(provider_result.episodes) != len(parse_result.episode_numbers):
continue
sickrage.app.log.debug(
"FOUND RESULT:[{}] QUALITY:[{}] URL:[{}]".format(provider_result.name, Quality.qualityStrings[provider_result.quality], provider_result.url)
)
if len(provider_result.episodes) == 1:
episode_number = provider_result.episodes[0]
sickrage.app.log.debug("Single episode result.")
elif len(provider_result.episodes) > 1:
episode_number = MULTI_EP_RESULT
sickrage.app.log.debug("Separating multi-episode result to check for later - result contains episodes: " + str(parse_result.episode_numbers))
else:
episode_number = SEASON_RESULT
sickrage.app.log.debug("Separating full season result to check for later")
if episode_number not in provider_results:
provider_results[int(episode_number)] = [provider_result]
else:
provider_results[int(episode_number)] += [provider_result]
return provider_results
def search_cache(self, ep_obj, manualSearch=False, downCurQuality=False):
season = ep_obj.scene_season if ep_obj.show.scene else ep_obj.season
episode = ep_obj.scene_episode if ep_obj.show.scene else ep_obj.episode
neededEps = {}
dbData = []
# get data from external database
if sickrage.app.config.enable_api_providers_cache and not self.provider.private:
try:
dbData += ProviderCacheAPI().get(self.providerID, ep_obj.show.indexerid, season, episode)['data']
except Exception:
pass
# get data from internal database
dbData += [x for x in sickrage.app.cache_db.get_many('providers', self.providerID)]
# for each cache entry
for curResult in (x for x in dbData if
x.get('indexerid') == ep_obj.show.indexerid and x.get('season') == season and "|{}|".format(
episode) in x.get('episodes', [])):
result = self.provider.getResult()
# ignore invalid and private IP address urls
if not validate_url(curResult["url"]):
if not curResult["url"].startswith('magnet'):
continue
elif is_ip_private(curResult["url"].split(r'//')[-1].split(r'/')[0]):
continue
# ignored/required words, and non-tv junk
if not show_names.filterBadReleases(curResult["name"]):
continue
# get the show object, or if it's not one of our shows then ignore it
result.show = findCertainShow(int(curResult["indexerid"]))
if not result.show:
continue
# skip if provider is anime only and show is not anime
if self.provider.anime_only and not result.show.is_anime:
sickrage.app.log.debug("" + str(result.show.name) + " is not an anime, skiping")
continue
# get season and ep data (ignoring multi-eps for now)
curSeason = int(curResult["season"])
if curSeason == -1:
continue
try:
result.episodes = [result.show.get_episode(curSeason, int(curEp)) for curEp in
filter(None, curResult["episodes"].split("|"))]
except EpisodeNotFoundException:
continue
result.quality = int(curResult["quality"])
result.release_group = curResult["release_group"]
result.version = curResult["version"]
# make sure we want the episode
wantEp = False
for curEp in result.episodes:
if result.show.want_episode(curEp.season,
curEp.episode,
result.quality,
manualSearch,
downCurQuality):
wantEp = True
if not wantEp:
sickrage.app.log.info("Skipping " + curResult["name"] + " because we don't want an episode that's " +
Quality.qualityStrings[result.quality])
continue
# build a result object
result.name = curResult["name"]
result.url = curResult["url"]
sickrage.app.log.info("Found result " + result.name + " at " + result.url)
result.seeders = curResult.get("seeders", -1)
result.leechers = curResult.get("leechers", -1)
result.size = curResult.get("size", -1)
result.content = None
# add it to the list
if ep_obj.episode not in neededEps:
neededEps[ep_obj.episode] = [result]
else:
neededEps[ep_obj.episode] += [result]
# datetime stamp this search so cache gets cleared
self.last_search = datetime.datetime.today()
return neededEps
def search_cache(self, ep_obj, manualSearch=False, downCurQuality=False):
neededEps = {}
dbData = []
# get data from external database
if sickrage.app.config.enable_api_providers_cache and not self.provider.private:
try:
dbData += ProviderCacheAPI().get(self.providerID,
ep_obj.show.indexerid,
ep_obj.season,
ep_obj.episode)['data']
except Exception:
pass
# get data from internal database
dbData += [
x for x in sickrage.app.cache_db.get_many('providers',
self.providerID)
]
# for each cache entry
for curResult in (x for x in dbData
if x['indexerid'] == ep_obj.show.indexerid
and x['season'] == ep_obj.season and "|" +
str(ep_obj.episode) + "|" in x['episodes']):
result = self.provider.getResult()
# ignore invalid urls
if not validate_url(curResult["url"]) and not curResult["url"].startswith('magnet') \
or is_ip_private(curResult["url"].split(r'//')[-1].split(r'/')[0]):
continue
# ignored/required words, and non-tv junk
if not show_names.filterBadReleases(curResult["name"]):
continue
# get the show object, or if it's not one of our shows then ignore it
result.show = findCertainShow(int(curResult["indexerid"]))
if not result.show:
continue
# skip if provider is anime only and show is not anime
if self.provider.anime_only and not result.show.is_anime:
sickrage.app.log.debug("" + str(result.show.name) +
" is not an anime, skiping")
continue
# get season and ep data (ignoring multi-eps for now)
curSeason = int(curResult["season"])
if curSeason == -1:
continue
try:
result.episodes = [
result.show.getEpisode(curSeason, int(curEp))
for curEp in filter(None, curResult["episodes"].split("|"))
]
except EpisodeNotFoundException:
continue
result.quality = int(curResult["quality"])
result.release_group = curResult["release_group"]
result.version = curResult["version"]
# make sure we want the episode
wantEp = False
for curEp in result.episodes:
if result.show.wantEpisode(curEp.season, curEp.episode,
result.quality, manualSearch,
downCurQuality):
wantEp = True
if not wantEp:
sickrage.app.log.info(
"Skipping " + curResult["name"] +
" because we don't want an episode that's " +
Quality.qualityStrings[result.quality])
continue
# build a result object
result.name = curResult["name"]
result.url = curResult["url"]
sickrage.app.log.info("Found result " + result.name + " at " +
result.url)
result.seeders = curResult.get("seeders", -1)
result.leechers = curResult.get("leechers", -1)
result.size = curResult.get("size", -1)
result.content = None
# add it to the list
if ep_obj.episode not in neededEps:
neededEps[ep_obj.episode] = [result]
else:
neededEps[ep_obj.episode] += [result]
# datetime stamp this search so cache gets cleared
self.last_search = datetime.datetime.today()
return neededEps
def addCacheEntry(self, name, url, seeders, leechers, size):
# check for existing entry in cache
if len([x for x in sickrage.app.cache_db.get_many('providers', self.providerID) if x.get('url') == url]):
return
# ignore invalid and private IP address urls
if not validate_url(url):
if not url.startswith('magnet'):
return
elif is_ip_private(url.split(r'//')[-1].split(r'/')[0]):
return
try:
# parse release name
parse_result = NameParser(validate_show=True).parse(name)
if parse_result.series_name and parse_result.quality != Quality.UNKNOWN:
season = parse_result.season_number if parse_result.season_number else 1
episodes = parse_result.episode_numbers
if season and episodes:
# store episodes as a seperated string
episodeText = "|" + "|".join(map(str, episodes)) + "|"
# get quality of release
quality = parse_result.quality
# get release group
release_group = parse_result.release_group
# get version
version = parse_result.version
dbData = {
'_t': 'providers',
'provider': self.providerID,
'name': name,
'season': season,
'episodes': episodeText,
'indexerid': parse_result.indexerid,
'url': url,
'time': int(time.mktime(datetime.datetime.today().timetuple())),
'quality': quality,
'release_group': release_group,
'version': version,
'seeders': try_int(seeders),
'leechers': try_int(leechers),
'size': try_int(size, -1)
}
# add to internal database
sickrage.app.cache_db.insert(dbData)
# add to external provider cache database
if sickrage.app.config.enable_api_providers_cache and not self.provider.private:
try:
sickrage.app.event_queue.fire_event(ProviderCacheAPI().add, data=dbData)
except Exception:
pass
sickrage.app.log.debug("SEARCH RESULT:[%s] ADDED TO CACHE!", name)
except (InvalidShowException, InvalidNameException):
pass
