class GenericProvider(object):
def __init__(self, name, url, private):
self.name = name
self.urls = {'base_url': url}
self.private = private
self.show = None
self.supports_backlog = False
self.supports_absolute_numbering = False
self.anime_only = False
self.search_mode = 'eponly'
self.search_fallback = False
self.enabled = False
self.enable_daily = False
self.enable_backlog = False
self.cache = TVCache(self)
self.proper_strings = ['PROPER|REPACK|REAL']
self.enable_cookies = False
self.cookies = ''
self.rss_cookies = ''
self.cookie_jar = dict()
@property
def id(self):
return str(re.sub(r"[^\w\d_]", "_", self.name.strip().lower()))
@property
def isEnabled(self):
return self.enabled
@property
def imageName(self):
return ""
def check_auth(self):
return True
def _doLogin(self):
return True
@classmethod
def get_subclasses(cls):
yield cls
if cls.__subclasses__():
for sub in cls.__subclasses__():
for s in sub.get_subclasses():
yield s
def getResult(self, episodes):
"""
Returns a result of the correct type for this provider
"""
return SearchResult(episodes)
def make_url(self, url):
urls = []
bt_cache_urls = [
'https://torrentproject.se/torrent/{torrent_hash}.torrent',
'https://btdig.com/torrent/{torrent_hash}.torrent',
'https://torrage.info/torrent/{torrent_hash}.torrent',
'https://thetorrent.org/torrent/{torrent_hash}.torrent',
'https://itorrents.org/torrent/{torrent_hash}.torrent'
]
if url.startswith('magnet'):
try:
torrent_hash = str(
re.findall(r'urn:btih:([\w]{32,40})', url)[0]).upper()
try:
torrent_name = re.findall('dn=([^&]+)', url)[0]
except Exception:
torrent_name = 'NO_DOWNLOAD_NAME'
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
if not torrent_hash:
sickrage.srCore.srLogger.error(
"Unable to extract torrent hash from magnet: " + url)
return urls
urls = [
x.format(torrent_hash=torrent_hash,
torrent_name=torrent_name) for x in bt_cache_urls
]
except Exception:
sickrage.srCore.srLogger.error(
"Unable to extract torrent hash or name from magnet: " +
url)
return urls
else:
urls = [url]
random.shuffle(urls)
return urls
def make_filename(self, name):
return ""
def downloadResult(self, result):
"""
Save the result to disk.
"""
# check for auth
if not self._doLogin:
return False
urls = self.make_url(result.url)
filename = self.make_filename(result.name)
for url in urls:
if 'NO_DOWNLOAD_NAME' in url:
continue
sickrage.srCore.srLogger.info("Downloading a result from " +
self.name + " at " + url)
# Support for Jackett/TorzNab
if url.endswith('torrent') and filename.endswith('nzb'):
filename = filename.rsplit('.', 1)[0] + '.' + 'torrent'
if sickrage.srCore.srWebSession.download(
url,
filename,
headers=(None, {
'Referer': '/'.join(url.split('/')[:3]) + '/'
})[url.startswith('http')]):
if self._verify_download(filename):
sickrage.srCore.srLogger.info("Saved result to " +
filename)
return True
else:
sickrage.srCore.srLogger.warning("Could not download %s" %
url)
remove_file_failed(filename)
if len(urls):
sickrage.srCore.srLogger.warning("Failed to download any results")
return False
def _verify_download(self, file_name=None):
"""
Checks the saved file to see if it was actually valid, if not then consider the download a failure.
"""
# primitive verification of torrents, just make sure we didn't get a text file or something
if file_name.endswith('torrent'):
try:
with io.open(file_name, 'rb') as f:
parser = guessParser(StringInputStream(f.read()))
if parser and parser._getMimeType(
) == 'application/x-bittorrent':
return True
except Exception as e:
sickrage.srCore.srLogger.debug(
"Failed to validate torrent file: {}".format(e.message))
sickrage.srCore.srLogger.debug(
"Result is not a valid torrent file")
return False
return True
def getQuality(self, item, anime=False):
"""
Figures out the quality of the given RSS item node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns a Quality value obtained from the node's data
"""
(title, url) = self._get_title_and_url(item)
quality = Quality.sceneQuality(title, anime)
return quality
def search(self,
search_params,
search_mode='eponly',
epcount=0,
age=0,
epObj=None):
return []
def _get_season_search_strings(self, episode):
return [{}]
def _get_episode_search_strings(self, eb_obj, add_string=''):
return [{}]
def _get_title_and_url(self, item):
"""
Retrieves the title and URL data from the item XML node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns: A tuple containing two strings representing title and URL respectively
"""
title = item.get('title', '').replace(' ', '.')
url = item.get('link', '').replace('&',
'&').replace('%26tr%3D', '&tr=')
return title, url
def _get_size(self, item):
"""Gets the size from the item"""
sickrage.srCore.srLogger.error(
"Provider type doesn't have ability to provide download size implemented yet"
)
return -1
def _get_files(self, url):
"""Gets dict of files with sizes from the item"""
sickrage.srCore.srLogger.error(
"Provider type doesn't have _get_files() implemented yet")
return {}
def findSearchResults(self,
show,
episodes,
search_mode,
manualSearch=False,
downCurQuality=False,
cacheOnly=False):
if not self.check_auth:
return
self.show = show
results = {}
itemList = []
searched_scene_season = None
for epObj in episodes:
# search cache for episode result
cacheResult = self.cache.search_cache(epObj, manualSearch,
downCurQuality)
if cacheResult:
if epObj.episode not in results:
results[epObj.episode] = cacheResult[epObj.episode]
else:
results[epObj.episode].extend(cacheResult[epObj.episode])
# found result, search next episode
continue
# skip if season already searched
if len(
episodes
) > 1 and search_mode == 'sponly' and searched_scene_season == epObj.scene_season:
continue
# mark season searched for season pack searches so we can skip later on
searched_scene_season = epObj.scene_season
# check if this is a cache only search
if cacheOnly:
continue
search_strings = []
if len(episodes) > 1 and search_mode == 'sponly':
# get season search results
search_strings = self._get_season_search_strings(epObj)
elif search_mode == 'eponly':
# get single episode search results
search_strings = self._get_episode_search_strings(epObj)
first = search_strings and isinstance(
search_strings[0], dict) and 'rid' in search_strings[0]
if first:
sickrage.srCore.srLogger.debug('First search_string has rid')
for curString in search_strings:
try:
itemList += self.search(curString,
search_mode,
len(episodes),
epObj=epObj)
except SAXParseException:
continue
if first:
first = False
if itemList:
sickrage.srCore.srLogger.debug(
'First search_string had rid, and returned results, skipping query by string'
)
break
else:
sickrage.srCore.srLogger.debug(
'First search_string had rid, but returned no results, searching with string query'
)
# if we found what we needed already from cache then return results and exit
if len(results) == len(episodes):
return results
# sort list by quality
if len(itemList):
items = {}
itemsUnknown = []
for item in itemList:
quality = self.getQuality(item, anime=show.is_anime)
if quality == Quality.UNKNOWN:
itemsUnknown += [item]
else:
if quality not in items:
items[quality] = [item]
else:
items[quality].append(item)
itemList = list(
itertools.chain(
*[v for (k, v) in sorted(items.items(), reverse=True)]))
itemList += itemsUnknown or []
# filter results
for item in itemList:
(title, url) = self._get_title_and_url(item)
# parse the file name
try:
myParser = NameParser(False)
parse_result = myParser.parse(title)
except InvalidNameException:
sickrage.srCore.srLogger.debug(
"Unable to parse the filename " + title +
" into a valid episode")
continue
except InvalidShowException:
sickrage.srCore.srLogger.debug(
"Unable to parse the filename " + title +
" into a valid show")
continue
showObj = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
addCacheEntry = False
if not (showObj.air_by_date or showObj.sports):
if search_mode == 'sponly':
if len(parse_result.episode_numbers):
sickrage.srCore.srLogger.debug(
"This is supposed to be a season pack search but the result "
+ title +
" is not a valid season pack, skipping it")
addCacheEntry = True
if len(parse_result.episode_numbers) and (
parse_result.season_number not in set(
[ep.season for ep in episodes]) or not [
ep for ep in episodes if ep.scene_episode
in parse_result.episode_numbers
]):
sickrage.srCore.srLogger.debug(
"The result " + title +
" doesn't seem to be a valid episode that we are trying to snatch, ignoring"
)
addCacheEntry = True
else:
if not len(
parse_result.episode_numbers
) and parse_result.season_number and not [
ep for ep in episodes
if ep.season == parse_result.season_number
and ep.episode in parse_result.episode_numbers
]:
sickrage.srCore.srLogger.debug(
"The result " + title +
" doesn't seem to be a valid season that we are trying to snatch, ignoring"
)
addCacheEntry = True
elif len(parse_result.episode_numbers) and not [
ep for ep in episodes
if ep.season == parse_result.season_number
and ep.episode in parse_result.episode_numbers
]:
sickrage.srCore.srLogger.debug(
"The result " + title +
" doesn't seem to be a valid episode that we are trying to snatch, ignoring"
)
addCacheEntry = True
if not addCacheEntry:
# we just use the existing info for normal searches
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
if not parse_result.is_air_by_date:
sickrage.srCore.srLogger.debug(
"This is supposed to be a date search but the result "
+ title + " didn't parse as one, skipping it")
addCacheEntry = True
else:
airdate = parse_result.air_date.toordinal()
dbData = [
x['doc'] for x in sickrage.srCore.mainDB.db.get_many(
'tv_episodes', showObj.indexerid, with_doc=True)
if x['doc']['airdate'] == airdate
]
if len(dbData) != 1:
sickrage.srCore.srLogger.warning(
"Tried to look up the date for the episode " +
title +
" but the database didn't give proper results, skipping it"
)
addCacheEntry = True
if not addCacheEntry:
actual_season = int(dbData[0]["season"])
actual_episodes = [int(dbData[0]["episode"])]
# add parsed result to cache for usage later on
if addCacheEntry:
sickrage.srCore.srLogger.debug(
"Adding item from search to cache: " + title)
self.cache.addCacheEntry(title, url, parse_result=parse_result)
continue
# make sure we want the episode
wantEp = True
for epNo in actual_episodes:
if not showObj.wantEpisode(actual_season, epNo, quality,
manualSearch, downCurQuality):
wantEp = False
break
if not wantEp:
sickrage.srCore.srLogger.info(
"RESULT:[{}] QUALITY:[{}] IGNORED!".format(
title, Quality.qualityStrings[quality]))
continue
# make a result object
epObjs = []
for curEp in actual_episodes:
epObjs.append(showObj.getEpisode(actual_season, curEp))
result = self.getResult(epObjs)
result.show = showObj
result.url = url
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(url)
result.files = self._get_files(url)
sickrage.srCore.srLogger.debug(
"FOUND RESULT:[{}] QUALITY:[{}] URL:[{}]".format(
title, Quality.qualityStrings[quality], url))
if len(epObjs) == 1:
epNum = epObjs[0].episode
sickrage.srCore.srLogger.debug("Single episode result.")
elif len(epObjs) > 1:
epNum = MULTI_EP_RESULT
sickrage.srCore.srLogger.debug(
"Separating multi-episode result to check for later - result contains episodes: "
+ str(parse_result.episode_numbers))
elif len(epObjs) == 0:
epNum = SEASON_RESULT
sickrage.srCore.srLogger.debug(
"Separating full season result to check for later")
if epNum not in results:
results[epNum] = [result]
else:
results[epNum].append(result)
return results
def find_propers(self, search_date=None):
results = self.cache.list_propers(search_date)
return [
Proper(x['name'], x['url'],
datetime.datetime.fromtimestamp(x['time']), self.show)
for x in results
]
def seed_ratio(self):
"""
Provider should override this value if custom seed ratio enabled
It should return the value of the provider seed ratio
"""
return ''
def add_cookies_from_ui(self):
"""
Adds the cookies configured from UI to the providers requests session
:return: A tuple with the the (success result, and a descriptive message in str)
"""
# This is the generic attribute used to manually add cookies for provider authentication
if self.enable_cookies and self.cookies:
cookie_validator = re.compile(r'^(\w+=\w+)(;\w+=\w+)*$')
if cookie_validator.match(self.cookies):
add_dict_to_cookiejar(
sickrage.srCore.srWebSession.cookies,
dict(x.rsplit('=', 1) for x in self.cookies.split(';')))
return True
return False
@classmethod
def getDefaultProviders(cls):
pass
@classmethod
def getProvider(cls, name):
providerMatch = [x for x in cls.getProviders() if x.name == name]
if len(providerMatch) == 1:
return providerMatch[0]
@classmethod
def getProviderByID(cls, id):
providerMatch = [x for x in cls.getProviders() if x.id == id]
if len(providerMatch) == 1:
return providerMatch[0]
@classmethod
def getProviders(cls):
modules = [TorrentProvider.type, NZBProvider.type]
for type in []:
modules += cls.loadProviders(type)
return modules
@classmethod
def loadProviders(cls, type):
providers = []
pregex = re.compile('^([^_]*?)\.py$', re.IGNORECASE)
path = os.path.join(os.path.dirname(__file__), type)
names = [pregex.match(m) for m in os.listdir(path)]
providers += [
cls.loadProvider(name.group(1), type) for name in names if name
]
return providers
@classmethod
def loadProvider(cls, name, type, *args, **kwargs):
import inspect
members = dict(
inspect.getmembers(
importlib.import_module('.{}.{}'.format(type, name),
'sickrage.providers'),
lambda x: hasattr(
x, 'type') and x not in [NZBProvider, TorrentProvider]))
return [
v for v in members.values()
if hasattr(v, 'type') and v.type == type
][0](*args, **kwargs)
class GenericProvider(object):
def __init__(self, name, url, private):
self.name = name
self.urls = {'base_url': url}
self.private = private
self.supports_backlog = True
self.supports_absolute_numbering = False
self.anime_only = False
self.search_mode = 'eponly'
self.search_fallback = False
self.enabled = False
self.enable_daily = True
self.enable_backlog = True
self.cache = TVCache(self)
self.proper_strings = ['PROPER|REPACK|REAL|RERIP']
self.search_separator = ' '
# cookies
self.enable_cookies = False
self.cookies = ''
# web session
self.session = WebSession(cloudflare=True)
@property
def id(self):
return str(re.sub(r"[^\w\d_]", "_", self.name.strip().lower()))
@property
def isEnabled(self):
return self.enabled
@property
def imageName(self):
return ""
@property
def seed_ratio(self):
return ''
@property
def isAlive(self):
return True
def _check_auth(self):
return True
def login(self):
return True
def getResult(self, season=None, episodes=None):
"""
Returns a result of the correct type for this provider
"""
return SearchResult(season, episodes)
def get_content(self, url):
if self.login():
headers = {}
if url.startswith('http'):
headers = {'Referer': '/'.join(url.split('/')[:3]) + '/'}
if not url.startswith('magnet'):
try:
return self.session.get(url, verify=False, headers=headers).content
except Exception:
pass
def make_filename(self, name):
return ""
def get_quality(self, item, anime=False):
"""
Figures out the quality of the given RSS item node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns a Quality value obtained from the node's data
"""
(title, url) = self._get_title_and_url(item)
quality = Quality.scene_quality(title, anime)
return quality
def search(self, search_strings, age=0, show_id=None, season=None, episode=None, **kwargs):
return []
@MainDB.with_session
def _get_season_search_strings(self, show_id, season, episode, session=None):
"""
Get season search strings.
"""
search_string = {
'Season': []
}
show_object = find_show(show_id, session=session)
episode_object = show_object.get_episode(season, episode)
for show_name in all_possible_show_names(show_id, episode_object.scene_season):
episode_string = "{}{}".format(show_name, self.search_separator)
if show_object.air_by_date or show_object.sports:
episode_string += str(episode_object.airdate).split('-')[0]
elif show_object.anime:
episode_string += 'Season'
else:
episode_string += 'S{season:0>2}'.format(season=episode_object.scene_season)
search_string['Season'].append(episode_string.strip())
return [search_string]
@MainDB.with_session
def _get_episode_search_strings(self, show_id, season, episode, add_string='', session=None):
"""
Get episode search strings.
"""
search_string = {
'Episode': []
}
show_object = find_show(show_id, session=session)
episode_object = show_object.get_episode(season, episode)
for show_name in all_possible_show_names(show_id, episode_object.scene_season):
episode_string = "{}{}".format(show_name, self.search_separator)
episode_string_fallback = None
if show_object.air_by_date:
episode_string += str(episode_object.airdate).replace('-', ' ')
elif show_object.sports:
episode_string += str(episode_object.airdate).replace('-', ' ')
episode_string += ('|', ' ')[len(self.proper_strings) > 1]
episode_string += episode_object.airdate.strftime('%b')
elif show_object.anime:
# If the show name is a season scene exception, we want to use the indexer episode number.
if (episode_object.scene_season > 1 and show_name in get_scene_exceptions(show_object.indexer_id, episode_object.scene_season)):
# This is apparently a season exception, let's use the scene_episode instead of absolute
ep = episode_object.scene_episode
else:
ep = episode_object.scene_absolute_number
episode_string += '{episode:0>2}'.format(episode=ep)
episode_string_fallback = episode_string + '{episode:0>3}'.format(episode=ep)
else:
episode_string += sickrage.app.naming_ep_type[2] % {
'seasonnumber': episode_object.scene_season,
'episodenumber': episode_object.scene_episode,
}
if add_string:
episode_string += self.search_separator + add_string
episode_string_fallback += self.search_separator + add_string
search_string['Episode'].append(episode_string.strip())
if episode_string_fallback:
search_string['Episode'].append(episode_string_fallback.strip())
return [search_string]
def _get_title_and_url(self, item):
"""
Retrieves the title and URL data from the item XML node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns: A tuple containing two strings representing title and URL respectively
"""
title = item.get('title', '').replace(' ', '.')
url = item.get('link', '').replace('&', '&').replace('%26tr%3D', '&tr=')
return title, url
def _get_size(self, item):
"""Gets the size from the item"""
sickrage.app.log.debug("Provider type doesn't have ability to provide download size implemented yet")
return -1
def _get_result_stats(self, item):
# Get seeders/leechers stats
seeders = item.get('seeders', -1)
leechers = item.get('leechers', -1)
return try_int(seeders, -1), try_int(leechers, -1)
@MainDB.with_session
def find_search_results(self, show_id, season, episode, search_mode, manualSearch=False, downCurQuality=False, cacheOnly=False, session=None):
provider_results = {}
item_list = []
if not self._check_auth:
return provider_results
show_object = find_show(show_id, session=session)
episode_object = show_object.get_episode(season, episode)
# search cache for episode result
provider_results = self.cache.search_cache(show_id, season, episode, manualSearch, downCurQuality)
# check if this is a cache only search
if cacheOnly:
return provider_results
search_strings = []
if search_mode == 'sponly':
# get season search results
search_strings = self._get_season_search_strings(show_id, season, episode)
elif search_mode == 'eponly':
# get single episode search results
search_strings = self._get_episode_search_strings(show_id, season, episode)
for curString in search_strings:
try:
item_list += self.search(curString, show_id=show_id, season=season, episode=episode)
except SAXParseException:
continue
# sort list by quality
if item_list:
# categorize the items into lists by quality
items = defaultdict(list)
for item in item_list:
items[self.get_quality(item, anime=show_object.is_anime)].append(item)
# temporarily remove the list of items with unknown quality
unknown_items = items.pop(Quality.UNKNOWN, [])
# make a generator to sort the remaining items by descending quality
items_list = (items[quality] for quality in sorted(items, reverse=True))
# unpack all of the quality lists into a single sorted list
items_list = list(itertools.chain(*items_list))
# extend the list with the unknown qualities, now sorted at the bottom of the list
items_list.extend(unknown_items)
# filter results
for item in item_list:
provider_result = self.getResult()
provider_result.name, provider_result.url = self._get_title_and_url(item)
# ignore invalid urls
if not validate_url(provider_result.url) and not provider_result.url.startswith('magnet'):
continue
try:
parse_result = NameParser(show_id=show_id).parse(provider_result.name)
except (InvalidNameException, InvalidShowException) as e:
sickrage.app.log.debug("{}".format(e))
continue
provider_result.show_id = parse_result.indexer_id
provider_result.quality = parse_result.quality
provider_result.release_group = parse_result.release_group
provider_result.version = parse_result.version
provider_result.size = self._get_size(item)
provider_result.seeders, provider_result.leechers = self._get_result_stats(item)
sickrage.app.log.debug("Adding item from search to cache: {}".format(provider_result.name))
self.cache.add_cache_entry(provider_result.name, provider_result.url, provider_result.seeders, provider_result.leechers, provider_result.size)
if not provider_result.show_id:
continue
provider_result_show_obj = find_show(provider_result.show_id, session=session)
if not provider_result_show_obj:
continue
if not parse_result.is_air_by_date and (provider_result_show_obj.air_by_date or provider_result_show_obj.sports):
sickrage.app.log.debug("This is supposed to be a date search but the result {} didn't parse as one, skipping it".format(provider_result.name))
continue
if search_mode == 'sponly':
if len(parse_result.episode_numbers):
sickrage.app.log.debug("This is supposed to be a season pack search but the result {} is not "
"a valid season pack, skipping it".format(provider_result.name))
continue
elif parse_result.season_number != (episode_object.season, episode_object.scene_season)[show_object.is_scene]:
sickrage.app.log.debug("This season result {} is for a season we are not searching for, skipping it".format(provider_result.name))
continue
else:
if not all([parse_result.season_number is not None, parse_result.episode_numbers,
parse_result.season_number == (episode_object.season, episode_object.scene_season)[show_object.is_scene],
(episode_object.episode, episode_object.scene_episode)[show_object.is_scene] in parse_result.episode_numbers]):
sickrage.app.log.debug("The result {} doesn't seem to be a valid episode "
"that we are trying to snatch, ignoring".format(provider_result.name))
continue
provider_result.season = int(parse_result.season_number)
provider_result.episodes = list(map(int, parse_result.episode_numbers))
# make sure we want the episode
for episode_number in provider_result.episodes.copy():
if not provider_result_show_obj.want_episode(provider_result.season, episode_number, provider_result.quality, manualSearch, downCurQuality):
sickrage.app.log.info("RESULT:[{}] QUALITY:[{}] IGNORED!".format(provider_result.name, Quality.qualityStrings[provider_result.quality]))
if episode_number in provider_result.episodes:
provider_result.episodes.remove(episode_number)
# detects if season pack and if not checks if we wanted any of the episodes
if len(provider_result.episodes) != len(parse_result.episode_numbers):
continue
sickrage.app.log.debug(
"FOUND RESULT:[{}] QUALITY:[{}] URL:[{}]".format(provider_result.name, Quality.qualityStrings[provider_result.quality], provider_result.url)
)
if len(provider_result.episodes) == 1:
episode_number = provider_result.episodes[0]
sickrage.app.log.debug("Single episode result.")
elif len(provider_result.episodes) > 1:
episode_number = MULTI_EP_RESULT
sickrage.app.log.debug("Separating multi-episode result to check for later - result contains episodes: " + str(parse_result.episode_numbers))
else:
episode_number = SEASON_RESULT
sickrage.app.log.debug("Separating full season result to check for later")
if episode_number not in provider_results:
provider_results[int(episode_number)] = [provider_result]
else:
provider_results[int(episode_number)] += [provider_result]
return provider_results
def find_propers(self, show_id, season, episode):
results = []
for term in self.proper_strings:
search_strngs = self._get_episode_search_strings(show_id, season, episode, add_string=term)
for item in self.search(search_strngs[0], show_id=show_id, season=season, episode=episode):
result = self.getResult(season, [episode])
result.name, result.url = self._get_title_and_url(item)
if not validate_url(result.url) and not result.url.startswith('magnet'):
continue
result.seeders, result.leechers = self._get_result_stats(item)
result.size = self._get_size(item)
result.date = datetime.datetime.today()
results.append(result)
return results
def add_cookies_from_ui(self):
"""
Add the cookies configured from UI to the providers requests session.
:return: dict
"""
if isinstance(self, TorrentRssProvider) and not self.cookies:
return {'result': True,
'message': 'This is a TorrentRss provider without any cookies provided. '
'Cookies for this provider are considered optional.'}
# This is the generic attribute used to manually add cookies for provider authentication
if not self.enable_cookies:
return {'result': False,
'message': 'Adding cookies is not supported for provider: {}'.format(self.name)}
if not self.cookies:
return {'result': False,
'message': 'No Cookies added from ui for provider: {}'.format(self.name)}
cookie_validator = re.compile(r'^([\w%]+=[\w%]+)(;[\w%]+=[\w%]+)*$')
if not cookie_validator.match(self.cookies):
sickrage.app.alerts.message(
'Failed to validate cookie for provider {}'.format(self.name),
'Cookie is not correctly formatted: {}'.format(self.cookies))
return {'result': False,
'message': 'Cookie is not correctly formatted: {}'.format(self.cookies)}
if hasattr(self, 'required_cookies') and not all(
req_cookie in [x.rsplit('=', 1)[0] for x in self.cookies.split(';')] for req_cookie in
self.required_cookies):
return {'result': False,
'message': "You haven't configured the required cookies. Please login at {provider_url}, "
"and make sure you have copied the following cookies: {required_cookies!r}"
.format(provider_url=self.name, required_cookies=self.required_cookies)}
# cookie_validator got at least one cookie key/value pair, let's return success
add_dict_to_cookiejar(self.session.cookies, dict(x.rsplit('=', 1) for x in self.cookies.split(';')))
return {'result': True,
'message': ''}
def check_required_cookies(self):
"""
Check if we have the required cookies in the requests sessions object.
Meaning that we've already successfully authenticated once, and we don't need to go through this again.
Note! This doesn't mean the cookies are correct!
"""
if hasattr(self, 'required_cookies'):
return all(dict_from_cookiejar(self.session.cookies).get(cookie) for cookie in self.required_cookies)
# A reminder for the developer, implementing cookie based authentication.
sickrage.app.log.error(
'You need to configure the required_cookies attribute, for the provider: {}'.format(self.name))
def cookie_login(self, check_login_text, check_url=None):
"""
Check the response for text that indicates a login prompt.
In that case, the cookie authentication was not successful.
:param check_login_text: A string that's visible when the authentication failed.
:param check_url: The url to use to test the login with cookies. By default the providers home page is used.
:return: False when authentication was not successful. True if successful.
"""
check_url = check_url or self.urls['base_url']
if self.check_required_cookies():
# All required cookies have been found within the current session, we don't need to go through this again.
return True
if self.cookies:
result = self.add_cookies_from_ui()
if not result['result']:
sickrage.app.alerts.message(result['message'])
sickrage.app.log.warning(result['message'])
return False
else:
sickrage.app.log.warning('Failed to login, you will need to add your cookies in the provider '
'settings')
sickrage.app.alerts.error(
'Failed to auth with {provider}'.format(provider=self.name),
'You will need to add your cookies in the provider settings')
return False
response = self.session.get(check_url)
if any([not response, not (response.text and response.status_code == 200),
check_login_text.lower() in response.text.lower()]):
sickrage.app.log.warning('Please configure the required cookies for this provider. Check your '
'provider settings')
sickrage.app.alerts.error(
'Wrong cookies for {}'.format(self.name),
'Check your provider settings'
)
self.session.cookies.clear()
return False
else:
return True
@classmethod
def getDefaultProviders(cls):
pass
@classmethod
def getProvider(cls, name):
providerMatch = [x for x in cls.get_providers() if x.name == name]
if len(providerMatch) == 1:
return providerMatch[0]
@classmethod
def getProviderByID(cls, id):
providerMatch = [x for x in cls.get_providers() if x.id == id]
if len(providerMatch) == 1:
return providerMatch[0]
@classmethod
def get_providers(cls):
modules = [TorrentProvider.type, NZBProvider.type]
for type in []:
modules += cls.load_providers(type)
return modules
@classmethod
def load_providers(cls, provider_type):
providers = []
for (__, name, __) in pkgutil.iter_modules([os.path.join(os.path.dirname(__file__), provider_type)]):
imported_module = importlib.import_module('.{}.{}'.format(provider_type, name), package='sickrage.providers')
for __, klass in inspect.getmembers(imported_module, predicate=lambda o: all([inspect.isclass(o) and issubclass(o, GenericProvider),
o is not NZBProvider, o is not TorrentProvider,
getattr(o, 'type', None) == provider_type])):
providers += [klass()]
break
return providers
class GenericProvider(object):
def __init__(self, name, url, private):
self.name = name
self.urls = {'base_url': url}
self.private = private
self.show = None
self.supports_backlog = False
self.supports_absolute_numbering = False
self.anime_only = False
self.search_mode = 'eponly'
self.search_fallback = False
self.enabled = False
self.enable_daily = False
self.enable_backlog = False
self.cache = TVCache(self)
self.proper_strings = ['PROPER|REPACK|REAL']
self.enable_cookies = False
self.cookies = ''
self.rss_cookies = ''
self.cookie_jar = dict()
@property
def id(self):
return str(re.sub(r"[^\w\d_]", "_", self.name.strip().lower()))
@property
def isEnabled(self):
return self.enabled
@property
def imageName(self):
return ""
def check_auth(self):
return True
def _doLogin(self):
return True
@classmethod
def get_subclasses(cls):
yield cls
if cls.__subclasses__():
for sub in cls.__subclasses__():
for s in sub.get_subclasses():
yield s
def getResult(self, episodes):
"""
Returns a result of the correct type for this provider
"""
return SearchResult(episodes)
def make_url(self, url):
urls = []
btcache_urls = [
'http://torrentproject.se/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
'http://torrage.info/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://itorrents.org/torrent/{torrent_hash}.torrent'
]
if url.startswith('magnet'):
try:
torrent_hash = str(re.findall(r'urn:btih:([\w]{32,40})', url)[0]).upper()
try:
torrent_name = re.findall('dn=([^&]+)', url)[0]
except Exception:
torrent_name = 'NO_DOWNLOAD_NAME'
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
if not torrent_hash:
sickrage.srCore.srLogger.error("Unable to extract torrent hash from magnet: " + url)
return urls
urls = [x.format(torrent_hash=torrent_hash, torrent_name=torrent_name) for x in btcache_urls]
except Exception:
sickrage.srCore.srLogger.error("Unable to extract torrent hash or name from magnet: " + url)
return urls
else:
urls = [url]
random.shuffle(urls)
return urls
def make_filename(self, name):
return ""
def downloadResult(self, result):
"""
Save the result to disk.
"""
# check for auth
if not self._doLogin:
return False
urls = self.make_url(result.url)
filename = self.make_filename(result.name)
for url in urls:
if 'NO_DOWNLOAD_NAME' in url:
continue
sickrage.srCore.srLogger.info("Downloading a result from " + self.name + " at " + url)
# Support for Jackett/TorzNab
if url.endswith('torrent') and filename.endswith('nzb'):
filename = filename.rsplit('.', 1)[0] + '.' + 'torrent'
if sickrage.srCore.srWebSession.download(url,
filename,
headers=(None, {
'Referer': '/'.join(url.split('/')[:3]) + '/'
})[url.startswith('http')]):
if self._verify_download(filename):
sickrage.srCore.srLogger.info("Saved result to " + filename)
return True
else:
sickrage.srCore.srLogger.warning("Could not download %s" % url)
remove_file_failed(filename)
if len(urls):
sickrage.srCore.srLogger.warning("Failed to download any results")
return False
def _verify_download(self, file_name=None):
"""
Checks the saved file to see if it was actually valid, if not then consider the download a failure.
"""
# primitive verification of torrents, just make sure we didn't get a text file or something
if file_name.endswith('torrent'):
try:
with open(file_name, 'rb') as file:
mime_type = guessParser(StringInputStream(file.read()))._getMimeType()
if mime_type == 'application/x-bittorrent':
return True
except Exception as e:
sickrage.srCore.srLogger.debug("Failed to validate torrent file: {}".format(e.message))
sickrage.srCore.srLogger.debug("Result is not a valid torrent file")
return False
return True
def getQuality(self, item, anime=False):
"""
Figures out the quality of the given RSS item node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns a Quality value obtained from the node's data
"""
(title, url) = self._get_title_and_url(item)
quality = Quality.sceneQuality(title, anime)
return quality
def search(self, search_params, search_mode='eponly', epcount=0, age=0, epObj=None):
return []
def _get_season_search_strings(self, episode):
return [{}]
def _get_episode_search_strings(self, eb_obj, add_string=''):
return [{}]
def _get_title_and_url(self, item):
"""
Retrieves the title and URL data from the item XML node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns: A tuple containing two strings representing title and URL respectively
"""
title = item.get('title', '').replace(' ', '.')
url = item.get('link', '').replace('&', '&').replace('%26tr%3D', '&tr=')
return title, url
def _get_size(self, item):
"""Gets the size from the item"""
sickrage.srCore.srLogger.error("Provider type doesn't have ability to provide download size implemented yet")
return -1
def _get_files(self, url):
"""Gets dict of files with sizes from the item"""
sickrage.srCore.srLogger.error("Provider type doesn't have _get_files() implemented yet")
return {}
def findSearchResults(self, show, episodes, search_mode, manualSearch=False, downCurQuality=False, cacheOnly=False):
if not self.check_auth:
return
self.show = show
results = {}
itemList = []
searched_scene_season = None
for epObj in episodes:
# search cache for episode result
cacheResult = self.cache.search_cache(epObj, manualSearch, downCurQuality)
if cacheResult:
if epObj.episode not in results:
results[epObj.episode] = cacheResult[epObj.episode]
else:
results[epObj.episode].extend(cacheResult[epObj.episode])
# found result, search next episode
continue
# skip if season already searched
if len(episodes) > 1 and search_mode == 'sponly' and searched_scene_season == epObj.scene_season:
continue
# mark season searched for season pack searches so we can skip later on
searched_scene_season = epObj.scene_season
# check if this is a cache only search
if cacheOnly:
continue
search_strings = []
if len(episodes) > 1 and search_mode == 'sponly':
# get season search results
search_strings = self._get_season_search_strings(epObj)
elif search_mode == 'eponly':
# get single episode search results
search_strings = self._get_episode_search_strings(epObj)
first = search_strings and isinstance(search_strings[0], dict) and 'rid' in search_strings[0]
if first:
sickrage.srCore.srLogger.debug('First search_string has rid')
for curString in search_strings:
try:
itemList += self.search(curString, search_mode, len(episodes), epObj=epObj)
except SAXParseException:
continue
if first:
first = False
if itemList:
sickrage.srCore.srLogger.debug(
'First search_string had rid, and returned results, skipping query by string')
break
else:
sickrage.srCore.srLogger.debug(
'First search_string had rid, but returned no results, searching with string query')
# if we found what we needed already from cache then return results and exit
if len(results) == len(episodes):
return results
# sort list by quality
if len(itemList):
items = {}
itemsUnknown = []
for item in itemList:
quality = self.getQuality(item, anime=show.is_anime)
if quality == Quality.UNKNOWN:
itemsUnknown += [item]
else:
if quality not in items:
items[quality] = [item]
else:
items[quality].append(item)
itemList = list(itertools.chain(*[v for (k, v) in sorted(items.items(), reverse=True)]))
itemList += itemsUnknown or []
# filter results
for item in itemList:
(title, url) = self._get_title_and_url(item)
# parse the file name
try:
myParser = NameParser(False)
parse_result = myParser.parse(title)
except InvalidNameException:
sickrage.srCore.srLogger.debug("Unable to parse the filename " + title + " into a valid episode")
continue
except InvalidShowException:
sickrage.srCore.srLogger.debug("Unable to parse the filename " + title + " into a valid show")
continue
showObj = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
addCacheEntry = False
if not (showObj.air_by_date or showObj.sports):
if search_mode == 'sponly':
if len(parse_result.episode_numbers):
sickrage.srCore.srLogger.debug(
"This is supposed to be a season pack search but the result " + title + " is not a valid season pack, skipping it")
addCacheEntry = True
if len(parse_result.episode_numbers) and (
parse_result.season_number not in set([ep.season for ep in episodes])
or not [ep for ep in episodes if ep.scene_episode in parse_result.episode_numbers]):
sickrage.srCore.srLogger.debug(
"The result " + title + " doesn't seem to be a valid episode that we are trying to snatch, ignoring")
addCacheEntry = True
else:
if not len(parse_result.episode_numbers) and parse_result.season_number and not [ep for ep in
episodes if
ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers]:
sickrage.srCore.srLogger.debug(
"The result " + title + " doesn't seem to be a valid season that we are trying to snatch, ignoring")
addCacheEntry = True
elif len(parse_result.episode_numbers) and not [ep for ep in episodes if
ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers]:
sickrage.srCore.srLogger.debug(
"The result " + title + " doesn't seem to be a valid episode that we are trying to snatch, ignoring")
addCacheEntry = True
if not addCacheEntry:
# we just use the existing info for normal searches
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
if not parse_result.is_air_by_date:
sickrage.srCore.srLogger.debug(
"This is supposed to be a date search but the result " + title + " didn't parse as one, skipping it")
addCacheEntry = True
else:
airdate = parse_result.air_date.toordinal()
dbData = [x['doc'] for x in sickrage.srCore.mainDB.db.get_many('tv_episodes', showObj.indexerid, with_doc=True)
if x['doc']['airdate'] == airdate]
if len(dbData) != 1:
sickrage.srCore.srLogger.warning(
"Tried to look up the date for the episode " + title + " but the database didn't give proper results, skipping it")
addCacheEntry = True
if not addCacheEntry:
actual_season = int(dbData[0]["season"])
actual_episodes = [int(dbData[0]["episode"])]
# add parsed result to cache for usage later on
if addCacheEntry:
sickrage.srCore.srLogger.debug("Adding item from search to cache: " + title)
self.cache.addCacheEntry(title, url, parse_result=parse_result)
continue
# make sure we want the episode
wantEp = True
for epNo in actual_episodes:
if not showObj.wantEpisode(actual_season, epNo, quality, manualSearch, downCurQuality):
wantEp = False
break
if not wantEp:
sickrage.srCore.srLogger.info(
"RESULT:[{}] QUALITY:[{}] IGNORED!".format(title, Quality.qualityStrings[quality]))
continue
# make a result object
epObjs = []
for curEp in actual_episodes:
epObjs.append(showObj.getEpisode(actual_season, curEp))
result = self.getResult(epObjs)
result.show = showObj
result.url = url
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(url)
result.files = self._get_files(url)
sickrage.srCore.srLogger.debug(
"FOUND RESULT:[{}] QUALITY:[{}] URL:[{}]".format(title, Quality.qualityStrings[quality], url))
if len(epObjs) == 1:
epNum = epObjs[0].episode
sickrage.srCore.srLogger.debug("Single episode result.")
elif len(epObjs) > 1:
epNum = MULTI_EP_RESULT
sickrage.srCore.srLogger.debug(
"Separating multi-episode result to check for later - result contains episodes: " + str(
parse_result.episode_numbers))
elif len(epObjs) == 0:
epNum = SEASON_RESULT
sickrage.srCore.srLogger.debug("Separating full season result to check for later")
if epNum not in results:
results[epNum] = [result]
else:
results[epNum].append(result)
return results
def find_propers(self, search_date=None):
results = self.cache.list_propers(search_date)
return [Proper(x['name'], x['url'], datetime.datetime.fromtimestamp(x['time']), self.show) for x in
results]
def seed_ratio(self):
'''
Provider should override this value if custom seed ratio enabled
It should return the value of the provider seed ratio
'''
return ''
def add_cookies_from_ui(self):
"""
Adds the cookies configured from UI to the providers requests session
:return: A tuple with the the (success result, and a descriptive message in str)
"""
# This is the generic attribute used to manually add cookies for provider authentication
if self.enable_cookies and self.cookies:
cookie_validator = re.compile(r'^(\w+=\w+)(;\w+=\w+)*$')
if cookie_validator.match(self.cookies):
add_dict_to_cookiejar(sickrage.srCore.srWebSession.cookies,
dict(x.rsplit('=', 1) for x in self.cookies.split(';')))
return True
return False
@classmethod
def getDefaultProviders(cls):
pass
@classmethod
def getProvider(cls, name):
providerMatch = [x for x in cls.getProviders() if x.name == name]
if len(providerMatch) == 1:
return providerMatch[0]
@classmethod
def getProviderByID(cls, id):
providerMatch = [x for x in cls.getProviders() if x.id == id]
if len(providerMatch) == 1:
return providerMatch[0]
@classmethod
def getProviders(cls):
modules = [TorrentProvider.type, NZBProvider.type]
for type in []:
modules += cls.loadProviders(type)
return modules
@classmethod
def loadProviders(cls, type):
providers = []
pregex = re.compile('^([^_]*?)\.py$', re.IGNORECASE)
path = os.path.join(os.path.dirname(__file__), type)
names = [pregex.match(m) for m in os.listdir(path)]
providers += [cls.loadProvider(name.group(1), type) for name in names if name]
return providers
@classmethod
def loadProvider(cls, name, type, *args, **kwargs):
import inspect
members = dict(
inspect.getmembers(
importlib.import_module('.{}.{}'.format(type, name), 'sickrage.providers'),
lambda x: hasattr(x, 'type') and x not in [NZBProvider, TorrentProvider])
)
return [v for v in members.values() if hasattr(v, 'type') and v.type == type][0](
*args, **kwargs)
