def __init__(self, provider_obj):
TVCache.__init__(self, provider_obj)
# only poll newznab providers every 30 minutes
self.minTime = 30
self.last_search = datetime.now()
def __init__(self, name):
# these need to be set in the subclass
self.name = name
self.urls = {}
self.url = ""
self.public = False
self.show = None
self.supportsBacklog = False
self.supportsAbsoluteNumbering = False
self.anime_only = False
self.search_mode = None
self.search_fallback = False
self.enabled = False
self.enable_daily = False
self.enable_backlog = False
self.cache = TVCache(self)
self.session = None
self.proper_strings = ["PROPER|REPACK|REAL"]
self.headers = {}
self.session = None
self.btCacheURLS = [
"http://torcache.net/torrent/{torrent_hash}.torrent",
"http://thetorrent.org/torrent/{torrent_hash}.torrent",
"http://btdig.com/torrent/{torrent_hash}.torrent",
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
# 'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
def __init__(self, provider_obj):
TVCache.__init__(self, provider_obj)
self.minTime = 20
class GenericProvider(object):
NZB = "nzb"
TORRENT = "torrent"
types = [NZB, TORRENT]
type = None
def __init__(self, name):
# these need to be set in the subclass
self.name = name
self.urls = {}
self.url = ""
self.public = False
self.show = None
self.supportsBacklog = False
self.supportsAbsoluteNumbering = False
self.anime_only = False
self.search_mode = None
self.search_fallback = False
self.enabled = False
self.enable_daily = False
self.enable_backlog = False
self.cache = TVCache(self)
self.session = None
self.proper_strings = ["PROPER|REPACK|REAL"]
self.headers = {}
self.session = None
self.btCacheURLS = [
"http://torcache.net/torrent/{torrent_hash}.torrent",
"http://thetorrent.org/torrent/{torrent_hash}.torrent",
"http://btdig.com/torrent/{torrent_hash}.torrent",
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
# 'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
@property
def id(self):
return self._makeID()
@property
def isActive(self):
return False
@property
def isEnabled(self):
return self.enabled
@property
def imageName(self):
return self.id + ".png"
def _makeID(self):
return str(re.sub(r"[^\w\d_]", "_", self.name.strip().lower()))
def _checkAuth(self):
return True
def _doLogin(self):
return True
@classmethod
def get_subclasses(cls):
yield cls
if cls.__subclasses__():
for sub in cls.__subclasses__():
for s in sub.get_subclasses():
yield s
def getResult(self, episodes):
"""
Returns a result of the correct type for this provider
"""
try:
result = {"nzb": NZBSearchResult, "torrent": TorrentSearchResult}[self.type](episodes)
except:
result = SearchResult(episodes)
result.provider = self
return result
def getURL(self, url, post_data=None, params=None, timeout=30, json=False, needBytes=False):
"""
By default this is just a simple urlopen call but this method should be overridden
for providers with special URL requirements (like cookies)
"""
return getURL(
url,
post_data=post_data,
params=params,
headers=self.headers,
timeout=timeout,
session=self.session,
json=json,
needBytes=needBytes,
)
def _makeURL(self, result):
urls = []
filename = ""
if result.url.startswith("magnet"):
try:
torrent_hash = re.findall(r"urn:btih:([\w]{32,40})", result.url)[0].upper()
try:
torrent_name = re.findall("dn=([^&]+)", result.url)[0]
except Exception:
torrent_name = "NO_DOWNLOAD_NAME"
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
if not torrent_hash:
sickrage.srLogger.error("Unable to extract torrent hash from magnet: " + result.url)
return urls, filename
urls = random.shuffle(
[x.format(torrent_hash=torrent_hash, torrent_name=torrent_name) for x in self.btCacheURLS]
)
except Exception:
sickrage.srLogger.error("Unable to extract torrent hash or name from magnet: " + result.url)
return urls, filename
else:
urls = [result.url]
if self.type == self.TORRENT:
filename = os.path.join(sickrage.srConfig.TORRENT_DIR, sanitizeFileName(result.name) + "." + self.type)
elif self.type == self.NZB:
filename = os.path.join(sickrage.srConfig.NZB_DIR, sanitizeFileName(result.name) + "." + self.type)
return urls, filename
def downloadResult(self, result):
"""
Save the result to disk.
"""
# check for auth
if not self._doLogin:
return False
urls, filename = self._makeURL(result)
for url in urls:
if "NO_DOWNLOAD_NAME" in url:
continue
if url.startswith("http"):
self.headers.update({"Referer": "/".join(url.split("/")[:3]) + "/"})
sickrage.srLogger.info("Downloading a result from " + self.name + " at " + url)
# Support for Jackett/TorzNab
if url.endswith(GenericProvider.TORRENT) and filename.endswith(GenericProvider.NZB):
filename = filename.rsplit(".", 1)[0] + "." + GenericProvider.TORRENT
if download_file(url, filename, session=self.session, headers=self.headers):
if self._verify_download(filename):
sickrage.srLogger.info("Saved result to " + filename)
return True
else:
sickrage.srLogger.warning("Could not download %s" % url)
remove_file_failed(filename)
if len(urls):
sickrage.srLogger.warning("Failed to download any results")
return False
def _verify_download(self, file_name=None):
"""
Checks the saved file to see if it was actually valid, if not then consider the download a failure.
"""
# primitive verification of torrents, just make sure we didn't get a text file or something
if file_name.endswith(GenericProvider.TORRENT):
try:
for byte in readFileBuffered(file_name):
mime_type = guessParser(StringInputStream(byte))._getMimeType()
if mime_type == "application/x-bittorrent":
# clean up
del mime_type
return True
except Exception as e:
sickrage.srLogger.debug("Failed to validate torrent file: {}".format(e.message))
sickrage.srLogger.debug("Result is not a valid torrent file")
return False
return True
def searchRSS(self, episodes):
return self.cache.findNeededEpisodes(episodes)
def getQuality(self, item, anime=False):
"""
Figures out the quality of the given RSS item node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns a Quality value obtained from the node's data
"""
(title, url) = self._get_title_and_url(item)
quality = Quality.sceneQuality(title, anime)
return quality
def _doSearch(self, search_params, search_mode="eponly", epcount=0, age=0, epObj=None):
return []
def _get_season_search_strings(self, episode):
return [{}]
def _get_episode_search_strings(self, eb_obj, add_string=""):
return [{}]
def _get_title_and_url(self, item):
"""
Retrieves the title and URL data from the item XML node
item: An elementtree.ElementTree element representing the <item> tag of the RSS feed
Returns: A tuple containing two strings representing title and URL respectively
"""
title = item.get("title", "")
if title:
title = "" + title.replace(" ", ".")
url = item.get("link", "")
if url:
url = url.replace("&", "&").replace("%26tr%3D", "&tr=")
return title, url
def _get_size(self, item):
"""Gets the size from the item"""
sickrage.srLogger.error("Provider type doesn't have _get_size() implemented yet")
return -1
def findSearchResults(self, show, episodes, search_mode, manualSearch=False, downCurQuality=False):
if not self._checkAuth:
return
self.show = show
results = {}
itemList = []
searched_scene_season = None
for epObj in episodes:
# search cache for episode result
cacheResult = self.cache.searchCache(epObj, manualSearch, downCurQuality)
if cacheResult:
if epObj.episode not in results:
results[epObj.episode] = cacheResult
else:
results[epObj.episode].extend(cacheResult)
# found result, search next episode
continue
# skip if season already searched
if len(episodes) > 1 and search_mode == "sponly" and searched_scene_season == epObj.scene_season:
continue
# mark season searched for season pack searches so we can skip later on
searched_scene_season = epObj.scene_season
search_strings = []
if len(episodes) > 1 and search_mode == "sponly":
# get season search results
search_strings = self._get_season_search_strings(epObj)
elif search_mode == "eponly":
# get single episode search results
search_strings = self._get_episode_search_strings(epObj)
first = search_strings and isinstance(search_strings[0], dict) and "rid" in search_strings[0]
if first:
sickrage.srLogger.debug("First search_string has rid")
for curString in search_strings:
itemList += self._doSearch(curString, search_mode, len(episodes), epObj=epObj)
if first:
first = False
if itemList:
sickrage.srLogger.debug(
"First search_string had rid, and returned results, skipping query by string"
)
break
else:
sickrage.srLogger.debug(
"First search_string had rid, but returned no results, searching with string query"
)
# if we found what we needed already from cache then return results and exit
if len(results) == len(episodes):
return results
# sort list by quality
if len(itemList):
items = {}
itemsUnknown = []
for item in itemList:
quality = self.getQuality(item, anime=show.is_anime)
if quality == Quality.UNKNOWN:
itemsUnknown += [item]
else:
if quality not in items:
items[quality] = [item]
else:
items[quality].append(item)
itemList = list(itertools.chain(*[v for (k, v) in sorted(items.iteritems(), reverse=True)]))
itemList += itemsUnknown if itemsUnknown else []
# filter results
cl = []
for item in itemList:
(title, url) = self._get_title_and_url(item)
# parse the file name
try:
myParser = NameParser(False)
parse_result = myParser.parse(title)
except InvalidNameException:
sickrage.srLogger.debug("Unable to parse the filename " + title + " into a valid episode")
continue
except InvalidShowException:
sickrage.srLogger.debug("Unable to parse the filename " + title + " into a valid show")
continue
showObj = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
addCacheEntry = False
if not (showObj.air_by_date or showObj.sports):
if search_mode == "sponly":
if len(parse_result.episode_numbers):
sickrage.srLogger.debug(
"This is supposed to be a season pack search but the result "
+ title
+ " is not a valid season pack, skipping it"
)
addCacheEntry = True
if len(parse_result.episode_numbers) and (
parse_result.season_number not in set([ep.season for ep in episodes])
or not [ep for ep in episodes if ep.scene_episode in parse_result.episode_numbers]
):
sickrage.srLogger.debug(
"The result "
+ title
+ " doesn't seem to be a valid episode that we are trying to snatch, ignoring"
)
addCacheEntry = True
else:
if (
not len(parse_result.episode_numbers)
and parse_result.season_number
and not [
ep
for ep in episodes
if ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers
]
):
sickrage.srLogger.debug(
"The result "
+ title
+ " doesn't seem to be a valid season that we are trying to snatch, ignoring"
)
addCacheEntry = True
elif len(parse_result.episode_numbers) and not [
ep
for ep in episodes
if ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers
]:
sickrage.srLogger.debug(
"The result "
+ title
+ " doesn't seem to be a valid episode that we are trying to snatch, ignoring"
)
addCacheEntry = True
if not addCacheEntry:
# we just use the existing info for normal searches
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
if not parse_result.is_air_by_date:
sickrage.srLogger.debug(
"This is supposed to be a date search but the result "
+ title
+ " didn't parse as one, skipping it"
)
addCacheEntry = True
else:
airdate = parse_result.air_date.toordinal()
sql_results = main_db.MainDB().select(
"SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?",
[showObj.indexerid, airdate],
)
if len(sql_results) != 1:
sickrage.srLogger.warning(
"Tried to look up the date for the episode "
+ title
+ " but the database didn't give proper results, skipping it"
)
addCacheEntry = True
if not addCacheEntry:
actual_season = int(sql_results[0][b"season"])
actual_episodes = [int(sql_results[0][b"episode"])]
# add parsed result to cache for usage later on
if addCacheEntry:
sickrage.srLogger.debug("Adding item from search to cache: " + title)
# pylint: disable=W0212
# Access to a protected member of a client class
ci = self.cache._addCacheEntry(title, url, parse_result=parse_result)
if ci is not None:
cl.append(ci)
continue
# make sure we want the episode
wantEp = True
for epNo in actual_episodes:
if not showObj.wantEpisode(actual_season, epNo, quality, manualSearch, downCurQuality):
wantEp = False
break
if not wantEp:
sickrage.srLogger.info(
"Ignoring result "
+ title
+ " because we don't want an episode that is "
+ Quality.qualityStrings[quality]
)
continue
sickrage.srLogger.debug("Found result " + title + " at " + url)
# make a result object
epObj = []
for curEp in actual_episodes:
epObj.append(showObj.getEpisode(actual_season, curEp))
result = self.getResult(epObj)
result.show = showObj
result.url = url
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(item)
if len(epObj) == 1:
epNum = epObj[0].episode
sickrage.srLogger.debug("Single episode result.")
elif len(epObj) > 1:
epNum = MULTI_EP_RESULT
sickrage.srLogger.debug(
"Separating multi-episode result to check for later - result contains episodes: "
+ str(parse_result.episode_numbers)
)
elif len(epObj) == 0:
epNum = SEASON_RESULT
sickrage.srLogger.debug("Separating full season result to check for later")
if epNum not in results:
results[epNum] = [result]
else:
results[epNum].append(result)
# check if we have items to add to cache
if len(cl) > 0:
self.cache._getDB().mass_action(cl)
del cl # cleanup
return results
def findPropers(self, search_date=None):
results = self.cache.listPropers(search_date)
return [Proper(x[b"name"], x[b"url"], datetime.fromtimestamp(x[b"time"]), self.show) for x in results]
def seedRatio(self):
"""
Provider should override this value if custom seed ratio enabled
It should return the value of the provider seed ratio
"""
return ""
@classmethod
def getDefaultProviders(cls):
return ""
@classmethod
def getProvider(cls, name):
providerMatch = [x for x in cls.getProviderList() if x.name == name]
if len(providerMatch) == 1:
return providerMatch[0]
@classmethod
def getProviderByID(cls, id):
providerMatch = [x for x in cls.getProviderList() if x.id == id]
if len(providerMatch) == 1:
return providerMatch[0]
@classmethod
def getProviderList(cls, data=None):
modules = []
for type in GenericProvider.types:
modules += cls.loadProviders(type)
return modules
@classmethod
def loadProviders(cls, type):
providers = []
pregex = re.compile("^([^_]*?)\.py$", re.IGNORECASE)
path = os.path.join(os.path.dirname(__file__), type)
names = [pregex.match(m) for m in os.listdir(path)]
providers += [cls.loadProvider(name.group(1), type) for name in names if name]
return providers
@classmethod
def loadProvider(cls, name, type, *args, **kwargs):
import inspect, sys
sys.path.append(os.path.join(os.path.dirname(__file__), "sickrage"))
members = dict(
inspect.getmembers(
importlib.import_module(".{}.{}".format(type, name), "providers"),
lambda x: hasattr(x, "type") and x not in [NZBProvider, TorrentProvider],
)
)
return [v for v in members.values() if hasattr(v, "type") and v.type == type][0](*args, **kwargs)
def __init__(self, provider_obj):
TVCache.__init__(self, provider_obj)
self.minTime = 15
