def __init__(self, provider_obj): TVCache.__init__(self, provider_obj) # only poll newznab providers every 30 minutes self.minTime = 30 self.last_search = datetime.now()
def __init__(self, name): # these need to be set in the subclass self.name = name self.urls = {} self.url = "" self.public = False self.show = None self.supportsBacklog = False self.supportsAbsoluteNumbering = False self.anime_only = False self.search_mode = None self.search_fallback = False self.enabled = False self.enable_daily = False self.enable_backlog = False self.cache = TVCache(self) self.session = None self.proper_strings = ["PROPER|REPACK|REAL"] self.headers = {} self.session = None self.btCacheURLS = [ "http://torcache.net/torrent/{torrent_hash}.torrent", "http://thetorrent.org/torrent/{torrent_hash}.torrent", "http://btdig.com/torrent/{torrent_hash}.torrent", # 'http://torrage.com/torrent/{torrent_hash}.torrent', # 'http://itorrents.org/torrent/{torrent_hash}.torrent', ]
def __init__(self, provider_obj): TVCache.__init__(self, provider_obj) self.minTime = 20
class GenericProvider(object): NZB = "nzb" TORRENT = "torrent" types = [NZB, TORRENT] type = None def __init__(self, name): # these need to be set in the subclass self.name = name self.urls = {} self.url = "" self.public = False self.show = None self.supportsBacklog = False self.supportsAbsoluteNumbering = False self.anime_only = False self.search_mode = None self.search_fallback = False self.enabled = False self.enable_daily = False self.enable_backlog = False self.cache = TVCache(self) self.session = None self.proper_strings = ["PROPER|REPACK|REAL"] self.headers = {} self.session = None self.btCacheURLS = [ "http://torcache.net/torrent/{torrent_hash}.torrent", "http://thetorrent.org/torrent/{torrent_hash}.torrent", "http://btdig.com/torrent/{torrent_hash}.torrent", # 'http://torrage.com/torrent/{torrent_hash}.torrent', # 'http://itorrents.org/torrent/{torrent_hash}.torrent', ] @property def id(self): return self._makeID() @property def isActive(self): return False @property def isEnabled(self): return self.enabled @property def imageName(self): return self.id + ".png" def _makeID(self): return str(re.sub(r"[^\w\d_]", "_", self.name.strip().lower())) def _checkAuth(self): return True def _doLogin(self): return True @classmethod def get_subclasses(cls): yield cls if cls.__subclasses__(): for sub in cls.__subclasses__(): for s in sub.get_subclasses(): yield s def getResult(self, episodes): """ Returns a result of the correct type for this provider """ try: result = {"nzb": NZBSearchResult, "torrent": TorrentSearchResult}[self.type](episodes) except: result = SearchResult(episodes) result.provider = self return result def getURL(self, url, post_data=None, params=None, timeout=30, json=False, needBytes=False): """ By default this is just a simple urlopen call but this method should be overridden for providers with special URL requirements (like cookies) """ return getURL( url, post_data=post_data, params=params, headers=self.headers, timeout=timeout, session=self.session, json=json, needBytes=needBytes, ) def _makeURL(self, result): urls = [] filename = "" if result.url.startswith("magnet"): try: torrent_hash = re.findall(r"urn:btih:([\w]{32,40})", result.url)[0].upper() try: torrent_name = re.findall("dn=([^&]+)", result.url)[0] except Exception: torrent_name = "NO_DOWNLOAD_NAME" if len(torrent_hash) == 32: torrent_hash = b16encode(b32decode(torrent_hash)).upper() if not torrent_hash: sickrage.srLogger.error("Unable to extract torrent hash from magnet: " + result.url) return urls, filename urls = random.shuffle( [x.format(torrent_hash=torrent_hash, torrent_name=torrent_name) for x in self.btCacheURLS] ) except Exception: sickrage.srLogger.error("Unable to extract torrent hash or name from magnet: " + result.url) return urls, filename else: urls = [result.url] if self.type == self.TORRENT: filename = os.path.join(sickrage.srConfig.TORRENT_DIR, sanitizeFileName(result.name) + "." + self.type) elif self.type == self.NZB: filename = os.path.join(sickrage.srConfig.NZB_DIR, sanitizeFileName(result.name) + "." + self.type) return urls, filename def downloadResult(self, result): """ Save the result to disk. """ # check for auth if not self._doLogin: return False urls, filename = self._makeURL(result) for url in urls: if "NO_DOWNLOAD_NAME" in url: continue if url.startswith("http"): self.headers.update({"Referer": "/".join(url.split("/")[:3]) + "/"}) sickrage.srLogger.info("Downloading a result from " + self.name + " at " + url) # Support for Jackett/TorzNab if url.endswith(GenericProvider.TORRENT) and filename.endswith(GenericProvider.NZB): filename = filename.rsplit(".", 1)[0] + "." + GenericProvider.TORRENT if download_file(url, filename, session=self.session, headers=self.headers): if self._verify_download(filename): sickrage.srLogger.info("Saved result to " + filename) return True else: sickrage.srLogger.warning("Could not download %s" % url) remove_file_failed(filename) if len(urls): sickrage.srLogger.warning("Failed to download any results") return False def _verify_download(self, file_name=None): """ Checks the saved file to see if it was actually valid, if not then consider the download a failure. """ # primitive verification of torrents, just make sure we didn't get a text file or something if file_name.endswith(GenericProvider.TORRENT): try: for byte in readFileBuffered(file_name): mime_type = guessParser(StringInputStream(byte))._getMimeType() if mime_type == "application/x-bittorrent": # clean up del mime_type return True except Exception as e: sickrage.srLogger.debug("Failed to validate torrent file: {}".format(e.message)) sickrage.srLogger.debug("Result is not a valid torrent file") return False return True def searchRSS(self, episodes): return self.cache.findNeededEpisodes(episodes) def getQuality(self, item, anime=False): """ Figures out the quality of the given RSS item node item: An elementtree.ElementTree element representing the <item> tag of the RSS feed Returns a Quality value obtained from the node's data """ (title, url) = self._get_title_and_url(item) quality = Quality.sceneQuality(title, anime) return quality def _doSearch(self, search_params, search_mode="eponly", epcount=0, age=0, epObj=None): return [] def _get_season_search_strings(self, episode): return [{}] def _get_episode_search_strings(self, eb_obj, add_string=""): return [{}] def _get_title_and_url(self, item): """ Retrieves the title and URL data from the item XML node item: An elementtree.ElementTree element representing the <item> tag of the RSS feed Returns: A tuple containing two strings representing title and URL respectively """ title = item.get("title", "") if title: title = "" + title.replace(" ", ".") url = item.get("link", "") if url: url = url.replace("&", "&").replace("%26tr%3D", "&tr=") return title, url def _get_size(self, item): """Gets the size from the item""" sickrage.srLogger.error("Provider type doesn't have _get_size() implemented yet") return -1 def findSearchResults(self, show, episodes, search_mode, manualSearch=False, downCurQuality=False): if not self._checkAuth: return self.show = show results = {} itemList = [] searched_scene_season = None for epObj in episodes: # search cache for episode result cacheResult = self.cache.searchCache(epObj, manualSearch, downCurQuality) if cacheResult: if epObj.episode not in results: results[epObj.episode] = cacheResult else: results[epObj.episode].extend(cacheResult) # found result, search next episode continue # skip if season already searched if len(episodes) > 1 and search_mode == "sponly" and searched_scene_season == epObj.scene_season: continue # mark season searched for season pack searches so we can skip later on searched_scene_season = epObj.scene_season search_strings = [] if len(episodes) > 1 and search_mode == "sponly": # get season search results search_strings = self._get_season_search_strings(epObj) elif search_mode == "eponly": # get single episode search results search_strings = self._get_episode_search_strings(epObj) first = search_strings and isinstance(search_strings[0], dict) and "rid" in search_strings[0] if first: sickrage.srLogger.debug("First search_string has rid") for curString in search_strings: itemList += self._doSearch(curString, search_mode, len(episodes), epObj=epObj) if first: first = False if itemList: sickrage.srLogger.debug( "First search_string had rid, and returned results, skipping query by string" ) break else: sickrage.srLogger.debug( "First search_string had rid, but returned no results, searching with string query" ) # if we found what we needed already from cache then return results and exit if len(results) == len(episodes): return results # sort list by quality if len(itemList): items = {} itemsUnknown = [] for item in itemList: quality = self.getQuality(item, anime=show.is_anime) if quality == Quality.UNKNOWN: itemsUnknown += [item] else: if quality not in items: items[quality] = [item] else: items[quality].append(item) itemList = list(itertools.chain(*[v for (k, v) in sorted(items.iteritems(), reverse=True)])) itemList += itemsUnknown if itemsUnknown else [] # filter results cl = [] for item in itemList: (title, url) = self._get_title_and_url(item) # parse the file name try: myParser = NameParser(False) parse_result = myParser.parse(title) except InvalidNameException: sickrage.srLogger.debug("Unable to parse the filename " + title + " into a valid episode") continue except InvalidShowException: sickrage.srLogger.debug("Unable to parse the filename " + title + " into a valid show") continue showObj = parse_result.show quality = parse_result.quality release_group = parse_result.release_group version = parse_result.version addCacheEntry = False if not (showObj.air_by_date or showObj.sports): if search_mode == "sponly": if len(parse_result.episode_numbers): sickrage.srLogger.debug( "This is supposed to be a season pack search but the result " + title + " is not a valid season pack, skipping it" ) addCacheEntry = True if len(parse_result.episode_numbers) and ( parse_result.season_number not in set([ep.season for ep in episodes]) or not [ep for ep in episodes if ep.scene_episode in parse_result.episode_numbers] ): sickrage.srLogger.debug( "The result " + title + " doesn't seem to be a valid episode that we are trying to snatch, ignoring" ) addCacheEntry = True else: if ( not len(parse_result.episode_numbers) and parse_result.season_number and not [ ep for ep in episodes if ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers ] ): sickrage.srLogger.debug( "The result " + title + " doesn't seem to be a valid season that we are trying to snatch, ignoring" ) addCacheEntry = True elif len(parse_result.episode_numbers) and not [ ep for ep in episodes if ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers ]: sickrage.srLogger.debug( "The result " + title + " doesn't seem to be a valid episode that we are trying to snatch, ignoring" ) addCacheEntry = True if not addCacheEntry: # we just use the existing info for normal searches actual_season = parse_result.season_number actual_episodes = parse_result.episode_numbers else: if not parse_result.is_air_by_date: sickrage.srLogger.debug( "This is supposed to be a date search but the result " + title + " didn't parse as one, skipping it" ) addCacheEntry = True else: airdate = parse_result.air_date.toordinal() sql_results = main_db.MainDB().select( "SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?", [showObj.indexerid, airdate], ) if len(sql_results) != 1: sickrage.srLogger.warning( "Tried to look up the date for the episode " + title + " but the database didn't give proper results, skipping it" ) addCacheEntry = True if not addCacheEntry: actual_season = int(sql_results[0][b"season"]) actual_episodes = [int(sql_results[0][b"episode"])] # add parsed result to cache for usage later on if addCacheEntry: sickrage.srLogger.debug("Adding item from search to cache: " + title) # pylint: disable=W0212 # Access to a protected member of a client class ci = self.cache._addCacheEntry(title, url, parse_result=parse_result) if ci is not None: cl.append(ci) continue # make sure we want the episode wantEp = True for epNo in actual_episodes: if not showObj.wantEpisode(actual_season, epNo, quality, manualSearch, downCurQuality): wantEp = False break if not wantEp: sickrage.srLogger.info( "Ignoring result " + title + " because we don't want an episode that is " + Quality.qualityStrings[quality] ) continue sickrage.srLogger.debug("Found result " + title + " at " + url) # make a result object epObj = [] for curEp in actual_episodes: epObj.append(showObj.getEpisode(actual_season, curEp)) result = self.getResult(epObj) result.show = showObj result.url = url result.name = title result.quality = quality result.release_group = release_group result.version = version result.content = None result.size = self._get_size(item) if len(epObj) == 1: epNum = epObj[0].episode sickrage.srLogger.debug("Single episode result.") elif len(epObj) > 1: epNum = MULTI_EP_RESULT sickrage.srLogger.debug( "Separating multi-episode result to check for later - result contains episodes: " + str(parse_result.episode_numbers) ) elif len(epObj) == 0: epNum = SEASON_RESULT sickrage.srLogger.debug("Separating full season result to check for later") if epNum not in results: results[epNum] = [result] else: results[epNum].append(result) # check if we have items to add to cache if len(cl) > 0: self.cache._getDB().mass_action(cl) del cl # cleanup return results def findPropers(self, search_date=None): results = self.cache.listPropers(search_date) return [Proper(x[b"name"], x[b"url"], datetime.fromtimestamp(x[b"time"]), self.show) for x in results] def seedRatio(self): """ Provider should override this value if custom seed ratio enabled It should return the value of the provider seed ratio """ return "" @classmethod def getDefaultProviders(cls): return "" @classmethod def getProvider(cls, name): providerMatch = [x for x in cls.getProviderList() if x.name == name] if len(providerMatch) == 1: return providerMatch[0] @classmethod def getProviderByID(cls, id): providerMatch = [x for x in cls.getProviderList() if x.id == id] if len(providerMatch) == 1: return providerMatch[0] @classmethod def getProviderList(cls, data=None): modules = [] for type in GenericProvider.types: modules += cls.loadProviders(type) return modules @classmethod def loadProviders(cls, type): providers = [] pregex = re.compile("^([^_]*?)\.py$", re.IGNORECASE) path = os.path.join(os.path.dirname(__file__), type) names = [pregex.match(m) for m in os.listdir(path)] providers += [cls.loadProvider(name.group(1), type) for name in names if name] return providers @classmethod def loadProvider(cls, name, type, *args, **kwargs): import inspect, sys sys.path.append(os.path.join(os.path.dirname(__file__), "sickrage")) members = dict( inspect.getmembers( importlib.import_module(".{}.{}".format(type, name), "providers"), lambda x: hasattr(x, "type") and x not in [NZBProvider, TorrentProvider], ) ) return [v for v in members.values() if hasattr(v, "type") and v.type == type][0](*args, **kwargs)
def __init__(self, provider_obj): TVCache.__init__(self, provider_obj) self.minTime = 15