def __parseSearchResult(self, data, target_url): results = [] with BS4Parser(data, "html5lib") as soup: lists = soup.find_all("ul") list_items = [] for ul_list in lists: curr_list_item = ul_list("li") if ul_list else [] list_items.extend(curr_list_item) # Continue only if one Release is found if len(list_items) < 1: logger.debug( "Data returned from provider does not contain any torrents" ) return [] for list_item in list_items: title = "{0}{1}".format( str(list_item.find("span").next_sibling), str(list_item.find("strong").text)) logger.debug("Found title {0}".format(title)) episode_url = "/#".join( list_item.find("a")["href"].rsplit("#", 1)) episode = episode_url.split("#", 1)[1] page_url = "{0}{1}".format(self.url, episode_url) show_id = self.__getShowId(page_url) if not show_id: logger.debug("Could not find show ID") continue fetch_params = { "method": "getshows", "type": "show", "mode": "filter", "showid": show_id, "value": episode } entries = self.__fetchUrls(target_url, fetch_params, title) results.extend(entries) return results
def __parseSearchResult(self, data, target_url): results = [] with BS4Parser(data, 'html5lib') as soup: lists = soup.find_all('ul') list_items = [] for ul_list in lists: curr_list_item = ul_list('li') if ul_list else [] list_items.extend(curr_list_item) # Continue only if one Release is found if len(list_items) < 1: logger.debug( 'Data returned from provider does not contain any torrents' ) return [] for list_item in list_items: title = '{0}{1}'.format( str(list_item.find('span').next_sibling), str(list_item.find('strong').text)) logger.debug('Found title {0}'.format(title)) episode_url = '/#'.join( list_item.find('a')['href'].rsplit('#', 1)) episode = episode_url.split('#', 1)[1] page_url = '{0}{1}'.format(self.url, episode_url) show_id = self.__getShowId(page_url) if not show_id: logger.debug('Could not find show ID') continue fetch_params = { 'method': 'getshows', 'type': 'show', 'mode': 'filter', 'showid': show_id, 'value': episode } entries = self.__fetchUrls(target_url, fetch_params, title) results.extend(entries) return results
def addShowToTraktWatchList(self): if settings.TRAKT_SYNC_WATCHLIST and settings.USE_TRAKT: logger.debug( "SHOW_WATCHLIST::ADD::START - Look for Shows to Add to Trakt Watchlist" ) if settings.showList is not None: trakt_data = [] for show in settings.showList: if not self._checkInList(show.idxr.slug, str(show.indexerid), "0", "0", List="Show"): logger.debug( "Adding Show: Indexer {0} {1} - {2} to Watchlist". format(show.idxr.name, str(show.indexerid), show.name)) show_el = { "title": show.name, "year": show.startyear, "ids": { show.idxr.slug: show.indexerid } } trakt_data.append(show_el) if trakt_data: try: data = {"shows": trakt_data} self.trakt_api.traktRequest("sync/watchlist", data, method="POST") self._getShowWatchlist() except traktException as e: logger.warning( "Could not connect to Trakt service. Error: {0}". format(str(e))) logger.debug( "SHOW_WATCHLIST::ADD::FINISH - Look for Shows to Add to Trakt Watchlist" )
def _send_join_msg(title, msg, id=None, apikey=None): """ Sends a Join notification :param title: The title of the notification to send :param msg: The message string to send :param id: The Device ID :param id: The User's API Key :returns: True if the message succeeded, False otherwise """ id = settings.JOIN_ID if id is None else id apikey = settings.JOIN_APIKEY if apikey is None else apikey logger.debug("Join in use with device ID: {0}".format(id)) message = "{0} : {1}".format(title.encode(), msg.encode()) params = { "apikey": apikey, "deviceId": id, "title": title, "text": message, "icon": "https://raw.githubusercontent.com/SickChill/SickChill/master/sickchill/gui/slick/images/sickchill.png", } payload = urllib.parse.urlencode(params) join_api = "https://joinjoaomgcd.appspot.com/_ah/api/messaging/v1/sendPush?" + payload logger.debug("Join url in use : {0}".format(join_api)) success = False try: urllib.request.urlopen(join_api) message = "Join message sent successfully." logger.debug("Join message returned : {0}".format(message)) success = True except Exception as e: message = "Error while sending Join message: {0} ".format(e) finally: logger.info(message) return success, message
def refresh_show(self, show, force=False): if self.is_being_refreshed(show) and not force: raise CantRefreshShowException('This show is already being refreshed, not refreshing again.') if self.is_being_updated(show) or self.is_in_update_queue(show): logger.debug( 'A refresh was attempted but there is already an update queued or in progress. Updates do a refresh at the end so I\'m skipping this request.') return if show.paused and not force: logger.debug('Skipping show [{0}] because it is paused.'.format(show.name)) return logger.debug('Queueing show refresh for {0}'.format(show.name)) queue_item_obj = QueueItemRefresh(show, force=force) self.add_item(queue_item_obj) return queue_item_obj
def run(self): """ Runs the task :return: None """ super().run() # noinspection PyBroadException try: logger.debug("Task for {} started".format(self.action_id)) self.last_result = self._send_discord() logger.debug("Task for {} completed".format(self.action_id)) # give the CPU a break time.sleep(common.cpu_presets[settings.CPU_PRESET]) except Exception: logger.debug(traceback.format_exc()) super().finish() self.finish()
def makeObject(cmd_arg, cur_path): if settings.USE_SYNOINDEX: synoindex_cmd = [ "/usr/syno/bin/synoindex", cmd_arg, os.path.abspath(cur_path) ] logger.debug("Executing command " + str(synoindex_cmd)) logger.debug("Absolute path to command: " + os.path.abspath(synoindex_cmd[0])) try: p = subprocess.Popen(synoindex_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=settings.DATA_DIR, universal_newlines=True) out, err = p.communicate() logger.debug( _("Script result: {0}").format(str(out or err).strip())) except OSError as e: logger.exception("Unable to run synoindex: " + str(e))
def _write_image(image_data, image_path, overwrite=False): """ Saves the data in image_data to the location image_path. Returns True/False to represent success or failure. image_data: binary image data to write to file image_path: file location to save the image to """ # don't bother overwriting it if not overwrite and os.path.isfile(image_path): logger.debug("Image already exists, not downloading") return False image_dir = os.path.dirname(image_path) if not image_data: logger.debug( "Unable to retrieve image to save in {0}, skipping".format( image_path)) return False try: if not os.path.isdir(image_dir): logger.debug("Metadata dir didn't exist, creating it at " + image_dir) os.makedirs(image_dir) helpers.chmodAsParent(image_dir) outFile = open(image_path, 'wb') outFile.write(image_data) outFile.close() helpers.chmodAsParent(image_path) except IOError as e: logger.error("Unable to write image to " + image_path + " - are you sure the show folder is writable? " + str(e)) return False return True
def is_valid(self, result): """ Check if result is valid according to white/blacklist for current show :param result: Result to analyse :return: False if result is not allowed in white/blacklist, True if it is """ if self.whitelist or self.blacklist: if not result.release_group: logger.debug("Failed to detect release group, invalid result") return False if result.release_group.lower() in [ x.lower() for x in self.whitelist ]: white_result = True elif not self.whitelist: white_result = True else: white_result = False if result.release_group.lower() in [ x.lower() for x in self.blacklist ]: black_result = False else: black_result = True logger.debug( "Whitelist check passed: {white}. Blacklist check passed: {black}" .format(white=white_result, black=black_result)) if white_result and black_result: return True else: return False else: logger.debug("No Whitelist and Blacklist defined, check passed.") return True
def removeShowFromSickChill(self): if settings.TRAKT_SYNC_WATCHLIST and settings.USE_TRAKT and settings.TRAKT_REMOVE_SHOW_FROM_SICKCHILL: logger.debug( "SHOW_SICKCHILL::REMOVE::START - Look for Shows to remove from SickChill" ) if settings.showList: for show in settings.showList: if show.status in ("Ended", "Canceled"): if not show.imdb_id: logger.warning( "Could not check trakt progress for {0} because the imdb id is missing from {} data, skipping" .format(show.name, show.idxr.name)) continue try: progress = self.trakt_api.traktRequest( "shows/" + show.imdb_id + "/progress/watched") or {} except traktException as e: logger.warning( "Could not connect to Trakt service. Aborting removing show {0} from SickChill. Error: {1}" .format(show.name, repr(e))) continue if not progress: continue if progress.get("aired", True) == progress.get( "completed", False): settings.showQueueScheduler.action.remove_show( show, full=True) logger.debug( "Show: {0} has been removed from SickChill". format(show.name)) logger.debug( "SHOW_SICKCHILL::REMOVE::FINISH - Trakt Show Watchlist")
def wrapper(*args, **kwargs): try: result = target(*args, **kwargs) except self.catch as e: if self.image_api and not kwargs.get('lang'): if args[1].lang != 'en': logger.debug( "Could not find the image on the indexer, re-trying to find it in english" ) kwargs['lang'] = 'en' return wrapper(*args, **kwargs) logger.debug( "Could not find item on the indexer: (Indexer probably doesn't have this item) [{error}]" .format(error=str(e))) result = self.default_return except RHTTPError as e: logger.debug( "Could not find item on the indexer: (Indexer probably doesn't have this item) [{error}]" .format(error=str(e))) result = self.default_return return result
def save_thumbnail(self, ep_obj): """ Retrieves a thumbnail and saves it to the correct spot. This method should not need to be overridden by implementing classes, changing get_episode_thumb_path and _get_episode_thumb_url should suffice. ep_obj: a TVEpisode object for which to generate a thumbnail """ thumb_url = sickchill.indexer.episode_image_url(ep_obj) if not thumb_url: logger.debug( "No thumb is available for this episode, not creating a thumb") return False file_path = self.get_episode_thumb_path(ep_obj) if not file_path: logger.debug( "Unable to find a file path to use for this thumbnail, not generating it" ) return False thumb_data = metadata_helpers.getShowImage(thumb_url) if not thumb_data: logger.debug( "No thumb is available for this episode, not creating a thumb") return False result = self._write_image(thumb_data, file_path) if not result: return False for cur_ep in [ep_obj] + ep_obj.relatedEps: cur_ep.hastbn = True return True
def search(self, search_strings, age=0, ep_obj=None): results = [] if not self.login(): return results # Search Params search_params = { 'user': self.username, 'passkey': self.passkey, 'search': '.', # Dummy query for RSS search, needs the search param sent. 'latest': 'true' } # Units units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in search_strings[mode]: if mode != 'RSS': logger.debug("Search string: {0}".format(search_string)) search_params['latest'] = 'false' search_params['search'] = search_string data = self.get_url(self.urls['search'], params=search_params, returns='text') if not data: logger.debug("No data returned from provider") continue result = json.loads(data) if 'results' in result: for torrent in result['results']: title = torrent['release_name'] download_url = torrent['download_url'] seeders = torrent['seeders'] leechers = torrent['leechers'] if seeders < self.minseed or leechers < self.minleech: logger.info( "Discarded {0} because with {1}/{2} seeders/leechers does not meet the requirement of {3}/{4} seeders/leechers" .format(title, seeders, leechers, self.minseed, self.minleech)) continue freeleech = torrent['freeleech'] if self.freeleech and not freeleech: continue size = torrent['size'] size = convert_size(size, units=units) or -1 item = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': '' } logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) if 'error' in result: logger.warning(result['error']) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def find_needed_episodes(self, episode, manualSearch=False, downCurQuality=False): needed_eps = {} cl = [] cache_db_con = self._get_db() if not episode: sql_results = cache_db_con.select( "SELECT * FROM results WHERE provider = ?", [self.provider_id]) elif not isinstance(episode, list): sql_results = cache_db_con.select( "SELECT * FROM results WHERE provider = ? AND indexerid = ? AND season = ? AND episodes LIKE ?", [ self.provider_id, episode.show.indexerid, episode.season, "%|" + str(episode.episode) + "|%" ]) else: for ep_obj in episode: cl.append([ "SELECT * FROM results WHERE provider = ? AND indexerid = ? AND season = ? AND episodes LIKE ? AND quality IN (" + ",".join([str(x) for x in ep_obj.wantedQuality]) + ")", [ self.provider_id, ep_obj.show.indexerid, ep_obj.season, "%|" + str(ep_obj.episode) + "|%" ] ]) sql_results = cache_db_con.mass_action(cl, fetchall=True) sql_results = list(itertools.chain(*sql_results)) # for each cache entry for cur_result in sql_results: # get the show object, or if it's not one of our shows then ignore it show_obj = Show.find(settings.showList, int(cur_result["indexerid"])) if not show_obj: continue # ignored/required words, and non-tv junk if not show_name_helpers.filter_bad_releases(cur_result["name"], show=show_obj): continue # skip if provider is anime only and show is not anime if self.provider.anime_only and not show_obj.is_anime: logger.debug("" + str(show_obj.name) + " is not an anime, skiping") continue # get season and ep data (ignoring multi-eps for now) cur_season = int(cur_result["season"]) if cur_season == -1: continue cur_ep = cur_result["episodes"].split("|")[1] if not cur_ep: continue cur_ep = int(cur_ep) cur_quality = int(cur_result["quality"]) cur_release_group = cur_result["release_group"] cur_version = cur_result["version"] # if the show says we want that episode then add it to the list if not show_obj.wantEpisode(cur_season, cur_ep, cur_quality, manualSearch, downCurQuality): logger.debug("Ignoring " + cur_result["name"]) continue ep_obj = show_obj.getEpisode(cur_season, cur_ep) # build a result object title = cur_result["name"] url = cur_result["url"] logger.info("Found result " + title + " at " + url) result = self.provider.get_result([ep_obj]) result.show = show_obj result.url = url result.name = title result.quality = cur_quality result.release_group = cur_release_group result.version = cur_version result.content = None # add it to the list if ep_obj not in needed_eps: needed_eps[ep_obj] = [result] else: needed_eps[ep_obj].append(result) # datetime stamp this search so cache gets cleared self.set_last_search() return needed_eps
def _add_cache_entry(self, name, url, size, seeders, leechers, parse_result=None, indexer_id=0): # check if we passed in a parsed result or should we try and create one if not parse_result: # create show_obj from indexer_id if available show_obj = None if indexer_id: show_obj = Show.find(settings.showList, indexer_id) try: parse_result = NameParser(showObj=show_obj).parse(name) except (InvalidNameException, InvalidShowException) as error: logger.debug("{0}".format(error)) return None if not parse_result or not parse_result.series_name: return None # if we made it this far then lets add the parsed result to cache for usage later on season = parse_result.season_number if parse_result.season_number else 1 episodes = parse_result.episode_numbers if season and episodes: # store episodes as a separated string episode_text = "|" + "|".join( {str(episode) for episode in sorted(episodes) if episode}) + "|" # get the current timestamp cur_timestamp = int( time.mktime(datetime.datetime.today().timetuple())) # get quality of release quality = parse_result.quality # get release group release_group = parse_result.release_group # get version version = parse_result.version logger.debug( _("Added RSS item: [{}] to cache: {}").format( name, self.provider_id)) return ({ 'provider': self.provider_id, 'name': name, 'season': season, 'episodes': episode_text, 'indexerid': parse_result.show.indexerid, 'url': url, 'time': cur_timestamp, 'quality': quality, 'release_group': release_group, 'version': version, 'seeders': seeders, 'leechers': leechers, 'size': size }, { 'url': url })
def search(self, search_strings, age=0, ep_obj=None): results = [] for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in search_strings[mode]: if mode == 'Season': search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string) if mode != 'RSS': logger.debug("Search string: {0}".format(search_string)) search_url = self.url + '/recherche/' + search_string.replace( '.', '-').replace(' ', '-') + '.html,trie-seeds-d' else: search_url = self.url + '/view_cat.php?categorie=series&trie=date-d' data = self.get_url(search_url, returns='text') if not data: continue with BS4Parser(data, 'html5lib') as html: torrent_rows = html(class_=re.compile('ligne[01]')) for result in torrent_rows: try: title = result.find(class_="titre").get_text( strip=True).replace("HDTV", "HDTV x264-CPasBien") title = re.sub(r' Saison', ' Season', title, flags=re.I) tmp = result.find("a")['href'].split( '/')[-1].replace('.html', '.torrent').strip() download_url = (self.url + '/telechargement/{0}'.format(tmp)) if not all([title, download_url]): continue seeders = try_int( result.find(class_="up").get_text(strip=True)) leechers = try_int( result.find(class_="down").get_text( strip=True)) if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.debug( "Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})" .format(title, seeders, leechers)) continue torrent_size = result.find(class_="poid").get_text( strip=True) units = ['o', 'Ko', 'Mo', 'Go', 'To', 'Po'] size = convert_size(torrent_size, units=units) or -1 item = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': '' } if mode != 'RSS': logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) except Exception: continue # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): results = [] if self.show and not self.show.is_anime: return results for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in {*search_strings[mode]}: if mode != "RSS": logger.debug( _("Search String: {search_string}".format( search_string=search_string))) search_params = { "terms": search_string, "type": 1, # get anime types } data = self.get_url(self.urls["search"], params=search_params, returns="text") if not data: continue with BS4Parser(data, "html5lib") as soup: torrent_table = soup.find("table", class_="listing") torrent_rows = torrent_table("tr") if torrent_table else [] # Continue only if one Release is found if len(torrent_rows) < 2: logger.debug( "Data returned from provider does not contain any torrents" ) continue a = 1 if len(torrent_rows[0]("td")) < 2 else 0 for top, bot in zip(torrent_rows[a::2], torrent_rows[a + 1::2]): try: desc_top = top.find("td", class_="desc-top") title = desc_top.get_text(strip=True) download_url = desc_top.find("a")["href"] desc_bottom = bot.find( "td", class_="desc-bot").get_text(strip=True) size = convert_size( desc_bottom.split("|")[1].strip( "Size: ")) or -1 stats = bot.find( "td", class_="stats").get_text(strip=True) sl = re.match( r"S:(?P<seeders>\d+)L:(?P<leechers>\d+)C:(?:\d+)ID:(?:\d+)", stats.replace(" ", "")) seeders = try_int(sl.group("seeders")) if sl else 0 leechers = try_int( sl.group("leechers")) if sl else 0 except Exception: continue if not all([title, download_url]): continue # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != "RSS": logger.debug( "Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})" .format(title, seeders, leechers)) continue item = { "title": title, "link": download_url, "size": size, "seeders": seeders, "leechers": leechers, "hash": "" } if mode != "RSS": logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get("seeders", 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): self.login() results = [] for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in search_strings[mode]: if mode != 'RSS': logger.debug( _("Search String: {search_string}".format( search_string=search_string))) # search string needs to be normalized, single quotes are apparently not allowed on the site # ç should also be replaced, people tend to use c instead replace_chars = {"'": '', "ç": 'c'} for k, v in replace_chars.items(): search_string = search_string.replace(k, v) logger.debug('Sanitized string: {0}'.format(search_string)) try: search_params = { 'category': '2145', 'sub_category': 'all', 'name': re.sub(r'[()]', '', search_string), 'do': 'search' } data = self.get_url(self.urls['search'], params=search_params, returns='text') if not data: continue if 'logout' not in data: logger.debug('Refreshing cookies') self.login() with BS4Parser(data, 'html5lib') as html: torrent_table = html.find(class_='table') torrent_rows = torrent_table( 'tr') if torrent_table else [] # Continue only if at least one Release is found if len(torrent_rows) < 2: logger.debug( 'Data returned from provider does not contain any torrents' ) continue # Skip column headers for result in torrent_rows[1:]: cells = result('td') if len(cells) < 9: continue title = cells[1].find('a').get_text(strip=True) id = cells[2].find('a')['target'] download_url = urljoin( self.url, 'engine/download_torrent?id=' + id) if not (title and download_url): continue seeders = try_int(cells[7].get_text(strip=True)) leechers = try_int(cells[8].get_text(strip=True)) torrent_size = cells[5].get_text() size = convert_size(torrent_size) or -1 # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.debug( 'Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})' .format(title, seeders, leechers)) continue item = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': '' } if mode != 'RSS': logger.debug( _('Found result: {title} with {seeders} seeders and {leechers} leechers' .format(title=title, seeders=seeders, leechers=leechers))) items.append(item) except (AttributeError, TypeError, KeyError, ValueError): logger.exception('Failed parsing provider {}.'.format( self.name)) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): results = [] lang_info = '' if not ep_obj or not ep_obj.show else ep_obj.show.lang """ Search query: https://www.elitetorrent.eu/torrents.php?cat=4&modo=listado&orden=fecha&pag=1&buscar=fringe cat = 4 => Shows modo = listado => display results mode orden = fecha => order buscar => Search show pag = 1 => page number """ search_params = { 'cat': 4, 'modo': 'listado', 'orden': 'fecha', 'pag': 1, 'buscar': '' } for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) # Only search if user conditions are true if self.onlyspasearch and lang_info != 'es' and mode != 'RSS': logger.debug( "Show info is not spanish, skipping provider search") continue for search_string in search_strings[mode]: if mode != 'RSS': logger.debug("Search string: {0}".format(search_string)) search_string = re.sub(r'S0*(\d*)E(\d*)', r'\1x\2', search_string) search_params['buscar'] = search_string.strip( ) if mode != 'RSS' else '' time.sleep(cpu_presets[settings.CPU_PRESET]) data = self.get_url(self.urls['search'], params=search_params, returns='text') if not data: continue try: with BS4Parser(data, 'html5lib') as html: torrent_table = html.find('table', class_='fichas-listado') torrent_rows = torrent_table( 'tr') if torrent_table else [] if len(torrent_rows) < 2: logger.debug( "Data returned from provider does not contain any torrents" ) continue for row in torrent_rows[1:]: try: download_url = self.urls[ 'base_url'] + row.find('a')['href'] """ Transform from https://elitetorrent.eu/torrent/40142/la-zona-1x02 to https://elitetorrent.eu/get-torrent/40142 """ download_url = re.sub(r'/torrent/(\d*)/.*', r'/get-torrent/\1', download_url) """ Trick for accents for this provider. - data = self.get_url(self.urls['search'], params=search_params, returns='text') - returns latin1 coded text and this makes that the title used for the search and the title retrieved from the parsed web page doesn't match so I get "No needed episodes found during backlog search for: XXXX" This is not the best solution but it works. First encode latin1 and then decode utf8 to remains str """ row_title = row.find('a', class_='nombre')['title'] title = self._processTitle( row_title.encode('latin-1').decode('utf8')) seeders = try_int( row.find('td', class_='semillas').get_text( strip=True)) leechers = try_int( row.find('td', class_='clientes').get_text( strip=True)) # seeders are not well reported. Set 1 in case of 0 seeders = max(1, seeders) # Provider does not provide size size = -1 except (AttributeError, TypeError, KeyError, ValueError): continue if not all([title, download_url]): continue # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.debug( "Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})" .format(title, seeders, leechers)) continue item = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': '' } if mode != 'RSS': logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) except Exception: logger.warning( "Failed parsing provider. Traceback: {0}".format( traceback.format_exc())) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): """ Searches indexer using the params in search_strings, either for latest releases, or a string/id search Returns: list of results in dict form """ results = [] if not self._check_auth(): return results if "gingadaddy" not in self.url: # gingadaddy has no caps. if not self.caps: self.get_newznab_categories(just_caps=True) if not self.caps: return results for mode in search_strings: search_params = { "t": ("search", "tvsearch")[bool(self.use_tv_search)], "limit": 100, "offset": 0, "cat": self.catIDs.strip(", ") or "5030,5040", "maxage": settings.USENET_RETENTION, } if self.needs_auth and self.key: search_params["apikey"] = self.key if mode != "RSS": if self.use_tv_search: if "tvdbid" in str(self.cap_tv_search): search_params["tvdbid"] = ep_obj.show.indexerid if ep_obj.show.air_by_date or ep_obj.show.sports: # date_str = str(ep_obj.airdate) # search_params['season'] = date_str.partition('-')[0] # search_params['ep'] = date_str.partition('-')[2].replace('-', '/') search_params["q"] = str(ep_obj.airdate) elif ep_obj.show.is_anime: search_params["ep"] = ep_obj.absolute_number else: search_params["season"] = ep_obj.scene_season search_params["ep"] = ep_obj.scene_episode if mode == "Season": search_params.pop("ep", "") if self.torznab: search_params.pop("ep", "") search_params.pop("season", "") items = [] logger.debug("Search Mode: {0}".format(mode)) for search_string in {*search_strings[mode]}: if mode != "RSS": logger.debug( _("Search String: {search_string}".format( search_string=search_string))) if "tvdbid" not in search_params: search_params["q"] = search_string time.sleep(cpu_presets[settings.CPU_PRESET]) data = self.get_url(urljoin(self.url, "api"), params=search_params, returns="text") if not data: break with BS4Parser(data, "html5lib") as html: if not self._check_auth_from_data(html): break # try: # self.torznab = 'xmlns:torznab' in html.rss.attrs # except AttributeError: # self.torznab = False for item in html("item"): try: title = item.title.get_text(strip=True) download_url = None if item.link: if validators.url( item.link.get_text(strip=True)): download_url = item.link.get_text( strip=True) elif validators.url(item.link.next.strip()): download_url = item.link.next.strip() if (not download_url, item.enclosure and validators.url( item.enclosure.get("url", "").strip())): download_url = item.enclosure.get("url", "").strip() if not (title and download_url): continue seeders = leechers = None if "gingadaddy" in self.url: size_regex = re.search(r"\d*.?\d* [KMGT]B", str(item.description)) item_size = size_regex.group( ) if size_regex else -1 else: item_size = item.size.get_text( strip=True) if item.size else -1 for attr in item.find_all( ["newznab:attr", "torznab:attr"]): item_size = attr["value"] if attr[ "name"] == "size" else item_size seeders = try_int( attr["value"] ) if attr["name"] == "seeders" else seeders leechers = try_int( attr["value"] ) if attr["name"] == "peers" else leechers if not item_size or (self.torznab and (seeders is None or leechers is None)): continue size = convert_size(item_size) or -1 result = { "title": title, "link": download_url, "size": size, "seeders": seeders, "leechers": leechers } items.append(result) except Exception: continue # Since we aren't using the search string, # break out of the search string loop if "tvdbid" in search_params: break if self.torznab: results.sort(key=lambda d: try_int(d.get("seeders", 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): results = [] if not self.login(): return results for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in {*search_strings[mode]}: if mode != 'RSS': logger.debug( _("Search String: {search_string}".format( search_string=search_string))) self.search_params['searchstr'] = search_string data = self.get_url(self.urls['search'], params=self.search_params, returns='text') if not data: logger.debug('URL did not return data') continue strTableStart = "<table class=\"torrent_table" startTableIndex = data.find(strTableStart) trimmedData = data[startTableIndex:] if not trimmedData: continue try: with BS4Parser(trimmedData, 'html5lib') as html: result_table = html.find('table', {'id': 'torrent_table'}) if not result_table: logger.debug( "Data returned from provider does not contain any torrents" ) continue result_tbody = result_table.find('tbody') entries = result_tbody.contents del entries[1::2] for result in entries[1:]: torrent = result('td') if len(torrent) <= 1: break allAs = (torrent[1])('a') try: notinternal = result.find( 'img', src='/static//common/user_upload.png') if self.ranked and notinternal: logger.debug( "Found a user uploaded release, Ignoring it.." ) continue freeleech = result.find( 'img', src='/static//common/browse/freeleech.png') if self.freeleech and not freeleech: continue title = allAs[2].string download_url = self.urls['base_url'] + allAs[ 0].attrs['href'] torrent_size = result.find( "td", class_="nobr").find_next_sibling( "td").string if torrent_size: size = convert_size(torrent_size) or -1 seeders = try_int( (result('td')[6]).text.replace(',', '')) leechers = try_int( (result('td')[7]).text.replace(',', '')) except (AttributeError, TypeError): continue if not title or not download_url: continue # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.debug( "Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})" .format(title, seeders, leechers)) continue item = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': '' } if mode != 'RSS': logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) except Exception: logger.exception( "Failed parsing provider. Traceback: {0}".format( traceback.format_exc())) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def find_search_results(self, show, episodes, search_mode, manual_search=False, download_current_quality=False): self._check_auth() self.show = show results = {} items_list = [] searched_scene_season = None for episode in episodes: cache_result = self.cache.search_cache( episode, manual_search=manual_search, down_cur_quality=download_current_quality) if cache_result: if episode.episode not in results: results[episode.episode] = cache_result else: results[episode.episode].extend(cache_result) continue if len( episodes ) > 1 and search_mode == 'sponly' and searched_scene_season == episode.scene_season: continue search_strings = [] searched_scene_season = episode.scene_season if len(episodes) > 1 and search_mode == 'sponly': search_strings = self.get_season_search_strings(episode) elif search_mode == 'eponly': search_strings = self.get_episode_search_strings(episode) for search_string in search_strings: items_list += self.search(search_string, ep_obj=episode) if len(results) == len(episodes): return results if items_list: items = {} unknown_items = [] for item in items_list: quality = self.get_quality(item, anime=show.is_anime) if quality == Quality.UNKNOWN: unknown_items.append(item) elif quality == Quality.NONE: pass # Skipping an HEVC when HEVC is not allowed by settings else: if quality not in items: items[quality] = [] items[quality].append(item) items_list = list( chain(*[v for (k_, v) in sorted(items.items(), reverse=True)])) items_list += unknown_items cl = [] for item in items_list: title, url = self._get_title_and_url(item) seeders, leechers = self._get_seeders_and_leechers(item) size = self._get_size(item) try: parse_result = NameParser( parse_method=('normal', 'anime')[show.is_anime]).parse(title) except (InvalidNameException, InvalidShowException) as error: logger.debug("{0}".format(error)) continue show_object = parse_result.show quality = parse_result.quality release_group = parse_result.release_group version = parse_result.version add_cache_entry = False if not (show_object.air_by_date or show_object.sports): if search_mode == 'sponly': if parse_result.episode_numbers: logger.debug( 'This is supposed to be a season pack search but the result {0} is not a valid season pack, skipping it' .format(title)) add_cache_entry = True elif not [ ep for ep in episodes if parse_result.season_number == (ep.season, ep.scene_season)[ep.show.is_scene] ]: logger.info( 'This season result {0} is for a season we are not searching for, skipping it' .format(title), logger.DEBUG) add_cache_entry = True else: if not all([ parse_result.season_number is not None, parse_result.episode_numbers, [ ep for ep in episodes if (ep.season, ep.scene_season)[ep.show.is_scene] == (parse_result.season_number, parse_result.scene_season)[ep.show.is_scene] and (ep.episode, ep.scene_episode)[ep.show.is_scene] in parse_result.episode_numbers ] ]) and not all([ # fallback for anime on absolute numbering parse_result.is_anime, parse_result.ab_episode_numbers is not None, [ ep for ep in episodes if ep.show.is_anime and ep. absolute_number in parse_result.ab_episode_numbers ] ]): logger.info( 'The result {0} doesn\'t seem to match an episode that we are currently trying to snatch, skipping it' .format(title)) add_cache_entry = True if not add_cache_entry: actual_season = parse_result.season_number actual_episodes = parse_result.episode_numbers else: same_day_special = False if not parse_result.is_air_by_date: logger.debug( 'This is supposed to be a date search but the result {0} didn\'t parse as one, skipping it' .format(title)) add_cache_entry = True else: air_date = parse_result.air_date.toordinal() db = DBConnection() sql_results = db.select( 'SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?', [show_object.indexerid, air_date]) if len(sql_results) == 2: if int(sql_results[0]['season']) == 0 and int( sql_results[1]['season']) != 0: actual_season = int(sql_results[1]['season']) actual_episodes = [int(sql_results[1]['episode'])] same_day_special = True elif int(sql_results[1]['season']) == 0 and int( sql_results[0]['season']) != 0: actual_season = int(sql_results[0]['season']) actual_episodes = [int(sql_results[0]['episode'])] same_day_special = True elif len(sql_results) != 1: logger.warning( 'Tried to look up the date for the episode {0} but the database didn\'t give proper results, skipping it' .format(title)) add_cache_entry = True if not add_cache_entry and not same_day_special: actual_season = int(sql_results[0]['season']) actual_episodes = [int(sql_results[0]['episode'])] if add_cache_entry: logger.debug( 'Adding item from search to cache: {0}'.format(title)) ci = self.cache._add_cache_entry(title, url, size, seeders, leechers, parse_result=parse_result) if ci is not None: cl.append(ci) continue episode_wanted = True for episode_number in actual_episodes: if not show_object.wantEpisode(actual_season, episode_number, quality, manual_search, download_current_quality): episode_wanted = False break if not episode_wanted: logger.debug(_('Ignoring result ') + f'{title}.') continue logger.debug( _('Found result {title} at {url}'.format(title=title, url=url))) episode_object = [] for current_episode in actual_episodes: episode_object.append( show_object.getEpisode(actual_season, current_episode)) result = self.get_result(episode_object) result.show = show_object result.url = url result.name = title result.quality = quality result.release_group = release_group result.version = version result.content = None result.size = self._get_size(item) if len(episode_object) == 1: episode_number = episode_object[0].episode logger.debug('Single episode result.') elif len(episode_object) > 1: episode_number = MULTI_EP_RESULT logger.debug( 'Separating multi-episode result to check for later - result contains episodes: {0}' .format(parse_result.episode_numbers)) elif len(episode_object) == 0: episode_number = SEASON_RESULT logger.debug( 'Separating full season result to check for later') if episode_number not in results: results[episode_number] = [result] else: results[episode_number].append(result) if cl: # Access to a protected member of a client class cache_db = self.cache._get_db() cache_db.mass_upsert('results', cl) return results
def search(self, search_params, age=0, ep_obj=None): results = [] if not self.login(): return results for mode in search_params: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in search_params[mode]: if mode != 'RSS': logger.debug(_("Search String: {search_string}".format(search_string=search_string))) query = {'sec': 'jax', 'cata': 'yes', 'search': search_string} query.update({"c" + str(i): 1 for i in self.categories}) data = self.get_url(self.urls['apisearch'], returns='text', post_data=query) if not data: continue with BS4Parser(data, 'html5lib') as html: torrent_table = html.find(id='torrenttable') if torrent_table: torrent_rows = torrent_table.findAll('tr') else: torrent_rows = [] # Continue only if one Release is found if len(torrent_rows) < 2: logger.debug("Data returned from provider does not contain any torrents") continue # Scenetime apparently uses different number of cells in #torrenttable based # on who you are. This works around that by extracting labels from the first # <tr> and using their index to find the correct download/seeders/leechers td. labels = [label.get_text(strip=True) or label.img['title'] for label in torrent_rows[0]('td')] for result in torrent_rows[1:]: try: cells = result('td') link = cells[labels.index('Name')].find('a') torrent_id = link['href'].replace('details.php?id=', '').split("&")[0] title = link.get_text(strip=True) download_url = self.urls['download'] % (torrent_id, "{0}.torrent".format(title.replace(" ", "."))) seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True)) leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True)) torrent_size = cells[labels.index('Size')].get_text() size = convert_size(torrent_size) or -1 except (AttributeError, TypeError, KeyError, ValueError): continue if not all([title, download_url]): continue # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.debug("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format (title, seeders, leechers)) continue item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''} if mode != 'RSS': logger.debug("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers)) items.append(item) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): results = [] if not self.login(): return results # TV, Episodes, BoxSets, Episodes HD, Animation, Anime, Cartoons # 2,26,27,32,7,34,35 for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in {*search_strings[mode]}: if mode != "RSS": logger.debug("Search string: {0}".format(search_string)) categories = ["2", "7", "35"] categories += ["26", "32"] if mode == "Episode" else ["27"] if self.show and self.show.is_anime: categories += ["34"] else: categories = ["2", "26", "27", "32", "7", "34", "35"] # Craft the query URL categories_url = 'categories/{categories}/'.format( categories=",".join(categories)) query_url = 'query/{query_string}'.format( query_string=search_string) params_url = urljoin(categories_url, query_url) search_url = urljoin(self.urls['search'], params_url) data = self.get_url(search_url, returns='json') if not data: logger.debug("No data returned from provider") continue # TODO: Handle more than 35 torrents in return. (Max 35 per call) torrent_list = data['torrentList'] if len(torrent_list) < 1: logger.debug( "Data returned from provider does not contain any torrents" ) continue for torrent in torrent_list: try: title = torrent['name'] download_url = urljoin( self.urls['download'], '{id}/{filename}'.format( id=torrent['fid'], filename=torrent['filename'])) seeders = torrent['seeders'] leechers = torrent['leechers'] if seeders < self.minseed or leechers < self.minleech: if mode != "RSS": logger.debug( "Discarding torrent because it doesn't meet the" " minimum seeders or leechers: {0} (S:{1} L:{2})" .format(title, seeders, leechers)) continue size = torrent['size'] item = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': '' } if mode != "RSS": logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) except Exception: continue # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): results = [] if self.show and not self.show.is_anime: return results for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in {*search_strings[mode]}: if mode != 'RSS': logger.debug( _("Search String: {search_string}".format( search_string=search_string))) search_params = { 'page': 'rss', 'c': '1_0', # Category: All anime 's': 'id', # Sort by: 'id'=Date / 'size' / 'name' / 'seeders' / 'leechers' / 'downloads' 'o': 'desc', # Sort direction: asc / desc 'f': ( '0', '2' )[self. confirmed] # Quality filter: 0 = None / 1 = No Remakes / 2 = Trusted Only } if mode != 'RSS': search_params['q'] = search_string results = [] data = self.cache.get_rss_feed(self.url, params=search_params)['entries'] if not data: logger.debug( 'Data returned from provider does not contain any torrents' ) continue for curItem in data: try: title = curItem['title'] download_url = curItem['link'] if not all([title, download_url]): continue seeders = try_int(curItem['nyaa_seeders']) leechers = try_int(curItem['nyaa_leechers']) torrent_size = curItem['nyaa_size'] info_hash = curItem['nyaa_infohash'] if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.debug( 'Discarding torrent because it doesn\'t meet the' ' minimum seeders or leechers: {0} (S:{1} L:{2})' .format(title, seeders, leechers)) continue size = convert_size( torrent_size, units=['BYTES', 'KIB', 'MIB', 'GIB', 'TIB', 'PIB' ]) or -1 result = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': info_hash } if mode != 'RSS': logger.debug( _('Found result: {title} with {seeders} seeders and {leechers} leechers' .format(title=title, seeders=seeders, leechers=leechers))) items.append(result) except Exception: continue # For each search mode sort all the items by seeders items.sort(key=lambda d: d.get('seeders', 0), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): results = [] if not (self.url and self.urls): self.find_domain() if not (self.url and self.urls): return results anime = (self.show and self.show.anime) or (ep_obj and ep_obj.show and ep_obj.show.anime) or False search_params = { "field": "seeders", "sorder": "desc", "category": ("tv", "anime")[anime] } for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in search_strings[mode]: # search_params["q"] = (search_string, None)[mode == "RSS"] search_params["field"] = ("seeders", "time_add")[mode == "RSS"] if mode != "RSS": if anime: continue logger.debug( _("Search String: {search_string}".format( search_string=search_string))) search_url = self.urls["search"].format(q=search_string) else: search_url = self.urls["rss"] if self.custom_url: if not validators.url(self.custom_url): logger.warning("Invalid custom url: {0}".format( self.custom_url)) return results search_url = urljoin(self.custom_url, search_url.split(self.url)[1]) data = self.get_url(search_url, params=OrderedDict( sorted(list(search_params.items()), key=lambda x: x[0])), returns="text") if not data: logger.info( "{url} did not return any data, it may be disabled. Trying to get a new domain" .format(url=self.url)) self.disabled_mirrors.append(self.url) self.find_domain() if self.url in self.disabled_mirrors: logger.info("Could not find a better mirror to try.") logger.info( "The search did not return data, if the results are on the site maybe try a custom url, or a different one" ) return results # This will recurse a few times until all of the mirrors are exhausted if none of them work. return self.search(search_strings, age, ep_obj) with BS4Parser(data, "html5lib") as html: labels = [ cell.get_text() for cell in html.find(class_="firstr")("th") ] logger.info("Found {} results".format( len(html("tr", **self.rows_selector)))) for result in html("tr", **self.rows_selector): try: download_url = urllib.parse.unquote_plus( result.find( title="Torrent magnet link")["href"].split( "url=")[1]) + self._custom_trackers parsed_magnet = urllib.parse.parse_qs(download_url) torrent_hash = self.hash_from_magnet(download_url) title = result.find(class_="torrentname").find( class_="cellMainLink").get_text(strip=True) if title.endswith("..."): title = parsed_magnet['dn'][0] if not (title and download_url): if mode != "RSS": logger.debug( "Discarding torrent because We could not parse the title and url" ) continue seeders = try_int( result.find(class_="green").get_text( strip=True)) leechers = try_int( result.find(class_="red").get_text(strip=True)) # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != "RSS": logger.debug( "Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})" .format(title, seeders, leechers)) continue if self.confirmed and not result.find( class_="ka-green"): if mode != "RSS": logger.debug( "Found result " + title + " but that doesn't seem like a verified result so I'm ignoring it" ) continue torrent_size = result("td")[labels.index( "size")].get_text(strip=True) size = convert_size(torrent_size) or -1 item = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': torrent_hash } if mode != "RSS": logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) except (AttributeError, TypeError, KeyError, ValueError, Exception): logger.info(traceback.format_exc()) continue # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): results = [] if self.show and not self.show.is_anime: return results for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in {*search_strings[mode]}: if mode != 'RSS': logger.debug( _("Search String: {search_string}".format( search_string=search_string))) search_params = { "terms": search_string, "type": 1, # get anime types } data = self.get_url(self.urls['search'], params=search_params, returns='text') if not data: continue with BS4Parser(data, 'html5lib') as soup: torrent_table = soup.find('table', class_='listing') torrent_rows = torrent_table('tr') if torrent_table else [] # Continue only if one Release is found if len(torrent_rows) < 2: logger.debug( "Data returned from provider does not contain any torrents" ) continue a = 1 if len(torrent_rows[0]('td')) < 2 else 0 for top, bot in zip(torrent_rows[a::2], torrent_rows[a + 1::2]): try: desc_top = top.find('td', class_='desc-top') title = desc_top.get_text(strip=True) download_url = desc_top.find('a')['href'] desc_bottom = bot.find( 'td', class_='desc-bot').get_text(strip=True) size = convert_size( desc_bottom.split('|')[1].strip( 'Size: ')) or -1 stats = bot.find( 'td', class_='stats').get_text(strip=True) sl = re.match( r'S:(?P<seeders>\d+)L:(?P<leechers>\d+)C:(?:\d+)ID:(?:\d+)', stats.replace(' ', '')) seeders = try_int(sl.group('seeders')) if sl else 0 leechers = try_int( sl.group('leechers')) if sl else 0 except Exception: continue if not all([title, download_url]): continue # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.debug( "Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})" .format(title, seeders, leechers)) continue item = { 'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': '' } if mode != 'RSS': logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def search(self, search_strings, age=0, ep_obj=None): results = [] if not self.login(): return results # Search Params search_params = { 'cat[]': ['TV|SD|VOSTFR', 'TV|HD|VOSTFR', 'TV|SD|VF', 'TV|HD|VF', 'TV|PACK|FR', 'TV|PACK|VOSTFR', 'TV|EMISSIONS', 'ANIME'], # Both ASC and DESC are available for sort direction 'way': 'DESC' } # Units units = ['O', 'KO', 'MO', 'GO', 'TO', 'PO'] for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in {*search_strings[mode]}: if mode != 'RSS': logger.debug('Search string: {0}'.format (search_string)) # Sorting: Available parameters: ReleaseName, Seeders, Leechers, Snatched, Size search_params['order'] = ('Seeders', 'Time')[mode == 'RSS'] search_params['search'] = re.sub(r'[()]', '', search_string) data = self.get_url(self.urls['search'], params=search_params, returns='text') if not data: continue with BS4Parser(data, 'html5lib') as html: torrent_table = html.find(class_='torrent_table') torrent_rows = torrent_table('tr') if torrent_table else [] # Continue only if at least one Release is found if len(torrent_rows) < 2: logger.debug('Data returned from provider does not contain any torrents') continue # Catégorie, Release, Date, DL, Size, C, S, L labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')] # Skip column headers for result in torrent_rows[1:]: cells = result('td') if len(cells) < len(labels): continue try: title = cells[labels.index('Release')].get_text(strip=True) download_url = urljoin(self.url, cells[labels.index('DL')].find('a', class_='tooltip')['href']) if not all([title, download_url]): continue seeders = try_int(cells[labels.index('S')].get_text(strip=True)) leechers = try_int(cells[labels.index('L')].get_text(strip=True)) # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.debug('Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})'.format (title, seeders, leechers)) continue size_index = labels.index('Size') if 'Size' in labels else labels.index('Taille') torrent_size = cells[size_index].get_text() size = convert_size(torrent_size, units=units) or -1 item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''} if mode != 'RSS': logger.debug(_('Found result: {title} with {seeders} seeders and {leechers} leechers'.format( title=title, seeders=seeders, leechers=leechers))) items.append(item) except Exception: continue # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def safe_to_update(self): def db_safe(): message = { 'equal': { 'type': logger.DEBUG, 'text': "We can proceed with the update. New update has same DB version" }, 'upgrade': { 'type': logger.WARNING, 'text': "We can't proceed with the update. New update has a new DB version. Please manually update" }, 'downgrade': { 'type': logger.ERROR, 'text': "We can't proceed with the update. New update has a old DB version. It's not possible to downgrade" }, } try: result = self.compare_db_version() if result in message: logger.log( message[result]['type'], message[result] ['text']) # unpack the result message into a log entry else: logger.warning( "We can't proceed with the update. Unable to check remote DB version. Error: {0}" .format(result)) return result in ['equal' ] # add future True results to the list except Exception as error: logger.warning( "We can't proceed with the update. Unable to compare DB version. Error: {0}" .format(repr(error))) return False def postprocessor_safe(): if not settings.autoPostProcessorScheduler.action.amActive: logger.debug( "We can proceed with the update. Post-Processor is not running" ) return True else: logger.debug( "We can't proceed with the update. Post-Processor is running" ) return False def showupdate_safe(): if not settings.showUpdateScheduler.action.amActive: logger.debug( "We can proceed with the update. Shows are not being updated" ) return True else: logger.debug( "We can't proceed with the update. Shows are being updated" ) return False db_safe = db_safe() postprocessor_safe = postprocessor_safe() showupdate_safe = showupdate_safe() if db_safe and postprocessor_safe and showupdate_safe: logger.debug("Proceeding with auto update") return True else: logger.debug("Auto update aborted") return False
def search(self, search_strings, age=0, ep_obj=None): results = [] if not self.login(): return results # http://speed.cd/browse/49/50/52/41/55/2/30/freeleech/deep/q/arrow # Search Params search_params = [ "browse", "41", # TV/Packs "2", # Episodes "49", # TV/HD "50", # TV/Sports "52", # TV/B-Ray "55", # TV/Kids "30", # Anime ] if self.freeleech: search_params.append("freeleech") search_params.append("deep") # Units units = ["B", "KB", "MB", "GB", "TB", "PB"] def process_column_header(td): result = "" img = td.find("img") if img: result = img.get("alt") if not result: result = img.get("title") if not result: anchor = td.find("a") if anchor: result = anchor.get_text(strip=True) if not result: result = td.get_text(strip=True) return result for mode in search_strings: items = [] logger.debug(_("Search Mode: {mode}".format(mode=mode))) for search_string in {*search_strings[mode]}: current_params = search_params if mode != "RSS": logger.debug( _("Search String: {search_string}".format( search_string=search_string))) current_params += [ "q", re.sub(r"[^\w\s]", "", search_string) ] data = self.get_url(urljoin(self.url, "/".join(current_params)), returns="text") if not data: continue with BS4Parser(data, "html5lib") as html: torrent_table = html.find("div", class_="boxContent") torrent_table = torrent_table.find( "table") if torrent_table else [] # noinspection PyCallingNonCallable torrent_rows = torrent_table("tr") if torrent_table else [] # Continue only if at least one Release is found if len(torrent_rows) < 2: logger.debug( "Data returned from provider does not contain any torrents" ) continue labels = [ process_column_header(label) for label in torrent_rows[0]("th") ] row_labels = [ process_column_header(label) for label in torrent_rows[1]("td") ] def label_index(name): if name in labels: return labels.index(name) return row_labels.index(name) # Skip column headers for result in torrent_rows[1:]: try: cells = result("td") title = cells[label_index("Title")].find( "a").get_text() download_url = urljoin( self.url, cells[label_index("Download")].a["href"]) if not all([title, download_url]): continue seeders = try_int(cells[label_index("Seeders") - 1].get_text(strip=True)) leechers = try_int(cells[label_index("Leechers") - 1].get_text(strip=True)) # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != "RSS": logger.debug( "Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})" .format(title, seeders, leechers)) continue torrent_size = cells[label_index("Size") - 1].get_text() size = convert_size(torrent_size, units=units) or -1 item = { "title": title, "link": download_url, "size": size, "seeders": seeders, "leechers": leechers, "hash": "" } if mode != "RSS": logger.debug( "Found result: {0} with {1} seeders and {2} leechers" .format(title, seeders, leechers)) items.append(item) except Exception as error: logger.debug(f"Speed.cd: {error}") logger.debug(traceback.format_exc()) continue # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get("seeders", 0)), reverse=True) results += items return results