def _test_filter_bad_releases(self, name, expected): """Test filter of bad releases. :param name: :param expected: :return: """ result = naming.filter_bad_releases(name) self.assertEqual(result, expected)
def find_needed_episodes(self, episodes, forced_search=False, down_cur_quality=False): """ Search cache for needed episodes. NOTE: This is currently only used by the Daily Search. The following checks are performed on the cache results: * Use the episodes current quality / wanted quality to decide if we want it * Filtered on ignored/required words, and non-tv junk * Filter out non-anime results on Anime only providers * Check if the series is still in our library :param episodes: Single or list of episode object(s) :param forced_search: Flag to mark that this is searched through a forced search :param down_cur_quality: Flag to mark that we want to include the episode(s) current quality :return dict(episode: [list of SearchResult objects]). """ results = defaultdict(list) cache_results = self.find_episodes(episodes) for episode_number, search_results in viewitems(cache_results): for search_result in search_results: # ignored/required words, and non-tv junk if not naming.filter_bad_releases(search_result.name): continue all_wanted = True for cur_ep in search_result.actual_episodes: # if the show says we want that episode then add it to the list if not search_result.series.want_episode(search_result.actual_season, cur_ep, search_result.quality, forced_search, down_cur_quality): log.debug('Ignoring {0} because one or more episodes are unwanted', search_result.name) all_wanted = False break if not all_wanted: continue log.debug( '{id}: Using cached results from {provider} for series {show_name!r} episode {ep}', { 'id': search_result.series.series_id, 'provider': self.provider.name, 'show_name': search_result.series.name, 'ep': episode_num(search_result.episodes[0].season, search_result.episodes[0].episode), } ) if forced_search: search_result.search_type = FORCED_SEARCH search_result.download_current_quality = down_cur_quality # add it to the list results[episode_number].append(search_result) return results
def find_needed_episodes(self, episodes, forced_search=False, down_cur_quality=False): """ Search cache for needed episodes. NOTE: This is currently only used by the Daily Search. The following checks are performed on the cache results: * Use the episodes current quality / wanted quality to decide if we want it * Filtered on ignored/required words, and non-tv junk * Filter out non-anime results on Anime only providers * Check if the series is still in our library :param episodes: Single or list of episode object(s) :param forced_search: Flag to mark that this is searched through a forced search :param down_cur_quality: Flag to mark that we want to include the episode(s) current quality :return dict(episode: [list of SearchResult objects]). """ results = defaultdict(list) cache_results = self.find_episodes(episodes) for episode_number, search_results in viewitems(cache_results): for search_result in search_results: # ignored/required words, and non-tv junk if not naming.filter_bad_releases(search_result.name): continue all_wanted = True for cur_ep in search_result.actual_episodes: # if the show says we want that episode then add it to the list if not search_result.series.want_episode(search_result.actual_season, cur_ep, search_result.quality, forced_search, down_cur_quality): log.debug('Ignoring {0} because one or more episodes are unwanted', search_result.name) all_wanted = False break if not all_wanted: continue log.debug( '{id}: Using cached results from {provider} for series {show_name!r} episode {ep}', { 'id': search_result.series.series_id, 'provider': self.provider.name, 'show_name': search_result.series.name, 'ep': episode_num(search_result.episodes[0].season, search_result.episodes[0].episode), } ) # FIXME: Should be changed to search_result.search_type search_result.forced_search = forced_search search_result.download_current_quality = down_cur_quality # add it to the list results[episode_number].append(search_result) return results
def pick_best_result(results): # pylint: disable=too-many-branches """ Find the best result out of a list of search results for a show. :param results: list of result objects :return: best result object """ results = results if isinstance(results, list) else [results] log.debug(u'Picking the best result out of {0}', [x.name for x in results]) best_result = None # find the best result for the current episode for cur_result in results: assert cur_result.series, 'Every SearchResult object should have a series object available at this point.' # Every SearchResult object should have a show attribute available at this point. series_obj = cur_result.series # build the black and white list if series_obj.is_anime: if not series_obj.release_groups.is_valid(cur_result): continue log.info(u'Quality of {0} is {1}', cur_result.name, Quality.qualityStrings[cur_result.quality]) allowed_qualities, preferred_qualities = series_obj.current_qualities if cur_result.quality not in allowed_qualities + preferred_qualities: log.debug(u'{0} is an unwanted quality, rejecting it', cur_result.name) continue wanted_ep = True if cur_result.actual_episodes: wanted_ep = False for episode in cur_result.actual_episodes: if series_obj.want_episode(cur_result.actual_season, episode, cur_result.quality, cur_result.forced_search, cur_result.download_current_quality, search_type=cur_result.search_type): wanted_ep = True if not wanted_ep: continue # If doesnt have min seeders OR min leechers then discard it if cur_result.seeders not in (-1, None) and cur_result.leechers not in (-1, None) \ and hasattr(cur_result.provider, u'minseed') and hasattr(cur_result.provider, u'minleech') \ and (int(cur_result.seeders) < int(cur_result.provider.minseed) or int(cur_result.leechers) < int(cur_result.provider.minleech)): log.info( u'Discarding torrent because it does not meet the minimum provider setting ' u'S:{0} L:{1}. Result has S:{2} L:{3}', cur_result.provider.minseed, cur_result.provider.minleech, cur_result.seeders, cur_result.leechers, ) continue ignored_words = series_obj.show_words().ignored_words required_words = series_obj.show_words().required_words found_ignored_word = naming.contains_at_least_one_word( cur_result.name, ignored_words) found_required_word = naming.contains_at_least_one_word( cur_result.name, required_words) if ignored_words and found_ignored_word: log.info(u'Ignoring {0} based on ignored words filter: {1}', cur_result.name, found_ignored_word) continue if required_words and not found_required_word: log.info(u'Ignoring {0} based on required words filter: {1}', cur_result.name, required_words) continue if not naming.filter_bad_releases(cur_result.name, parse=False): continue if hasattr(cur_result, u'size'): if app.USE_FAILED_DOWNLOADS and failed_history.has_failed( cur_result.name, cur_result.size, cur_result.provider.name): log.info(u'{0} has previously failed, rejecting it', cur_result.name) continue preferred_words = [] if app.PREFERRED_WORDS: preferred_words = [_.lower() for _ in app.PREFERRED_WORDS] undesired_words = [] if app.UNDESIRED_WORDS: undesired_words = [_.lower() for _ in app.UNDESIRED_WORDS] if not best_result: best_result = cur_result if Quality.is_higher_quality(best_result.quality, cur_result.quality, allowed_qualities, preferred_qualities): best_result = cur_result elif best_result.quality == cur_result.quality: if any(ext in cur_result.name.lower() for ext in preferred_words): log.info(u'Preferring {0} (preferred words)', cur_result.name) best_result = cur_result if cur_result.proper_tags: log.info( u'Preferring {0} (repack/proper/real/rerip over nuked)', cur_result.name) best_result = cur_result if any(ext in best_result.name.lower() for ext in undesired_words) and not any( ext in cur_result.name.lower() for ext in undesired_words): log.info(u'Unwanted release {0} (contains undesired word(s))', cur_result.name) best_result = cur_result if best_result: log.debug(u'Picked {0} as the best', best_result.name) else: log.debug(u'No result picked.') return best_result
def find_needed_episodes(self, episode, forced_search=False, down_cur_quality=False): """Find needed episodes.""" needed_eps = {} results = [] cache_db_con = self._get_db() if not episode: sql_results = cache_db_con.select( b'SELECT * FROM [{name}]'.format(name=self.provider_id)) elif not isinstance(episode, list): sql_results = cache_db_con.select( b'SELECT * FROM [{name}] ' b'WHERE indexerid = ? AND' b' season = ? AND' b' episodes LIKE ?'.format(name=self.provider_id), [ episode.series.indexerid, episode.season, b'%|{0}|%'.format(episode.episode) ]) else: for ep_obj in episode: results.append([ b'SELECT * FROM [{name}] ' b'WHERE indexerid = ? AND' b' season = ? AND' b' episodes LIKE ? AND ' b' quality IN ({qualities})'.format( name=self.provider_id, qualities=','.join( (str(x) for x in ep_obj.wanted_quality))), [ ep_obj.series.indexerid, ep_obj.season, b'%|{0}|%'.format(ep_obj.episode) ] ]) if results: # Only execute the query if we have results sql_results = cache_db_con.mass_action(results, fetchall=True) sql_results = list(itertools.chain(*sql_results)) else: sql_results = [] log.debug( '{id}: No cached results in {provider} for series {show_name!r} episode {ep}', { 'id': episode[0].series.indexerid, 'provider': self.provider.name, 'show_name': episode[0].series.name, 'ep': episode_num(episode[0].season, episode[0].episode), }) # for each cache entry for cur_result in sql_results: search_result = self.provider.get_result() # ignored/required words, and non-tv junk if not naming.filter_bad_releases(cur_result[b'name']): continue # get the show, or ignore if it's not one of our shows show_obj = Show.find(app.showList, int(cur_result[b'indexerid'])) if not show_obj: continue # skip if provider is anime only and show is not anime if self.provider.anime_only and not show_obj.is_anime: log.debug('{0} is not an anime, skipping', show_obj.name) continue # get season and ep data (ignoring multi-eps for now) search_result.season = int(cur_result[b'season']) if search_result.season == -1: continue cur_ep = cur_result[b'episodes'].split('|')[1] if not cur_ep: continue cur_ep = int(cur_ep) search_result.quality = int(cur_result[b'quality']) search_result.release_group = cur_result[b'release_group'] search_result.version = cur_result[b'version'] # if the show says we want that episode then add it to the list if not show_obj.want_episode(search_result.season, cur_ep, search_result.quality, forced_search, down_cur_quality): log.debug('Ignoring {0}', cur_result[b'name']) continue search_result.episodes = [ show_obj.get_episode(search_result.season, cur_ep) ] search_result.actual_episodes = [search_result.episodes[0].episode] search_result.actual_season = search_result.season # build a result object search_result.name = cur_result[b'name'] search_result.url = cur_result[b'url'] log.debug( '{id}: Using cached results from {provider} for series {show_name!r} episode {ep}', { 'id': search_result.episodes[0].series.indexerid, 'provider': self.provider.name, 'show_name': search_result.episodes[0].series.name, 'ep': episode_num(search_result.episodes[0].season, search_result.episodes[0].episode), }) # Map the remaining attributes search_result.show = show_obj search_result.seeders = cur_result[b'seeders'] search_result.leechers = cur_result[b'leechers'] search_result.size = cur_result[b'size'] search_result.pubdate = cur_result[b'pubdate'] search_result.proper_tags = cur_result[b'proper_tags'].split( '|') if cur_result[b'proper_tags'] else '' search_result.content = None # FIXME: Should be changed to search_result.search_type search_result.forced_search = forced_search search_result.download_current_quality = down_cur_quality episode_object = search_result.episodes[0] # add it to the list if episode_object not in needed_eps: needed_eps[episode_object] = [search_result] else: needed_eps[episode_object].append(search_result) # datetime stamp this search so cache gets cleared self.searched = time() return needed_eps
def filter_results(results): """ Filter wanted results out of a list of search results for a show. :param results: list of result objects :return: list of wanted result objects """ results = results if isinstance(results, list) else [results] wanted_results = [] # find the best result for the current episode for cur_result in results: assert cur_result.series, 'Every SearchResult object should have a series object available at this point.' # Every SearchResult object should have a show attribute available at this point. series_obj = cur_result.series # build the black and white list if series_obj.is_anime: if not series_obj.release_groups.is_valid(cur_result): continue log.info(u'Quality of {0} is {1}', cur_result.name, Quality.qualityStrings[cur_result.quality]) allowed_qualities, preferred_qualities = series_obj.current_qualities if cur_result.quality not in allowed_qualities + preferred_qualities: log.debug(u'{0} is an unwanted quality, rejecting it', cur_result.name) continue wanted_ep = True if cur_result.actual_episodes: wanted_ep = False for episode in cur_result.actual_episodes: if series_obj.want_episode(cur_result.actual_season, episode, cur_result.quality, cur_result.forced_search, cur_result.download_current_quality, search_type=cur_result.search_type): wanted_ep = True if not wanted_ep: continue # If doesnt have min seeders OR min leechers then discard it if cur_result.seeders not in (-1, None) and cur_result.leechers not in (-1, None) \ and hasattr(cur_result.provider, u'minseed') and hasattr(cur_result.provider, u'minleech') \ and (int(cur_result.seeders) < int(cur_result.provider.minseed) or int(cur_result.leechers) < int(cur_result.provider.minleech)): log.info( u'Discarding torrent because it does not meet the minimum provider setting ' u'S:{0} L:{1}. Result has S:{2} L:{3}', cur_result.provider.minseed, cur_result.provider.minleech, cur_result.seeders, cur_result.leechers, ) continue ignored_words = series_obj.show_words().ignored_words required_words = series_obj.show_words().required_words found_ignored_word = naming.contains_at_least_one_word( cur_result.name, ignored_words) found_required_word = naming.contains_at_least_one_word( cur_result.name, required_words) if ignored_words and found_ignored_word: log.info(u'Ignoring {0} based on ignored words filter: {1}', cur_result.name, found_ignored_word) continue if required_words and not found_required_word: log.info(u'Ignoring {0} based on required words filter: {1}', cur_result.name, required_words) continue if not naming.filter_bad_releases(cur_result.name, parse=False): continue if hasattr(cur_result, u'size'): if app.USE_FAILED_DOWNLOADS and failed_history.has_failed( cur_result.name, cur_result.size, cur_result.provider.name): log.info(u'{0} has previously failed, rejecting it', cur_result.name) continue wanted_results.append(cur_result) if wanted_results: log.debug(u'Found wanted results.') else: log.debug(u'No wanted results found.') return wanted_results
def get_provider_cache_results(indexer, show_all_results=None, perform_search=None, show=None, season=None, episode=None, manual_search_type=None, **search_show): """Check all provider cache tables for search results.""" down_cur_quality = 0 show_obj = Show.find(app.showList, int(show)) preferred_words = show_obj.show_words().preferred_words undesired_words = show_obj.show_words().undesired_words ignored_words = show_obj.show_words().ignored_words required_words = show_obj.show_words().required_words main_db_con = db.DBConnection('cache.db') provider_results = { 'last_prov_updates': {}, 'error': {}, 'found_items': [] } original_thread_name = threading.currentThread().name sql_total = [] combined_sql_q = [] combined_sql_params = [] for cur_provider in enabled_providers('manualsearch'): threading.currentThread().name = '{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name) # Let's check if this provider table already exists table_exists = main_db_con.select( b"SELECT name " b"FROM sqlite_master " b"WHERE type='table'" b" AND name=?", [cur_provider.get_id()]) columns = [ i[1] for i in main_db_con.select("PRAGMA table_info('{0}')".format( cur_provider.get_id())) ] if table_exists else [] minseed = int(cur_provider.minseed) if getattr(cur_provider, 'minseed', None) else -1 minleech = int(cur_provider.minleech) if getattr( cur_provider, 'minleech', None) else -1 # TODO: the implicit sqlite rowid is used, should be replaced with an explicit PK column # If table doesn't exist, start a search to create table and new columns seeders, leechers and size required_columns = ['seeders', 'leechers', 'size', 'proper_tags'] if table_exists and all(required_column in columns for required_column in required_columns): # The default sql, that's executed for each providers cache table common_sql = ( b"SELECT rowid, ? AS 'provider_type', ? AS 'provider_image'," b" ? AS 'provider', ? AS 'provider_id', ? 'provider_minseed'," b" ? 'provider_minleech', name, season, episodes, indexerid," b" url, time, proper_tags, quality, release_group, version," b" seeders, leechers, size, time, pubdate " b"FROM '{provider_id}' " b"WHERE indexerid = ? AND quality > 0 ".format( provider_id=cur_provider.get_id())) # Let's start by adding the default parameters, which are used to subsitute the '?'s. add_params = [ cur_provider.provider_type.title(), cur_provider.image_name(), cur_provider.name, cur_provider.get_id(), minseed, minleech, show ] if manual_search_type != 'season': # If were not looking for all results, meaning don't do the filter on season + ep, add sql if not int(show_all_results): # If it's an episode search, pass season and episode. common_sql += " AND season = ? AND episodes LIKE ? " add_params += [season, "%|{0}|%".format(episode)] else: # If were not looking for all results, meaning don't do the filter on season + ep, add sql if not int(show_all_results): list_of_episodes = '{0}{1}'.format( ' episodes LIKE ', ' AND episodes LIKE '.join( ['?' for _ in show_obj.get_all_episodes(season)])) common_sql += " AND season = ? AND (episodes LIKE ? OR {list_of_episodes})".format( list_of_episodes=list_of_episodes) add_params += [season, '||'] # When the episodes field is empty. add_params += [ '%|{episode}|%'.format(episode=ep.episode) for ep in show_obj.get_all_episodes(season) ] # Add the created sql, to lists, that are used down below to perform one big UNIONED query combined_sql_q.append(common_sql) combined_sql_params += add_params # Get the last updated cache items timestamp last_update = main_db_con.select( b"SELECT max(time) AS lastupdate " b"FROM '{provider_id}'".format( provider_id=cur_provider.get_id())) provider_results['last_prov_updates'][ cur_provider.get_id()] = last_update[0][ 'lastupdate'] if last_update[0]['lastupdate'] else 0 # Check if we have the combined sql strings if combined_sql_q: sql_prepend = b"SELECT * FROM (" sql_append = b") ORDER BY CASE quality WHEN '{quality_unknown}' THEN -1 ELSE CAST(quality AS DECIMAL) END DESC, " \ b" proper_tags DESC, seeders DESC".format(quality_unknown=Quality.UNKNOWN) # Add all results sql_total += main_db_con.select( b'{0} {1} {2}'.format(sql_prepend, ' UNION ALL '.join(combined_sql_q), sql_append), combined_sql_params) # Always start a search when no items found in cache if not sql_total or int(perform_search): # retrieve the episode object and fail if we can't get one ep_obj = get_episode(show, season, episode) if isinstance(ep_obj, str): provider_results[ 'error'] = 'Something went wrong when starting the manual search for show {0}, \ and episode: {1}x{2}'.format(show_obj.name, season, episode) # make a queue item for it and put it on the queue ep_queue_item = ForcedSearchQueueItem(ep_obj.series, [ep_obj], bool(int(down_cur_quality)), True, manual_search_type) # pylint: disable=maybe-no-member app.forced_search_queue_scheduler.action.add_item(ep_queue_item) # give the CPU a break and some time to start the queue time.sleep(cpu_presets[app.CPU_PRESET]) else: cached_results = [dict(row) for row in sql_total] for i in cached_results: i['quality_name'] = Quality.split_quality(int(i['quality'])) i['time'] = datetime.fromtimestamp(i['time']) i['release_group'] = i['release_group'] or 'None' i['provider_img_link'] = 'images/providers/' + i[ 'provider_image'] or 'missing.png' i['provider'] = i['provider'] if i[ 'provider_image'] else 'missing provider' i['proper_tags'] = i['proper_tags'].replace('|', ', ') i['pretty_size'] = pretty_file_size( i['size']) if i['size'] > -1 else 'N/A' i['seeders'] = i['seeders'] if i['seeders'] >= 0 else '-' i['leechers'] = i['leechers'] if i['leechers'] >= 0 else '-' i['pubdate'] = sbdatetime.convert_to_setting( parser.parse(i['pubdate'])).strftime( app.DATE_PRESET + ' ' + app.TIME_PRESET) if i['pubdate'] else '-' release_group = i['release_group'] if ignored_words and release_group in ignored_words: i['rg_highlight'] = 'ignored' elif required_words and release_group in required_words: i['rg_highlight'] = 'required' elif preferred_words and release_group in preferred_words: i['rg_highlight'] = 'preferred' elif undesired_words and release_group in undesired_words: i['rg_highlight'] = 'undesired' else: i['rg_highlight'] = '' if contains_at_least_one_word(i['name'], required_words): i['name_highlight'] = 'required' elif contains_at_least_one_word( i['name'], ignored_words) or not filter_bad_releases( i['name'], parse=False): i['name_highlight'] = 'ignored' elif contains_at_least_one_word(i['name'], undesired_words): i['name_highlight'] = 'undesired' elif contains_at_least_one_word(i['name'], preferred_words): i['name_highlight'] = 'preferred' else: i['name_highlight'] = '' i['seed_highlight'] = 'ignored' if i.get( 'provider_minseed') > i.get('seeders', -1) >= 0 else '' i['leech_highlight'] = 'ignored' if i.get( 'provider_minleech') > i.get('leechers', -1) >= 0 else '' provider_results['found_items'] = cached_results # Remove provider from thread name before return results threading.currentThread().name = original_thread_name # Sanitize the last_prov_updates key provider_results['last_prov_updates'] = json.dumps( provider_results['last_prov_updates']) return provider_results
def find_needed_episodes(self, episode, forced_search=False, down_cur_quality=False): """Find needed episodes.""" needed_eps = {} results = [] cache_db_con = self._get_db() if not episode: sql_results = cache_db_con.select( b'SELECT * FROM [{name}]'.format(name=self.provider_id)) elif not isinstance(episode, list): sql_results = cache_db_con.select( b'SELECT * FROM [{name}] ' b'WHERE indexer = ? AND' b' indexerid = ? AND' b' season = ? AND' b' episodes LIKE ?'.format(name=self.provider_id), [ episode.series.indexer, episode.series.series_id, episode.season, '%|{0}|%'.format(episode.episode) ]) else: for ep_obj in episode: results.append([ b'SELECT * FROM [{name}] ' b'WHERE indexer = ? AND ' b' indexerid = ? AND' b' season = ? AND' b' episodes LIKE ? AND ' b' quality IN ({qualities})'.format( name=self.provider_id, qualities=','.join( (str(x) for x in ep_obj.wanted_quality))), [ ep_obj.series.indexer, ep_obj.series.series_id, ep_obj.season, '%|{0}|%'.format(ep_obj.episode) ] ]) if results: # Only execute the query if we have results sql_results = cache_db_con.mass_action(results, fetchall=True) sql_results = list(itertools.chain(*sql_results)) else: sql_results = [] log.debug( '{id}: No cached results in {provider} for series {show_name!r} episode {ep}', { 'id': episode[0].series.series_id, 'provider': self.provider.name, 'show_name': episode[0].series.name, 'ep': episode_num(episode[0].season, episode[0].episode), }) # for each cache entry for cur_result in sql_results: if cur_result[b'indexer'] is None: log.debug( 'Ignoring result: {0}, missing indexer. This is probably a result added' ' prior to medusa version 0.2.0', cur_result[b'name']) continue search_result = self.provider.get_result() # ignored/required words, and non-tv junk if not naming.filter_bad_releases(cur_result[b'name']): continue # get the show, or ignore if it's not one of our shows series_obj = Show.find_by_id(app.showList, int(cur_result[b'indexer']), int(cur_result[b'indexerid'])) if not series_obj: continue # skip if provider is anime only and show is not anime if self.provider.anime_only and not series_obj.is_anime: log.debug('{0} is not an anime, skipping', series_obj.name) continue # build a result object search_result.quality = int(cur_result[b'quality']) search_result.release_group = cur_result[b'release_group'] search_result.version = cur_result[b'version'] search_result.name = cur_result[b'name'] search_result.url = cur_result[b'url'] search_result.season = int(cur_result[b'season']) search_result.actual_season = search_result.season sql_episodes = cur_result[b'episodes'].strip('|') # TODO: Add support for season results # Season result if not sql_episodes: ep_objs = self.series.get_all_episodes(search_result.season) actual_episodes = [ep.episode for ep in ep_objs] episode_number = SEASON_RESULT # Multi or single episode result else: actual_episodes = [int(ep) for ep in sql_episodes.split('|')] ep_objs = [ series_obj.get_episode(search_result.season, ep) for ep in actual_episodes ] if len(actual_episodes) == 1: episode_number = actual_episodes[0] else: episode_number = MULTI_EP_RESULT all_wanted = True for cur_ep in actual_episodes: # if the show says we want that episode then add it to the list if not series_obj.want_episode( search_result.season, cur_ep, search_result.quality, forced_search, down_cur_quality): log.debug( 'Ignoring {0} because one or more episodes are unwanted', cur_result[b'name']) all_wanted = False break if not all_wanted: continue search_result.episodes = ep_objs search_result.actual_episodes = actual_episodes log.debug( '{id}: Using cached results from {provider} for series {show_name!r} episode {ep}', { 'id': search_result.episodes[0].series.series_id, 'provider': self.provider.name, 'show_name': search_result.episodes[0].series.name, 'ep': episode_num(search_result.episodes[0].season, search_result.episodes[0].episode), }) # Map the remaining attributes search_result.series = series_obj search_result.seeders = cur_result[b'seeders'] search_result.leechers = cur_result[b'leechers'] search_result.size = cur_result[b'size'] search_result.pubdate = cur_result[b'pubdate'] search_result.proper_tags = cur_result[b'proper_tags'].split( '|') if cur_result[b'proper_tags'] else '' search_result.content = None # FIXME: Should be changed to search_result.search_type search_result.forced_search = forced_search search_result.download_current_quality = down_cur_quality # add it to the list if episode_number not in needed_eps: needed_eps[episode_number] = [search_result] else: needed_eps[episode_number].append(search_result) # datetime stamp this search so cache gets cleared self.searched = time() return needed_eps
def filter_results(results): """ Filter wanted results out of a list of search results for a show. :param results: list of result objects :return: list of wanted result objects """ results = results if isinstance(results, list) else [results] wanted_results = [] # find the best result for the current episode for cur_result in results: assert cur_result.series, 'Every SearchResult object should have a series object available at this point.' # Every SearchResult object should have a show attribute available at this point. series_obj = cur_result.series # build the black and white list if series_obj.is_anime: if not series_obj.release_groups.is_valid(cur_result): continue log.info(u'Quality of {0} is {1}', cur_result.name, Quality.qualityStrings[cur_result.quality]) allowed_qualities, preferred_qualities = series_obj.current_qualities if cur_result.quality not in allowed_qualities + preferred_qualities: log.debug(u'{0} is an unwanted quality, rejecting it', cur_result.name) continue wanted_ep = True if cur_result.actual_episodes: wanted_ep = False for episode in cur_result.actual_episodes: if series_obj.want_episode(cur_result.actual_season, episode, cur_result.quality, cur_result.forced_search, cur_result.download_current_quality, search_type=cur_result.search_type): wanted_ep = True if not wanted_ep: continue # If doesnt have min seeders OR min leechers then discard it if cur_result.seeders not in (-1, None) and cur_result.leechers not in (-1, None) \ and hasattr(cur_result.provider, u'minseed') and hasattr(cur_result.provider, u'minleech') \ and (int(cur_result.seeders) < int(cur_result.provider.minseed) or int(cur_result.leechers) < int(cur_result.provider.minleech)): log.info( u'Discarding torrent because it does not meet the minimum provider setting ' u'S:{0} L:{1}. Result has S:{2} L:{3}', cur_result.provider.minseed, cur_result.provider.minleech, cur_result.seeders, cur_result.leechers, ) continue ignored_words = series_obj.show_words().ignored_words required_words = series_obj.show_words().required_words found_ignored_word = naming.contains_at_least_one_word(cur_result.name, ignored_words) found_required_word = naming.contains_at_least_one_word(cur_result.name, required_words) if ignored_words and found_ignored_word: log.info(u'Ignoring {0} based on ignored words filter: {1}', cur_result.name, found_ignored_word) continue if required_words and not found_required_word: log.info(u'Ignoring {0} based on required words filter: {1}', cur_result.name, required_words) continue if not naming.filter_bad_releases(cur_result.name, parse=False): continue if hasattr(cur_result, u'size'): if app.USE_FAILED_DOWNLOADS and failed_history.has_failed(cur_result.name, cur_result.size, cur_result.provider.name): log.info(u'{0} has previously failed, rejecting it', cur_result.name) continue wanted_results.append(cur_result) if wanted_results: log.debug(u'Found wanted results.') else: log.debug(u'No wanted results found.') return wanted_results
def get_provider_cache_results(series_obj, show_all_results=None, perform_search=None, season=None, episode=None, manual_search_type=None, **search_show): """Check all provider cache tables for search results.""" down_cur_quality = 0 preferred_words = series_obj.show_words().preferred_words undesired_words = series_obj.show_words().undesired_words ignored_words = series_obj.show_words().ignored_words required_words = series_obj.show_words().required_words main_db_con = db.DBConnection('cache.db') provider_results = {'last_prov_updates': {}, 'error': {}, 'found_items': []} original_thread_name = threading.currentThread().name cached_results_total = [] combined_sql_q = [] combined_sql_params = [] for cur_provider in enabled_providers('manualsearch'): threading.currentThread().name = '{thread} :: [{provider}]'.format(thread=original_thread_name, provider=cur_provider.name) # Let's check if this provider table already exists table_exists = main_db_con.select( 'SELECT name ' 'FROM sqlite_master ' "WHERE type='table'" ' AND name=?', [cur_provider.get_id()] ) columns = [] if table_exists: table_columns = main_db_con.select("PRAGMA table_info('{0}')".format(cur_provider.get_id())) columns = [table_column['name'] for table_column in table_columns] minseed = int(cur_provider.minseed) if getattr(cur_provider, 'minseed', None) else -1 minleech = int(cur_provider.minleech) if getattr(cur_provider, 'minleech', None) else -1 # TODO: the implicit sqlite rowid is used, should be replaced with an explicit PK column # If table doesn't exist, start a search to create table and new columns seeders, leechers and size required_columns = ['indexer', 'indexerid', 'seeders', 'leechers', 'size', 'proper_tags', 'date_added'] if table_exists and all(required_column in columns for required_column in required_columns): # The default sql, that's executed for each providers cache table common_sql = ( "SELECT rowid, ? AS 'provider_type', ? AS 'provider_image'," " ? AS 'provider', ? AS 'provider_id', ? 'provider_minseed'," " ? 'provider_minleech', name, season, episodes, indexer, indexerid," ' url, proper_tags, quality, release_group, version,' ' seeders, leechers, size, time, pubdate, date_added ' "FROM '{provider_id}' " 'WHERE indexer = ? AND indexerid = ? AND quality > 0 '.format( provider_id=cur_provider.get_id() ) ) # Let's start by adding the default parameters, which are used to substitute the '?'s. add_params = [cur_provider.provider_type.title(), cur_provider.image_name(), cur_provider.name, cur_provider.get_id(), minseed, minleech, series_obj.indexer, series_obj.series_id] if manual_search_type != 'season': # If were not looking for all results, meaning don't do the filter on season + ep, add sql if not int(show_all_results): # If it's an episode search, pass season and episode. common_sql += ' AND season = ? AND episodes LIKE ? ' add_params += [season, '%|{0}|%'.format(episode)] else: # If were not looking for all results, meaning don't do the filter on season + ep, add sql if not int(show_all_results): list_of_episodes = '{0}{1}'.format(' episodes LIKE ', ' AND episodes LIKE '.join( ['?' for _ in series_obj.get_all_episodes(season)] )) common_sql += ' AND season = ? AND (episodes LIKE ? OR {list_of_episodes})'.format( list_of_episodes=list_of_episodes ) add_params += [season, '||'] # When the episodes field is empty. add_params += ['%|{episode}|%'.format(episode=ep.episode) for ep in series_obj.get_all_episodes(season)] # Add the created sql, to lists, that are used down below to perform one big UNIONED query combined_sql_q.append(common_sql) combined_sql_params += add_params # Get the last updated cache items timestamp last_update = main_db_con.select('SELECT max(time) AS lastupdate ' "FROM '{provider_id}'".format(provider_id=cur_provider.get_id())) provider_results['last_prov_updates'][cur_provider.get_id()] = last_update[0]['lastupdate'] if last_update[0]['lastupdate'] else 0 # Check if we have the combined sql strings if combined_sql_q: sql_prepend = 'SELECT * FROM (' sql_append = ') ORDER BY quality DESC, proper_tags DESC, seeders DESC' # Add all results cached_results_total += main_db_con.select('{0} {1} {2}'. format(sql_prepend, ' UNION ALL '.join(combined_sql_q), sql_append), combined_sql_params) # Always start a search when no items found in cache if not cached_results_total or int(perform_search): # retrieve the episode object and fail if we can't get one ep_obj = series_obj.get_episode(season, episode) if isinstance(ep_obj, str): provider_results['error'] = 'Something went wrong when starting the manual search for show {0}, \ and episode: {1}x{2}'.format(series_obj.name, season, episode) # make a queue item for it and put it on the queue ep_queue_item = ForcedSearchQueueItem(ep_obj.series, [ep_obj], bool(int(down_cur_quality)), True, manual_search_type) # pylint: disable=maybe-no-member app.forced_search_queue_scheduler.action.add_item(ep_queue_item) # give the CPU a break and some time to start the queue time.sleep(cpu_presets[app.CPU_PRESET]) else: for i in cached_results_total: threading.currentThread().name = '{thread} :: [{provider}]'.format( thread=original_thread_name, provider=i['provider']) i['quality_name'] = Quality.split_quality(int(i['quality'])) i['time'] = datetime.fromtimestamp(i['time']) i['release_group'] = i['release_group'] or 'None' i['provider_img_link'] = 'images/providers/' + i['provider_image'] or 'missing.png' i['provider'] = i['provider'] if i['provider_image'] else 'missing provider' i['proper_tags'] = i['proper_tags'].replace('|', ', ') i['pretty_size'] = pretty_file_size(i['size']) if i['size'] > -1 else 'N/A' i['seeders'] = i['seeders'] if i['seeders'] >= 0 else '-' i['leechers'] = i['leechers'] if i['leechers'] >= 0 else '-' i['pubdate'] = parser.parse(i['pubdate']).astimezone(app_timezone) if i['pubdate'] else '' i['date_added'] = datetime.fromtimestamp(float(i['date_added']), tz=app_timezone) if i['date_added'] else '' release_group = i['release_group'] if ignored_words and release_group in ignored_words: i['rg_highlight'] = 'ignored' elif required_words and release_group in required_words: i['rg_highlight'] = 'required' elif preferred_words and release_group in preferred_words: i['rg_highlight'] = 'preferred' elif undesired_words and release_group in undesired_words: i['rg_highlight'] = 'undesired' else: i['rg_highlight'] = '' if contains_at_least_one_word(i['name'], required_words): i['name_highlight'] = 'required' elif contains_at_least_one_word(i['name'], ignored_words) or not filter_bad_releases(i['name'], parse=False): i['name_highlight'] = 'ignored' elif contains_at_least_one_word(i['name'], undesired_words): i['name_highlight'] = 'undesired' elif contains_at_least_one_word(i['name'], preferred_words): i['name_highlight'] = 'preferred' else: i['name_highlight'] = '' i['seed_highlight'] = 'ignored' if i['seeders'] == '-' or i['provider_minseed'] <= i['seeders']: i['seed_highlight'] = '' i['leech_highlight'] = 'ignored' if i['leechers'] == '-' or i['provider_minleech'] <= i['leechers']: i['leech_highlight'] = '' provider_results['found_items'] = cached_results_total # Remove provider from thread name before return results threading.currentThread().name = original_thread_name # Sanitize the last_prov_updates key provider_results['last_prov_updates'] = json.dumps(provider_results['last_prov_updates']) return provider_results