def search_for_needed_episodes(force=False): """ Check providers for details on wanted episodes. :return: episodes we have a search hit for """ found_results = {} show_list = app.showList from_date = datetime.date.fromordinal(1) episodes = [] for cur_show in show_list: if cur_show.paused: log.debug( u'Not checking for needed episodes of {0} because the show is paused', cur_show.name) continue episodes.extend(wanted_episodes(cur_show, from_date)) if not episodes and not force: # nothing wanted so early out, ie: avoid whatever arbitrarily # complex thing a provider cache update entails, for example, # reading rss feeds return list(itervalues(found_results)) original_thread_name = threading.currentThread().name providers = enabled_providers(u'daily') if not providers: log.warning( u'No NZB/Torrent providers found or enabled in the application config for daily searches.' u' Please check your settings') return list(itervalues(found_results)) log.info(u'Using daily search providers') for cur_provider in providers: threading.currentThread().name = u'{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name) cur_provider.cache.update_cache() for cur_provider in providers: threading.currentThread().name = u'{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name) try: cur_found_results = cur_provider.search_rss(episodes) except AuthException as error: log.error(u'Authentication error: {0}', ex(error)) continue # pick a single result for each episode, respecting existing results for cur_ep in cur_found_results: if not cur_ep.series or cur_ep.series.paused: log.debug(u'Skipping {0} because the show is paused ', cur_ep.pretty_name()) continue best_result = pick_best_result(cur_found_results[cur_ep]) # if all results were rejected move on to the next episode if not best_result: log.debug(u'All found results for {0} were rejected.', cur_ep.pretty_name()) continue # if it's already in the list (from another provider) and the newly found quality is no better then skip it if cur_ep in found_results and best_result.quality <= found_results[ cur_ep].quality: continue # Skip the result if search delay is enabled for the provider. if delay_search(best_result): continue found_results[cur_ep] = best_result threading.currentThread().name = original_thread_name return list(itervalues(found_results))
def search_providers(series_obj, episodes, forced_search=False, down_cur_quality=False, manual_search=False, manual_search_type=u'episode'): """ Walk providers for information on shows. :param series_obj: Show we are looking for :param episodes: List, episodes we hope to find :param forced_search: Boolean, is this a forced search? :param down_cur_quality: Boolean, should we re-download currently available quality file :param manual_search: Boolean, should we choose what to download? :param manual_search_type: Episode or Season search :return: results for search """ found_results = {} final_results = [] manual_search_results = [] # build name cache for show name_cache.build_name_cache(series_obj) original_thread_name = threading.currentThread().name if manual_search: log.info(u'Using manual search providers') providers = enabled_providers(u'manualsearch') else: log.info(u'Using backlog search providers') providers = enabled_providers(u'backlog') if not providers: log.warning( u'No NZB/Torrent providers found or enabled in the application config for {0} searches.' u' Please check your settings', 'manual' if manual_search else 'backlog') threading.currentThread().name = original_thread_name for cur_provider in providers: threading.currentThread( ).name = original_thread_name + u' :: [' + cur_provider.name + u']' if cur_provider.anime_only and not series_obj.is_anime: log.debug(u'{0} is not an anime, skipping', series_obj.name) continue found_results[cur_provider.name] = {} search_count = 0 search_mode = cur_provider.search_mode # Always search for episode when manually searching when in sponly if search_mode == u'sponly' and (forced_search or manual_search): search_mode = u'eponly' if manual_search and manual_search_type == u'season': search_mode = u'sponly' while True: search_count += 1 if search_mode == u'eponly': log.info(u'Performing episode search for {0}', series_obj.name) else: log.info(u'Performing season pack search for {0}', series_obj.name) try: search_results = cur_provider.find_search_results( series_obj, episodes, search_mode, forced_search, down_cur_quality, manual_search, manual_search_type) except AuthException as error: log.error(u'Authentication error: {0}', ex(error)) break if search_results: # make a list of all the results for this provider for cur_ep in search_results: if cur_ep in found_results[cur_provider.name]: found_results[cur_provider. name][cur_ep] += search_results[cur_ep] else: found_results[ cur_provider.name][cur_ep] = search_results[cur_ep] # Sort the list by seeders if possible if cur_provider.provider_type == u'torrent' or getattr( cur_provider, u'torznab', None): found_results[cur_provider.name][cur_ep].sort( key=lambda d: int(d.seeders), reverse=True) break elif not cur_provider.search_fallback or search_count == 2: break # Don't fallback when doing manual season search if manual_search_type == u'season': break if search_mode == u'sponly': log.debug(u'Fallback episode search initiated') search_mode = u'eponly' else: log.debug(u'Fallback season pack search initiate') search_mode = u'sponly' # skip to next provider if we have no results to process if not found_results[cur_provider.name]: continue # Update the cache if a manual search is being run if manual_search: # Let's create a list with episodes that we where looking for if manual_search_type == u'season': # If season search type, we only want season packs searched_episode_list = [SEASON_RESULT] else: searched_episode_list = [ episode_obj.episode for episode_obj in episodes ] + [MULTI_EP_RESULT] for searched_episode in searched_episode_list: if (searched_episode in search_results and cur_provider.cache.update_cache_manual_search( search_results[searched_episode])): # If we have at least a result from one provider, it's good enough to be marked as result manual_search_results.append(True) # Continue because we don't want to pick best results as we are running a manual search by user continue # pick the best season NZB best_season_result = None if SEASON_RESULT in found_results[cur_provider.name]: best_season_result = pick_best_result( found_results[cur_provider.name][SEASON_RESULT]) highest_quality_overall = 0 for cur_episode in found_results[cur_provider.name]: for cur_result in found_results[cur_provider.name][cur_episode]: if cur_result.quality > highest_quality_overall: highest_quality_overall = cur_result.quality log.debug(u'The highest quality of any match is {0}', Quality.qualityStrings[highest_quality_overall]) # see if every episode is wanted if best_season_result: searched_seasons = {str(x.season) for x in episodes} # get the quality of the season nzb season_quality = best_season_result.quality log.debug(u'The quality of the season {0} is {1}', best_season_result.provider.provider_type, Quality.qualityStrings[season_quality]) main_db_con = db.DBConnection() selection = main_db_con.select( 'SELECT episode ' 'FROM tv_episodes ' 'WHERE indexer = ?' ' AND showid = ?' ' AND ( season IN ( {0} ) )'.format( ','.join(searched_seasons)), [series_obj.indexer, series_obj.series_id]) all_eps = [int(x[b'episode']) for x in selection] log.debug(u'Episode list: {0}', all_eps) all_wanted = True any_wanted = False for cur_ep_num in all_eps: for season in {x.season for x in episodes}: if not series_obj.want_episode(season, cur_ep_num, season_quality, down_cur_quality): all_wanted = False else: any_wanted = True # if we need every ep in the season and there's nothing better then # just download this and be done with it (unless single episodes are preferred) if all_wanted and best_season_result.quality == highest_quality_overall: log.info( u'All episodes in this season are needed, downloading {0} {1}', best_season_result.provider.provider_type, best_season_result.name) ep_objs = [] for cur_ep_num in all_eps: for season in {x.season for x in episodes}: ep_objs.append( series_obj.get_episode(season, cur_ep_num)) best_season_result.episodes = ep_objs # Remove provider from thread name before return results threading.currentThread().name = original_thread_name return [best_season_result] elif not any_wanted: log.debug( u'No episodes in this season are needed at this quality, ignoring {0} {1}', best_season_result.provider.provider_type, best_season_result.name) else: # Some NZB providers (e.g. Jackett) can also download torrents, but torrents cannot be split like NZB if (best_season_result.provider.provider_type == GenericProvider.NZB and not best_season_result.url.endswith( GenericProvider.TORRENT)): log.debug( u'Breaking apart the NZB and adding the individual ones to our results' ) # if not, break it apart and add them as the lowest priority results individual_results = nzb_splitter.split_result( best_season_result) for cur_result in individual_results: if len(cur_result.episodes) == 1: ep_number = cur_result.episodes[0].episode elif len(cur_result.episodes) > 1: ep_number = MULTI_EP_RESULT if ep_number in found_results[cur_provider.name]: found_results[cur_provider.name][ep_number].append( cur_result) else: found_results[cur_provider.name][ep_number] = [ cur_result ] # If this is a torrent all we can do is leech the entire torrent, # user will have to select which eps not do download in his torrent client else: # Season result from Torrent Provider must be a full-season torrent, # creating multi-ep result for it. log.info( u'Adding multi-ep result for full-season torrent.' u' Undesired episodes can be skipped in torrent client if desired!' ) ep_objs = [] for cur_ep_num in all_eps: for season in {x.season for x in episodes}: ep_objs.append( series_obj.get_episode(season, cur_ep_num)) best_season_result.episodes = ep_objs if MULTI_EP_RESULT in found_results[cur_provider.name]: found_results[cur_provider.name][ MULTI_EP_RESULT].append(best_season_result) else: found_results[cur_provider.name][MULTI_EP_RESULT] = [ best_season_result ] # go through multi-ep results and see if we really want them or not, get rid of the rest multi_results = {} if MULTI_EP_RESULT in found_results[cur_provider.name]: for _multi_result in found_results[ cur_provider.name][MULTI_EP_RESULT]: log.debug( u'Seeing if we want to bother with multi-episode result {0}', _multi_result.name) # Filter result by ignore/required/whitelist/blacklist/quality, etc multi_result = pick_best_result(_multi_result) if not multi_result: continue # see how many of the eps that this result covers aren't covered by single results needed_eps = [] not_needed_eps = [] for ep_obj in multi_result.episodes: # if we have results for the episode if ep_obj.episode in found_results[cur_provider.name] and \ len(found_results[cur_provider.name][ep_obj.episode]) > 0: not_needed_eps.append(ep_obj.episode) else: needed_eps.append(ep_obj.episode) log.debug( u'Single-ep check result is needed_eps: {0}, not_needed_eps: {1}', needed_eps, not_needed_eps) if not needed_eps: log.debug( u'All of these episodes were covered by single episode results,' u' ignoring this multi-episode result') continue # check if these eps are already covered by another multi-result multi_needed_eps = [] multi_not_needed_eps = [] for ep_obj in multi_result.episodes: if ep_obj.episode in multi_results: multi_not_needed_eps.append(ep_obj.episode) else: multi_needed_eps.append(ep_obj.episode) log.debug( u'Multi-ep check result is multi_needed_eps: {0}, multi_not_needed_eps: {1}', multi_needed_eps, multi_not_needed_eps) if not multi_needed_eps: log.debug( u'All of these episodes were covered by another multi-episode nzb, ' u'ignoring this multi-ep result') continue # don't bother with the single result if we're going to get it with a multi result for ep_obj in multi_result.episodes: multi_results[ep_obj.episode] = multi_result if ep_obj.episode in found_results[cur_provider.name]: log.debug( u'A needed multi-episode result overlaps with a single-episode result for episode {0},' u' removing the single-episode results from the list', ep_obj.episode, ) del found_results[cur_provider.name][ep_obj.episode] # of all the single ep results narrow it down to the best one for each episode final_results += set(multi_results.values()) for cur_ep in found_results[cur_provider.name]: if cur_ep in (MULTI_EP_RESULT, SEASON_RESULT): continue if not found_results[cur_provider.name][cur_ep]: continue # if all results were rejected move on to the next episode best_result = pick_best_result( found_results[cur_provider.name][cur_ep]) if not best_result: continue # add result if its not a duplicate and found = False for i, result in enumerate(final_results): for best_resultEp in best_result.episodes: if best_resultEp in result.episodes: if result.quality < best_result.quality: final_results.pop(i) else: found = True if not found: # Skip the result if search delay is enabled for the provider. if not delay_search(best_result): final_results += [best_result] # Remove provider from thread name before return results threading.currentThread().name = original_thread_name if manual_search: # If results in manual search return True, else False return any(manual_search_results) else: return final_results
def search_providers(series_obj, episodes, forced_search=False, down_cur_quality=False, manual_search=False, manual_search_type=u'episode'): """ Walk providers for information on shows. :param series_obj: Show we are looking for :param episodes: List, episodes we hope to find :param forced_search: Boolean, is this a forced search? :param down_cur_quality: Boolean, should we re-download currently available quality file :param manual_search: Boolean, should we choose what to download? :param manual_search_type: Episode or Season search :return: results for search """ found_results = {} manual_search_results = [] multi_results = [] single_results = [] # build name cache for show name_cache.build_name_cache(series_obj) original_thread_name = threading.currentThread().name if manual_search: log.info(u'Using manual search providers') providers = enabled_providers(u'manualsearch') else: log.info(u'Using backlog search providers') providers = enabled_providers(u'backlog') if not providers: log.warning( u'No NZB/Torrent providers found or enabled in the application config for {0} searches.' u' Please check your settings', 'manual' if manual_search else 'backlog') threading.currentThread().name = original_thread_name for cur_provider in providers: threading.currentThread( ).name = '{original_thread_name} :: [{provider}]'.format( original_thread_name=original_thread_name, provider=cur_provider.name) if cur_provider.anime_only and not series_obj.is_anime: log.debug(u'{0} is not an anime, skipping', series_obj.name) continue found_results[cur_provider.name] = {} search_count = 0 search_mode = cur_provider.search_mode # Always search for episode when manually searching when in sponly if search_mode == u'sponly' and (forced_search or manual_search): search_mode = u'eponly' if manual_search and manual_search_type == u'season': search_mode = u'sponly' while True: search_count += 1 if search_mode == u'eponly': log.info(u'Performing episode search for {0}', series_obj.name) else: log.info(u'Performing season pack search for {0}', series_obj.name) try: search_results = [] cache_search_results = [] cache_multi = [] cache_single = [] if not manual_search: cache_search_results = cur_provider.search_results_in_cache( episodes) if cache_search_results: # From our provider multi_episode and single_episode results, collect candidates. cache_found_results = list_results_for_provider( cache_search_results, found_results, cur_provider) # We're passing the empty lists, because we don't want to include previous candidates cache_multi, cache_single = collect_candidates( cache_found_results, cur_provider, [], [], series_obj, down_cur_quality) # For now we only search if we didn't get any results back from cache, # but we might wanna check if there was something useful in cache. if not (cache_multi or cache_single): log.debug( u'Could not find any candidates in cache, searching provider.' ) search_results = cur_provider.find_search_results( series_obj, episodes, search_mode, forced_search, down_cur_quality, manual_search, manual_search_type) # Update the list found_results found_results = list_results_for_provider( search_results, found_results, cur_provider) else: found_results = cache_found_results except AuthException as error: log.error(u'Authentication error: {0!r}', error) break if search_results or cache_search_results: break elif not cur_provider.search_fallback or search_count == 2: break # Don't fallback when doing manual season search if manual_search_type == u'season': break if search_mode == u'sponly': log.debug(u'Fallback episode search initiated') search_mode = u'eponly' else: log.debug(u'Fallback season pack search initiated') search_mode = u'sponly' # skip to next provider if we have no results to process if not found_results[cur_provider.name]: continue # Update the cache if a manual search is being run if manual_search: # Let's create a list with episodes that we where looking for if manual_search_type == u'season': # If season search type, we only want season packs searched_episode_list = [SEASON_RESULT] else: searched_episode_list = [ episode_obj.episode for episode_obj in episodes ] + [MULTI_EP_RESULT] for searched_episode in searched_episode_list: if (searched_episode in search_results and cur_provider.cache.update_cache_manual_search( search_results[searched_episode])): # If we have at least a result from one provider, it's good enough to be marked as result manual_search_results.append(True) # Continue because we don't want to pick best results as we are running a manual search by user continue multi_results, single_results = collect_candidates( found_results, cur_provider, multi_results, single_results, series_obj, down_cur_quality) # Remove provider from thread name before return results threading.currentThread().name = original_thread_name if manual_search: # If results in manual search return True, else False return any(manual_search_results) else: return combine_results(multi_results, single_results)
def _get_proper_results(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements """Retrieve a list of recently aired episodes, and search for these episodes in the different providers.""" propers = {} # For each provider get the list of propers original_thread_name = threading.currentThread().name providers = enabled_providers('backlog') search_date = datetime.datetime.today() - datetime.timedelta( days=app.PROPERS_SEARCH_DAYS) main_db_con = db.DBConnection() if not app.POSTPONE_IF_NO_SUBS: # Get the recently aired (last 2 days) shows from DB search_q_params = ','.join('?' for _ in Quality.DOWNLOADED) recently_aired = main_db_con.select( b'SELECT showid, season, episode, status, airdate' b' FROM tv_episodes' b' WHERE airdate >= ?' b' AND status IN ({0})'.format(search_q_params), [search_date.toordinal()] + Quality.DOWNLOADED) else: # Get recently subtitled episodes (last 2 days) from DB # Episode status becomes downloaded only after found subtitles last_subtitled = search_date.strftime(History.date_format) recently_aired = main_db_con.select( b'SELECT showid, season, episode FROM history ' b"WHERE date >= ? AND action LIKE '%10'", [last_subtitled]) if not recently_aired: log.info('No recently aired new episodes, nothing to search for') return [] # Loop through the providers, and search for releases for cur_provider in providers: threading.currentThread().name = '{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name) log.info('Searching for any new PROPER releases from {provider}', {'provider': cur_provider.name}) try: cur_propers = cur_provider.find_propers(recently_aired) except AuthException as e: log.debug('Authentication error: {error}', {'error': ex(e)}) continue # if they haven't been added by a different provider than add the proper to the list for proper in cur_propers: name = self._sanitize_name(proper.name) if name not in propers: log.debug('Found new possible proper result: {name}', {'name': proper.name}) propers[name] = proper threading.currentThread().name = original_thread_name # take the list of unique propers and get it sorted by sorted_propers = sorted(propers.values(), key=operator.attrgetter('date'), reverse=True) final_propers = [] # Keep only items from last PROPER_SEARCH_DAYS setting in processed propers: latest_proper = datetime.datetime.now() - datetime.timedelta( days=app.PROPERS_SEARCH_DAYS) self.processed_propers = [ p for p in self.processed_propers if p.get('date') >= latest_proper ] # Get proper names from processed propers processed_propers_names = [ proper.get('name') for proper in self.processed_propers if proper.get('name') ] for cur_proper in sorted_propers: if not self.ignore_processed_propers and cur_proper.name in processed_propers_names: log.debug(u'Proper already processed. Skipping: {proper_name}', {'proper_name': cur_proper.name}) continue try: cur_proper.parse_result = NameParser().parse(cur_proper.name) except (InvalidNameException, InvalidShowException) as error: log.debug('{error}', {'error': error}) continue if not cur_proper.parse_result.proper_tags: log.info('Skipping non-proper: {name}', {'name': cur_proper.name}) continue log.debug( 'Proper tags for {proper}: {tags}', { 'proper': cur_proper.name, 'tags': cur_proper.parse_result.proper_tags }) if not cur_proper.parse_result.series_name: log.debug('Ignoring invalid show: {name}', {'name': cur_proper.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({ 'name': cur_proper.name, 'date': cur_proper.date }) continue if not cur_proper.parse_result.episode_numbers: log.debug('Ignoring full season instead of episode: {name}', {'name': cur_proper.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({ 'name': cur_proper.name, 'date': cur_proper.date }) continue log.debug( 'Successful match! Matched {original_name} to show {new_name}', { 'original_name': cur_proper.parse_result.original_name, 'new_name': cur_proper.parse_result.show.name }) # Map the indexerid in the db to the show's indexerid cur_proper.indexerid = cur_proper.parse_result.show.indexerid # Map the indexer in the db to the show's indexer cur_proper.indexer = cur_proper.parse_result.show.indexer # Map our Proper instance cur_proper.show = cur_proper.parse_result.show cur_proper.actual_season = cur_proper.parse_result.season_number \ if cur_proper.parse_result.season_number is not None else 1 cur_proper.actual_episodes = cur_proper.parse_result.episode_numbers cur_proper.release_group = cur_proper.parse_result.release_group cur_proper.version = cur_proper.parse_result.version cur_proper.quality = cur_proper.parse_result.quality cur_proper.content = None cur_proper.proper_tags = cur_proper.parse_result.proper_tags # filter release, in this case, it's just a quality gate. As we only send one result. best_result = pick_best_result(cur_proper) if not best_result: log.info('Rejected proper: {name}', {'name': cur_proper.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({ 'name': cur_proper.name, 'date': cur_proper.date }) continue # only get anime proper if it has release group and version if best_result.show.is_anime: if not best_result.release_group and best_result.version == -1: log.info( 'Ignoring proper without release group and version: {name}', {'name': best_result.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({ 'name': cur_proper.name, 'date': cur_proper.date }) continue # check if we have the episode as DOWNLOADED main_db_con = db.DBConnection() sql_results = main_db_con.select( b"SELECT status, release_name FROM tv_episodes WHERE " b"showid = ? AND season = ? AND episode = ? AND status LIKE '%04'", [ best_result.indexerid, best_result.actual_season, best_result.actual_episodes[0] ]) if not sql_results: log.info( "Ignoring proper because this episode doesn't have 'DOWNLOADED' status: {name}", {'name': best_result.name}) continue # only keep the proper if we have already downloaded an episode with the same quality _, old_quality = Quality.split_composite_status( int(sql_results[0][b'status'])) if old_quality != best_result.quality: log.info( 'Ignoring proper because quality is different: {name}', {'name': best_result.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({ 'name': cur_proper.name, 'date': cur_proper.date }) continue # only keep the proper if we have already downloaded an episode with the same codec release_name = sql_results[0][b'release_name'] if release_name: current_codec = NameParser()._parse_string( release_name).video_codec # Ignore proper if codec differs from downloaded release codec if all([ current_codec, best_result.parse_result.video_codec, best_result.parse_result.video_codec != current_codec ]): log.info( 'Ignoring proper because codec is different: {name}', {'name': best_result.name}) if best_result.name not in processed_propers_names: self.processed_propers.append({ 'name': best_result.name, 'date': best_result.date }) continue else: log.debug( "Coudn't find a release name in database. Skipping codec comparison for: {name}", {'name': best_result.name}) # check if we actually want this proper (if it's the right release group and a higher version) if best_result.show.is_anime: main_db_con = db.DBConnection() sql_results = main_db_con.select( b'SELECT release_group, version FROM tv_episodes WHERE showid = ? AND season = ? AND episode = ?', [ best_result.indexerid, best_result.actual_season, best_result.actual_episodes[0] ]) old_version = int(sql_results[0][b'version']) old_release_group = (sql_results[0][b'release_group']) if -1 < old_version < best_result.version: log.info( 'Found new anime version {new} to replace existing version {old}: {name}', { 'old': old_version, 'new': best_result.version, 'name': best_result.name }) else: log.info( 'Ignoring proper with the same or lower version: {name}', {'name': best_result.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({ 'name': best_result.name, 'date': best_result.date }) continue if old_release_group != best_result.release_group: log.info( 'Ignoring proper from release group {new} instead of current group {old}', { 'new': best_result.release_group, 'old': old_release_group }) if best_result.name not in processed_propers_names: self.processed_propers.append({ 'name': best_result.name, 'date': best_result.date }) continue # if the show is in our list and there hasn't been a proper already added for that particular episode # then add it to our list of propers if best_result.indexerid != -1 and ( best_result.indexerid, best_result.actual_season, best_result.actual_episodes[0]) not in map( operator.attrgetter('indexerid', 'actual_season', 'actual_episode'), final_propers): log.info('Found a desired proper: {name}', {'name': best_result.name}) final_propers.append(best_result) if best_result.name not in processed_propers_names: self.processed_propers.append({ 'name': best_result.name, 'date': best_result.date }) return final_propers
def get_provider_cache_results(indexer, show_all_results=None, perform_search=None, show=None, season=None, episode=None, manual_search_type=None, **search_show): """Check all provider cache tables for search results.""" down_cur_quality = 0 show_obj = Show.find(app.showList, int(show)) preferred_words = show_obj.show_words().preferred_words undesired_words = show_obj.show_words().undesired_words ignored_words = show_obj.show_words().ignored_words required_words = show_obj.show_words().required_words main_db_con = db.DBConnection('cache.db') provider_results = { 'last_prov_updates': {}, 'error': {}, 'found_items': [] } original_thread_name = threading.currentThread().name sql_total = [] combined_sql_q = [] combined_sql_params = [] for cur_provider in enabled_providers('manualsearch'): threading.currentThread().name = '{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name) # Let's check if this provider table already exists table_exists = main_db_con.select( b"SELECT name " b"FROM sqlite_master " b"WHERE type='table'" b" AND name=?", [cur_provider.get_id()]) columns = [ i[1] for i in main_db_con.select("PRAGMA table_info('{0}')".format( cur_provider.get_id())) ] if table_exists else [] minseed = int(cur_provider.minseed) if getattr(cur_provider, 'minseed', None) else -1 minleech = int(cur_provider.minleech) if getattr( cur_provider, 'minleech', None) else -1 # TODO: the implicit sqlite rowid is used, should be replaced with an explicit PK column # If table doesn't exist, start a search to create table and new columns seeders, leechers and size required_columns = ['seeders', 'leechers', 'size', 'proper_tags'] if table_exists and all(required_column in columns for required_column in required_columns): # The default sql, that's executed for each providers cache table common_sql = ( b"SELECT rowid, ? AS 'provider_type', ? AS 'provider_image'," b" ? AS 'provider', ? AS 'provider_id', ? 'provider_minseed'," b" ? 'provider_minleech', name, season, episodes, indexerid," b" url, time, proper_tags, quality, release_group, version," b" seeders, leechers, size, time, pubdate " b"FROM '{provider_id}' " b"WHERE indexerid = ? AND quality > 0 ".format( provider_id=cur_provider.get_id())) # Let's start by adding the default parameters, which are used to subsitute the '?'s. add_params = [ cur_provider.provider_type.title(), cur_provider.image_name(), cur_provider.name, cur_provider.get_id(), minseed, minleech, show ] if manual_search_type != 'season': # If were not looking for all results, meaning don't do the filter on season + ep, add sql if not int(show_all_results): # If it's an episode search, pass season and episode. common_sql += " AND season = ? AND episodes LIKE ? " add_params += [season, "%|{0}|%".format(episode)] else: # If were not looking for all results, meaning don't do the filter on season + ep, add sql if not int(show_all_results): list_of_episodes = '{0}{1}'.format( ' episodes LIKE ', ' AND episodes LIKE '.join( ['?' for _ in show_obj.get_all_episodes(season)])) common_sql += " AND season = ? AND (episodes LIKE ? OR {list_of_episodes})".format( list_of_episodes=list_of_episodes) add_params += [season, '||'] # When the episodes field is empty. add_params += [ '%|{episode}|%'.format(episode=ep.episode) for ep in show_obj.get_all_episodes(season) ] # Add the created sql, to lists, that are used down below to perform one big UNIONED query combined_sql_q.append(common_sql) combined_sql_params += add_params # Get the last updated cache items timestamp last_update = main_db_con.select( b"SELECT max(time) AS lastupdate " b"FROM '{provider_id}'".format( provider_id=cur_provider.get_id())) provider_results['last_prov_updates'][ cur_provider.get_id()] = last_update[0][ 'lastupdate'] if last_update[0]['lastupdate'] else 0 # Check if we have the combined sql strings if combined_sql_q: sql_prepend = b"SELECT * FROM (" sql_append = b") ORDER BY CASE quality WHEN '{quality_unknown}' THEN -1 ELSE CAST(quality AS DECIMAL) END DESC, " \ b" proper_tags DESC, seeders DESC".format(quality_unknown=Quality.UNKNOWN) # Add all results sql_total += main_db_con.select( b'{0} {1} {2}'.format(sql_prepend, ' UNION ALL '.join(combined_sql_q), sql_append), combined_sql_params) # Always start a search when no items found in cache if not sql_total or int(perform_search): # retrieve the episode object and fail if we can't get one ep_obj = get_episode(show, season, episode) if isinstance(ep_obj, str): provider_results[ 'error'] = 'Something went wrong when starting the manual search for show {0}, \ and episode: {1}x{2}'.format(show_obj.name, season, episode) # make a queue item for it and put it on the queue ep_queue_item = ForcedSearchQueueItem(ep_obj.series, [ep_obj], bool(int(down_cur_quality)), True, manual_search_type) # pylint: disable=maybe-no-member app.forced_search_queue_scheduler.action.add_item(ep_queue_item) # give the CPU a break and some time to start the queue time.sleep(cpu_presets[app.CPU_PRESET]) else: cached_results = [dict(row) for row in sql_total] for i in cached_results: i['quality_name'] = Quality.split_quality(int(i['quality'])) i['time'] = datetime.fromtimestamp(i['time']) i['release_group'] = i['release_group'] or 'None' i['provider_img_link'] = 'images/providers/' + i[ 'provider_image'] or 'missing.png' i['provider'] = i['provider'] if i[ 'provider_image'] else 'missing provider' i['proper_tags'] = i['proper_tags'].replace('|', ', ') i['pretty_size'] = pretty_file_size( i['size']) if i['size'] > -1 else 'N/A' i['seeders'] = i['seeders'] if i['seeders'] >= 0 else '-' i['leechers'] = i['leechers'] if i['leechers'] >= 0 else '-' i['pubdate'] = sbdatetime.convert_to_setting( parser.parse(i['pubdate'])).strftime( app.DATE_PRESET + ' ' + app.TIME_PRESET) if i['pubdate'] else '-' release_group = i['release_group'] if ignored_words and release_group in ignored_words: i['rg_highlight'] = 'ignored' elif required_words and release_group in required_words: i['rg_highlight'] = 'required' elif preferred_words and release_group in preferred_words: i['rg_highlight'] = 'preferred' elif undesired_words and release_group in undesired_words: i['rg_highlight'] = 'undesired' else: i['rg_highlight'] = '' if contains_at_least_one_word(i['name'], required_words): i['name_highlight'] = 'required' elif contains_at_least_one_word( i['name'], ignored_words) or not filter_bad_releases( i['name'], parse=False): i['name_highlight'] = 'ignored' elif contains_at_least_one_word(i['name'], undesired_words): i['name_highlight'] = 'undesired' elif contains_at_least_one_word(i['name'], preferred_words): i['name_highlight'] = 'preferred' else: i['name_highlight'] = '' i['seed_highlight'] = 'ignored' if i.get( 'provider_minseed') > i.get('seeders', -1) >= 0 else '' i['leech_highlight'] = 'ignored' if i.get( 'provider_minleech') > i.get('leechers', -1) >= 0 else '' provider_results['found_items'] = cached_results # Remove provider from thread name before return results threading.currentThread().name = original_thread_name # Sanitize the last_prov_updates key provider_results['last_prov_updates'] = json.dumps( provider_results['last_prov_updates']) return provider_results
def search_for_needed_episodes(scheduler_start_time, force=False): """Search providers for needed episodes. :param force: run the search even if no episodes are needed :param scheduler_start_time: timestamp of the start of the search scheduler :return: list of found episodes """ show_list = app.showList from_date = datetime.date.fromordinal(1) episodes = [] for cur_show in show_list: if cur_show.paused: log.debug( u'Not checking for needed episodes of {0} because the show is paused', cur_show.name, ) continue episodes.extend(wanted_episodes(cur_show, from_date)) if not episodes and not force: # nothing wanted so early out, ie: avoid whatever arbitrarily # complex thing a provider cache update entails, for example, # reading rss feeds return [] providers = enabled_providers(u'daily') if not providers: log.warning( u'No NZB/Torrent providers found or enabled in the application config for daily searches.' u' Please check your settings') return [] original_thread_name = threading.currentThread().name log.info(u'Using daily search providers') for cur_provider in providers: threading.currentThread().name = u'{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name) cur_provider.cache.update_cache(scheduler_start_time) single_results = {} multi_results = [] for cur_provider in providers: threading.currentThread().name = u'{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name) try: found_results = cur_provider.cache.find_needed_episodes(episodes) except AuthException as error: log.error(u'Authentication error: {0}', ex(error)) continue # pick a single result for each episode, respecting existing results for episode_no, results in iteritems(found_results): if results[0].series.paused: log.debug(u'Skipping {0} because the show is paused.', results[0].series.name) continue # if all results were rejected move on to the next episode wanted_results = filter_results(results) if not wanted_results: log.debug(u'All found results for {0} were rejected.', results[0].series.name) continue best_result = pick_result(wanted_results) # Skip the result if search delay is enabled for the provider. if delay_search(best_result): continue if episode_no in (SEASON_RESULT, MULTI_EP_RESULT): multi_results.append(best_result) else: # if it's already in the list (from another provider) and # the newly found quality is no better then skip it if episode_no in single_results: allowed_qualities, preferred_qualities = results[ 0].series.current_qualities if not Quality.is_higher_quality( single_results[episode_no].quality, best_result.quality, allowed_qualities, preferred_qualities): continue single_results[episode_no] = best_result threading.currentThread().name = original_thread_name return combine_results(multi_results, list(itervalues(single_results)))
def search_providers(series_obj, episodes, forced_search=False, down_cur_quality=False, manual_search=False, manual_search_type=u'episode'): """ Walk providers for information on shows. :param series_obj: Show we are looking for :param episodes: List, episodes we hope to find :param forced_search: Boolean, is this a forced search? :param down_cur_quality: Boolean, should we re-download currently available quality file :param manual_search: Boolean, should we choose what to download? :param manual_search_type: Episode or Season search :return: results for search """ found_results = {} manual_search_results = [] multi_results = [] single_results = [] # build name cache for show name_cache.build_name_cache(series_obj) original_thread_name = threading.currentThread().name if manual_search: log.info(u'Using manual search providers') providers = enabled_providers(u'manualsearch') else: log.info(u'Using backlog search providers') providers = enabled_providers(u'backlog') if not providers: log.warning(u'No NZB/Torrent providers found or enabled in the application config for {0} searches.' u' Please check your settings', 'manual' if manual_search else 'backlog') threading.currentThread().name = original_thread_name for cur_provider in providers: threading.currentThread().name = '{original_thread_name} :: [{provider}]'.format( original_thread_name=original_thread_name, provider=cur_provider.name ) if cur_provider.anime_only and not series_obj.is_anime: log.debug(u'{0} is not an anime, skipping', series_obj.name) continue found_results[cur_provider.name] = {} search_count = 0 search_mode = cur_provider.search_mode # Always search for episode when manually searching when in sponly if search_mode == u'sponly' and (forced_search or manual_search): search_mode = u'eponly' if manual_search and manual_search_type == u'season': search_mode = u'sponly' while True: search_count += 1 if search_mode == u'eponly': log.info(u'Performing episode search for {0}', series_obj.name) else: log.info(u'Performing season pack search for {0}', series_obj.name) try: search_results = [] cache_search_results = [] cache_multi = [] cache_single = [] if not manual_search: cache_search_results = cur_provider.search_results_in_cache(episodes) if cache_search_results: # From our provider multi_episode and single_episode results, collect candidates. cache_found_results = list_results_for_provider(cache_search_results, found_results, cur_provider) # We're passing the empty lists, because we don't want to include previous candidates cache_multi, cache_single = collect_candidates(cache_found_results, cur_provider, [], [], series_obj, down_cur_quality) # For now we only search if we didn't get any results back from cache, # but we might wanna check if there was something useful in cache. if not (cache_multi or cache_single): log.debug(u'Could not find any candidates in cache, searching provider.') search_results = cur_provider.find_search_results(series_obj, episodes, search_mode, forced_search, down_cur_quality, manual_search, manual_search_type) # Update the list found_results found_results = list_results_for_provider(search_results, found_results, cur_provider) else: found_results = cache_found_results except AuthException as error: log.error(u'Authentication error: {0!r}', error) break if search_results or cache_search_results: break elif not cur_provider.search_fallback or search_count == 2: break # Don't fallback when doing manual season search if manual_search_type == u'season': break if search_mode == u'sponly': log.debug(u'Fallback episode search initiated') search_mode = u'eponly' else: log.debug(u'Fallback season pack search initiated') search_mode = u'sponly' # skip to next provider if we have no results to process if not found_results[cur_provider.name]: continue # Update the cache if a manual search is being run if manual_search: # Let's create a list with episodes that we where looking for if manual_search_type == u'season': # If season search type, we only want season packs searched_episode_list = [SEASON_RESULT] else: searched_episode_list = [episode_obj.episode for episode_obj in episodes] + [MULTI_EP_RESULT] for searched_episode in searched_episode_list: if (searched_episode in search_results and cur_provider.cache.update_cache_manual_search(search_results[searched_episode])): # If we have at least a result from one provider, it's good enough to be marked as result manual_search_results.append(True) # Continue because we don't want to pick best results as we are running a manual search by user continue multi_results, single_results = collect_candidates(found_results, cur_provider, multi_results, single_results, series_obj, down_cur_quality) # Remove provider from thread name before return results threading.currentThread().name = original_thread_name if manual_search: # If results in manual search return True, else False return any(manual_search_results) else: return combine_results(multi_results, single_results)
def search_for_needed_episodes(scheduler_start_time, force=False): """Search providers for needed episodes. :param force: run the search even if no episodes are needed :param scheduler_start_time: timestamp of the start of the search scheduler :return: list of found episodes """ show_list = app.showList from_date = datetime.date.fromordinal(1) episodes = [] for cur_show in show_list: if cur_show.paused: log.debug( u'Not checking for needed episodes of {0} because the show is paused', cur_show.name, ) continue episodes.extend(wanted_episodes(cur_show, from_date)) if not episodes and not force: # nothing wanted so early out, ie: avoid whatever arbitrarily # complex thing a provider cache update entails, for example, # reading rss feeds return [] providers = enabled_providers(u'daily') if not providers: log.warning( u'No NZB/Torrent providers found or enabled in the application config for daily searches.' u' Please check your settings' ) return [] original_thread_name = threading.currentThread().name log.info(u'Using daily search providers') for cur_provider in providers: threading.currentThread().name = u'{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name ) cur_provider.cache.update_cache(scheduler_start_time) single_results = {} multi_results = [] for cur_provider in providers: threading.currentThread().name = u'{thread} :: [{provider}]'.format( thread=original_thread_name, provider=cur_provider.name ) try: found_results = cur_provider.cache.find_needed_episodes(episodes) except AuthException as error: log.error(u'Authentication error: {0}', ex(error)) continue # pick a single result for each episode, respecting existing results for episode_no, results in iteritems(found_results): if results[0].series.paused: log.debug(u'Skipping {0} because the show is paused.', results[0].series.name) continue # if all results were rejected move on to the next episode wanted_results = filter_results(results) if not wanted_results: log.debug(u'All found results for {0} were rejected.', results[0].series.name) continue best_result = pick_result(wanted_results) # Skip the result if search delay is enabled for the provider. if delay_search(best_result): continue if episode_no in (SEASON_RESULT, MULTI_EP_RESULT): multi_results.append(best_result) else: # if it's already in the list (from another provider) and # the newly found quality is no better then skip it if episode_no in single_results: allowed_qualities, preferred_qualities = results[0].series.current_qualities if not Quality.is_higher_quality(single_results[episode_no].quality, best_result.quality, allowed_qualities, preferred_qualities): continue single_results[episode_no] = best_result threading.currentThread().name = original_thread_name return combine_results(multi_results, list(itervalues(single_results)))
def search_providers(series_obj, episodes, forced_search=False, down_cur_quality=False, manual_search=False, manual_search_type=u'episode'): """ Walk providers for information on shows. :param series_obj: Show we are looking for :param episodes: List, episodes we hope to find :param forced_search: Boolean, is this a forced search? :param down_cur_quality: Boolean, should we re-download currently available quality file :param manual_search: Boolean, should we choose what to download? :param manual_search_type: Episode or Season search :return: results for search """ found_results = {} manual_search_results = [] multi_results = [] single_results = [] # build name cache for show name_cache.build_name_cache(series_obj) original_thread_name = threading.currentThread().name if manual_search: log.info(u'Using manual search providers') providers = enabled_providers(u'manualsearch') else: log.info(u'Using backlog search providers') providers = enabled_providers(u'backlog') if not providers: log.warning( u'No NZB/Torrent providers found or enabled in the application config for {0} searches.' u' Please check your settings', 'manual' if manual_search else 'backlog') threading.currentThread().name = original_thread_name for cur_provider in providers: threading.currentThread( ).name = original_thread_name + u' :: [' + cur_provider.name + u']' if cur_provider.anime_only and not series_obj.is_anime: log.debug(u'{0} is not an anime, skipping', series_obj.name) continue found_results[cur_provider.name] = {} search_count = 0 search_mode = cur_provider.search_mode # Always search for episode when manually searching when in sponly if search_mode == u'sponly' and (forced_search or manual_search): search_mode = u'eponly' if manual_search and manual_search_type == u'season': search_mode = u'sponly' while True: search_count += 1 if search_mode == u'eponly': log.info(u'Performing episode search for {0}', series_obj.name) else: log.info(u'Performing season pack search for {0}', series_obj.name) try: search_results = cur_provider.find_search_results( series_obj, episodes, search_mode, forced_search, down_cur_quality, manual_search, manual_search_type) except AuthException as error: log.error(u'Authentication error: {0}', ex(error)) break if search_results: # make a list of all the results for this provider for cur_ep in search_results: if cur_ep in found_results[cur_provider.name]: found_results[cur_provider. name][cur_ep] += search_results[cur_ep] else: found_results[ cur_provider.name][cur_ep] = search_results[cur_ep] # Sort the list by seeders if possible if cur_provider.provider_type == u'torrent' or getattr( cur_provider, u'torznab', None): found_results[cur_provider.name][cur_ep].sort( key=lambda d: int(d.seeders), reverse=True) break elif not cur_provider.search_fallback or search_count == 2: break # Don't fallback when doing manual season search if manual_search_type == u'season': break if search_mode == u'sponly': log.debug(u'Fallback episode search initiated') search_mode = u'eponly' else: log.debug(u'Fallback season pack search initiated') search_mode = u'sponly' # skip to next provider if we have no results to process if not found_results[cur_provider.name]: continue # Update the cache if a manual search is being run if manual_search: # Let's create a list with episodes that we where looking for if manual_search_type == u'season': # If season search type, we only want season packs searched_episode_list = [SEASON_RESULT] else: searched_episode_list = [ episode_obj.episode for episode_obj in episodes ] + [MULTI_EP_RESULT] for searched_episode in searched_episode_list: if (searched_episode in search_results and cur_provider.cache.update_cache_manual_search( search_results[searched_episode])): # If we have at least a result from one provider, it's good enough to be marked as result manual_search_results.append(True) # Continue because we don't want to pick best results as we are running a manual search by user continue # Collect candidates for multi-episode or season results candidates = (candidate for result, candidate in iteritems( found_results[cur_provider.name]) if result in (SEASON_RESULT, MULTI_EP_RESULT)) candidates = list(itertools.chain(*candidates)) if candidates: multi_results += collect_multi_candidates(candidates, series_obj, episodes, down_cur_quality) # Collect candidates for single-episode results single_results = collect_single_candidates( found_results[cur_provider.name], single_results) # Remove provider from thread name before return results threading.currentThread().name = original_thread_name if manual_search: # If results in manual search return True, else False return any(manual_search_results) else: return combine_results(multi_results, single_results)
def _get_proper_results(self): # pylint: disable=too-many-locals, too-many-branches, too-many-statements """Retrieve a list of recently aired episodes, and search for these episodes in the different providers.""" propers = {} # For each provider get the list of propers original_thread_name = threading.currentThread().name providers = enabled_providers('backlog') search_date = datetime.datetime.today() - datetime.timedelta(days=app.PROPERS_SEARCH_DAYS) main_db_con = db.DBConnection() if not app.POSTPONE_IF_NO_SUBS: # Get the recently aired (last 2 days) shows from DB recently_aired = main_db_con.select( 'SELECT indexer, showid, season, episode, status, airdate' ' FROM tv_episodes' ' WHERE airdate >= ?' ' AND status = ?', [search_date.toordinal(), DOWNLOADED] ) else: # Get recently subtitled episodes (last 2 days) from DB # Episode status becomes downloaded only after found subtitles last_subtitled = search_date.strftime(History.date_format) recently_aired = main_db_con.select('SELECT indexer_id AS indexer, showid, season, episode FROM history ' 'WHERE date >= ? AND action = ?', [last_subtitled, SUBTITLED]) if not recently_aired: log.info('No recently aired new episodes, nothing to search for') return [] # Loop through the providers, and search for releases for cur_provider in providers: threading.currentThread().name = '{thread} :: [{provider}]'.format(thread=original_thread_name, provider=cur_provider.name) log.info('Searching for any new PROPER releases from {provider}', {'provider': cur_provider.name}) try: cur_propers = cur_provider.find_propers(recently_aired) except AuthException as e: log.debug('Authentication error: {error}', {'error': ex(e)}) continue # if they haven't been added by a different provider than add the proper to the list for proper in cur_propers: name = self._sanitize_name(proper.name) if name not in propers: log.debug('Found new possible proper result: {name}', {'name': proper.name}) propers[name] = proper threading.currentThread().name = original_thread_name # take the list of unique propers and get it sorted by sorted_propers = sorted(list(itervalues(propers)), key=operator.attrgetter('date'), reverse=True) final_propers = [] # Keep only items from last PROPER_SEARCH_DAYS setting in processed propers: latest_proper = datetime.datetime.now() - datetime.timedelta(days=app.PROPERS_SEARCH_DAYS) self.processed_propers = [p for p in self.processed_propers if p.get('date') >= latest_proper] # Get proper names from processed propers processed_propers_names = [proper.get('name') for proper in self.processed_propers if proper.get('name')] for cur_proper in sorted_propers: if not self.ignore_processed_propers and cur_proper.name in processed_propers_names: log.debug(u'Proper already processed. Skipping: {proper_name}', {'proper_name': cur_proper.name}) continue try: cur_proper.parse_result = NameParser().parse(cur_proper.name) except (InvalidNameException, InvalidShowException) as error: log.debug('{error}', {'error': error}) continue if not cur_proper.parse_result.proper_tags: log.info('Skipping non-proper: {name}', {'name': cur_proper.name}) continue if not cur_proper.series.episodes.get(cur_proper.parse_result.season_number) or \ any([ep for ep in cur_proper.parse_result.episode_numbers if not cur_proper.series.episodes[cur_proper.parse_result.season_number].get(ep)]): log.info('Skipping proper for wrong season/episode: {name}', {'name': cur_proper.name}) continue log.debug('Proper tags for {proper}: {tags}', { 'proper': cur_proper.name, 'tags': cur_proper.parse_result.proper_tags }) if not cur_proper.parse_result.series_name: log.debug('Ignoring invalid show: {name}', {'name': cur_proper.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date}) continue if not cur_proper.parse_result.episode_numbers: log.debug('Ignoring full season instead of episode: {name}', {'name': cur_proper.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date}) continue log.debug('Successful match! Matched {original_name} to show {new_name}', {'original_name': cur_proper.parse_result.original_name, 'new_name': cur_proper.parse_result.series.name }) # Map the indexerid in the db to the show's indexerid cur_proper.indexerid = cur_proper.parse_result.series.indexerid # Map the indexer in the db to the show's indexer cur_proper.indexer = cur_proper.parse_result.series.indexer # Map our Proper instance cur_proper.series = cur_proper.parse_result.series cur_proper.actual_season = cur_proper.parse_result.season_number \ if cur_proper.parse_result.season_number is not None else 1 cur_proper.actual_episodes = cur_proper.parse_result.episode_numbers cur_proper.release_group = cur_proper.parse_result.release_group cur_proper.version = cur_proper.parse_result.version cur_proper.quality = cur_proper.parse_result.quality cur_proper.content = None cur_proper.proper_tags = cur_proper.parse_result.proper_tags # filter release, in this case, it's just a quality gate. As we only send one result. wanted_results = filter_results(cur_proper) best_result = pick_result(wanted_results) if not best_result: log.info('Rejected proper: {name}', {'name': cur_proper.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date}) continue # only get anime proper if it has release group and version if best_result.series.is_anime: if not best_result.release_group and best_result.version == -1: log.info('Ignoring proper without release group and version: {name}', {'name': best_result.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date}) continue # check if we have the episode as DOWNLOADED main_db_con = db.DBConnection() sql_results = main_db_con.select('SELECT quality, release_name ' 'FROM tv_episodes WHERE indexer = ? ' 'AND showid = ? AND season = ? ' 'AND episode = ? AND status = ?', [best_result.indexer, best_result.series.indexerid, best_result.actual_season, best_result.actual_episodes[0], DOWNLOADED]) if not sql_results: log.info("Ignoring proper because this episode doesn't have 'DOWNLOADED' status: {name}", { 'name': best_result.name }) continue # only keep the proper if we have already downloaded an episode with the same quality old_quality = int(sql_results[0]['quality']) if old_quality != best_result.quality: log.info('Ignoring proper because quality is different: {name}', {'name': best_result.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({'name': cur_proper.name, 'date': cur_proper.date}) continue # only keep the proper if we have already downloaded an episode with the same codec release_name = sql_results[0]['release_name'] if release_name: release_name_guess = NameParser()._parse_string(release_name) current_codec = release_name_guess.video_codec # Ignore proper if codec differs from downloaded release codec if all([current_codec, best_result.parse_result.video_codec, best_result.parse_result.video_codec != current_codec]): log.info('Ignoring proper because codec is different: {name}', {'name': best_result.name}) if best_result.name not in processed_propers_names: self.processed_propers.append({'name': best_result.name, 'date': best_result.date}) continue streaming_service = release_name_guess.guess.get(u'streaming_service') # Ignore proper if streaming service differs from downloaded release streaming service if best_result.parse_result.guess.get(u'streaming_service') != streaming_service: log.info('Ignoring proper because streaming service is different: {name}', {'name': best_result.name}) if best_result.name not in processed_propers_names: self.processed_propers.append({'name': best_result.name, 'date': best_result.date}) continue else: log.debug("Coudn't find a release name in database. Skipping codec comparison for: {name}", { 'name': best_result.name }) # check if we actually want this proper (if it's the right release group and a higher version) if best_result.series.is_anime: main_db_con = db.DBConnection() sql_results = main_db_con.select( 'SELECT release_group, version ' 'FROM tv_episodes WHERE indexer = ? AND showid = ? ' 'AND season = ? AND episode = ?', [best_result.indexer, best_result.series.indexerid, best_result.actual_season, best_result.actual_episodes[0]]) old_version = int(sql_results[0]['version']) old_release_group = (sql_results[0]['release_group']) if -1 < old_version < best_result.version: log.info('Found new anime version {new} to replace existing version {old}: {name}', {'old': old_version, 'new': best_result.version, 'name': best_result.name }) else: log.info('Ignoring proper with the same or lower version: {name}', {'name': best_result.name}) if cur_proper.name not in processed_propers_names: self.processed_propers.append({'name': best_result.name, 'date': best_result.date}) continue if old_release_group != best_result.release_group: log.info('Ignoring proper from release group {new} instead of current group {old}', {'new': best_result.release_group, 'old': old_release_group}) if best_result.name not in processed_propers_names: self.processed_propers.append({'name': best_result.name, 'date': best_result.date}) continue # if the show is in our list and there hasn't been a proper already added for that particular episode # then add it to our list of propers if best_result.indexerid != -1 and ( best_result.indexerid, best_result.actual_season, best_result.actual_episodes ) not in list(map(operator.attrgetter('indexerid', 'actual_season', 'actual_episodes'), final_propers)): log.info('Found a desired proper: {name}', {'name': best_result.name}) final_propers.append(best_result) if best_result.name not in processed_propers_names: self.processed_propers.append({'name': best_result.name, 'date': best_result.date}) return final_propers
def get_provider_cache_results(series_obj, show_all_results=None, perform_search=None, season=None, episode=None, manual_search_type=None, **search_show): """Check all provider cache tables for search results.""" down_cur_quality = 0 preferred_words = series_obj.show_words().preferred_words undesired_words = series_obj.show_words().undesired_words ignored_words = series_obj.show_words().ignored_words required_words = series_obj.show_words().required_words main_db_con = db.DBConnection('cache.db') provider_results = {'last_prov_updates': {}, 'error': {}, 'found_items': []} original_thread_name = threading.currentThread().name cached_results_total = [] combined_sql_q = [] combined_sql_params = [] for cur_provider in enabled_providers('manualsearch'): threading.currentThread().name = '{thread} :: [{provider}]'.format(thread=original_thread_name, provider=cur_provider.name) # Let's check if this provider table already exists table_exists = main_db_con.select( 'SELECT name ' 'FROM sqlite_master ' "WHERE type='table'" ' AND name=?', [cur_provider.get_id()] ) columns = [] if table_exists: table_columns = main_db_con.select("PRAGMA table_info('{0}')".format(cur_provider.get_id())) columns = [table_column['name'] for table_column in table_columns] minseed = int(cur_provider.minseed) if getattr(cur_provider, 'minseed', None) else -1 minleech = int(cur_provider.minleech) if getattr(cur_provider, 'minleech', None) else -1 # TODO: the implicit sqlite rowid is used, should be replaced with an explicit PK column # If table doesn't exist, start a search to create table and new columns seeders, leechers and size required_columns = ['indexer', 'indexerid', 'seeders', 'leechers', 'size', 'proper_tags', 'date_added'] if table_exists and all(required_column in columns for required_column in required_columns): # The default sql, that's executed for each providers cache table common_sql = ( "SELECT rowid, ? AS 'provider_type', ? AS 'provider_image'," " ? AS 'provider', ? AS 'provider_id', ? 'provider_minseed'," " ? 'provider_minleech', name, season, episodes, indexer, indexerid," ' url, proper_tags, quality, release_group, version,' ' seeders, leechers, size, time, pubdate, date_added ' "FROM '{provider_id}' " 'WHERE indexer = ? AND indexerid = ? AND quality > 0 '.format( provider_id=cur_provider.get_id() ) ) # Let's start by adding the default parameters, which are used to substitute the '?'s. add_params = [cur_provider.provider_type.title(), cur_provider.image_name(), cur_provider.name, cur_provider.get_id(), minseed, minleech, series_obj.indexer, series_obj.series_id] if manual_search_type != 'season': # If were not looking for all results, meaning don't do the filter on season + ep, add sql if not int(show_all_results): # If it's an episode search, pass season and episode. common_sql += ' AND season = ? AND episodes LIKE ? ' add_params += [season, '%|{0}|%'.format(episode)] else: # If were not looking for all results, meaning don't do the filter on season + ep, add sql if not int(show_all_results): list_of_episodes = '{0}{1}'.format(' episodes LIKE ', ' AND episodes LIKE '.join( ['?' for _ in series_obj.get_all_episodes(season)] )) common_sql += ' AND season = ? AND (episodes LIKE ? OR {list_of_episodes})'.format( list_of_episodes=list_of_episodes ) add_params += [season, '||'] # When the episodes field is empty. add_params += ['%|{episode}|%'.format(episode=ep.episode) for ep in series_obj.get_all_episodes(season)] # Add the created sql, to lists, that are used down below to perform one big UNIONED query combined_sql_q.append(common_sql) combined_sql_params += add_params # Get the last updated cache items timestamp last_update = main_db_con.select('SELECT max(time) AS lastupdate ' "FROM '{provider_id}'".format(provider_id=cur_provider.get_id())) provider_results['last_prov_updates'][cur_provider.get_id()] = last_update[0]['lastupdate'] if last_update[0]['lastupdate'] else 0 # Check if we have the combined sql strings if combined_sql_q: sql_prepend = 'SELECT * FROM (' sql_append = ') ORDER BY quality DESC, proper_tags DESC, seeders DESC' # Add all results cached_results_total += main_db_con.select('{0} {1} {2}'. format(sql_prepend, ' UNION ALL '.join(combined_sql_q), sql_append), combined_sql_params) # Always start a search when no items found in cache if not cached_results_total or int(perform_search): # retrieve the episode object and fail if we can't get one ep_obj = series_obj.get_episode(season, episode) if isinstance(ep_obj, str): provider_results['error'] = 'Something went wrong when starting the manual search for show {0}, \ and episode: {1}x{2}'.format(series_obj.name, season, episode) # make a queue item for it and put it on the queue ep_queue_item = ForcedSearchQueueItem(ep_obj.series, [ep_obj], bool(int(down_cur_quality)), True, manual_search_type) # pylint: disable=maybe-no-member app.forced_search_queue_scheduler.action.add_item(ep_queue_item) # give the CPU a break and some time to start the queue time.sleep(cpu_presets[app.CPU_PRESET]) else: for i in cached_results_total: threading.currentThread().name = '{thread} :: [{provider}]'.format( thread=original_thread_name, provider=i['provider']) i['quality_name'] = Quality.split_quality(int(i['quality'])) i['time'] = datetime.fromtimestamp(i['time']) i['release_group'] = i['release_group'] or 'None' i['provider_img_link'] = 'images/providers/' + i['provider_image'] or 'missing.png' i['provider'] = i['provider'] if i['provider_image'] else 'missing provider' i['proper_tags'] = i['proper_tags'].replace('|', ', ') i['pretty_size'] = pretty_file_size(i['size']) if i['size'] > -1 else 'N/A' i['seeders'] = i['seeders'] if i['seeders'] >= 0 else '-' i['leechers'] = i['leechers'] if i['leechers'] >= 0 else '-' i['pubdate'] = parser.parse(i['pubdate']).astimezone(app_timezone) if i['pubdate'] else '' i['date_added'] = datetime.fromtimestamp(float(i['date_added']), tz=app_timezone) if i['date_added'] else '' release_group = i['release_group'] if ignored_words and release_group in ignored_words: i['rg_highlight'] = 'ignored' elif required_words and release_group in required_words: i['rg_highlight'] = 'required' elif preferred_words and release_group in preferred_words: i['rg_highlight'] = 'preferred' elif undesired_words and release_group in undesired_words: i['rg_highlight'] = 'undesired' else: i['rg_highlight'] = '' if contains_at_least_one_word(i['name'], required_words): i['name_highlight'] = 'required' elif contains_at_least_one_word(i['name'], ignored_words) or not filter_bad_releases(i['name'], parse=False): i['name_highlight'] = 'ignored' elif contains_at_least_one_word(i['name'], undesired_words): i['name_highlight'] = 'undesired' elif contains_at_least_one_word(i['name'], preferred_words): i['name_highlight'] = 'preferred' else: i['name_highlight'] = '' i['seed_highlight'] = 'ignored' if i['seeders'] == '-' or i['provider_minseed'] <= i['seeders']: i['seed_highlight'] = '' i['leech_highlight'] = 'ignored' if i['leechers'] == '-' or i['provider_minleech'] <= i['leechers']: i['leech_highlight'] = '' provider_results['found_items'] = cached_results_total # Remove provider from thread name before return results threading.currentThread().name = original_thread_name # Sanitize the last_prov_updates key provider_results['last_prov_updates'] = json.dumps(provider_results['last_prov_updates']) return provider_results