def pick_best_result(results, show, quality_list=None): logger.log( u'Picking the best result out of %s' % [x.name for x in results], logger.DEBUG) # find the best result for the current episode best_result = None for cur_result in results: logger.log( u'Quality is %s for [%s]' % (Quality.qualityStrings[cur_result.quality], cur_result.name)) if show.is_anime and not show.release_groups.is_valid(cur_result): continue if quality_list and cur_result.quality not in quality_list: logger.log(u'Rejecting unwanted quality [%s]' % cur_result.name, logger.DEBUG) continue if not pass_show_wordlist_checks(cur_result.name, show): continue cur_size = getattr(cur_result, 'size', None) if sickbeard.USE_FAILED_DOWNLOADS and None is not cur_size and failed_history.has_failed( cur_result.name, cur_size, cur_result.provider.name): logger.log(u'Rejecting previously failed [%s]' % cur_result.name) continue if not best_result or best_result.quality < cur_result.quality != Quality.UNKNOWN: best_result = cur_result elif best_result.quality == cur_result.quality: if cur_result.properlevel > best_result.properlevel and \ (not cur_result.is_repack or cur_result.release_group == best_result.release_group): best_result = cur_result elif cur_result.properlevel == best_result.properlevel: if 'xvid' in best_result.name.lower( ) and 'x264' in cur_result.name.lower(): logger.log(u'Preferring (x264 over xvid) [%s]' % cur_result.name) best_result = cur_result elif 'internal' in best_result.name.lower( ) and 'internal' not in cur_result.name.lower(): best_result = cur_result if best_result: logger.log(u'Picked as the best [%s]' % best_result.name, logger.DEBUG) else: logger.log(u'No result picked.', logger.DEBUG) return best_result
def search_providers(show, episodes, manual_search=False, torrent_only=False, try_other_searches=False, old_status=None, scheduled=False): found_results = {} final_results = [] search_done = False orig_thread_name = threading.currentThread().name use_quality_list = None if any([episodes]): old_status = old_status or failed_history.find_old_status( episodes[0]) or episodes[0].status if old_status: status, quality = Quality.splitCompositeStatus(old_status) use_quality_list = (status not in (common.WANTED, common.FAILED, common.UNAIRED, common.SKIPPED, common.IGNORED, common.UNKNOWN)) provider_list = [ x for x in sickbeard.providers.sortedProviderList() if x.is_active() and x.enable_backlog and (not torrent_only or x.providerType == GenericProvider.TORRENT) and ( not scheduled or x.enable_scheduled_backlog) ] for cur_provider in provider_list: if cur_provider.anime_only and not show.is_anime: logger.log(u'%s is not an anime, skipping' % show.name, logger.DEBUG) continue threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name) provider_id = cur_provider.get_id() found_results[provider_id] = {} search_count = 0 search_mode = cur_provider.search_mode while True: search_count += 1 if 'eponly' == search_mode: logger.log(u'Performing episode search for %s' % show.name) else: logger.log(u'Performing season pack search for %s' % show.name) try: cur_provider.cache._clearCache() search_results = cur_provider.find_search_results( show, episodes, search_mode, manual_search, try_other_searches=try_other_searches) if any(search_results): logger.log(', '.join([ '%s %s candidate%s' % (len(v), (('multiep', 'season')[SEASON_RESULT == k], 'episode')['ep' in search_mode], helpers.maybe_plural(len(v))) for (k, v) in search_results.iteritems() ])) except exceptions.AuthException as e: logger.log(u'Authentication error: %s' % ex(e), logger.ERROR) break except Exception as e: logger.log( u'Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR) logger.log(traceback.format_exc(), logger.ERROR) break finally: threading.currentThread().name = orig_thread_name search_done = True if len(search_results): # make a list of all the results for this provider for cur_ep in search_results: # skip non-tv crap search_results[cur_ep] = filter( lambda ep_item: show_name_helpers.pass_wordlist_checks( ep_item.name, parse=False, indexer_lookup=False) and ep_item.show == show, search_results[cur_ep]) if cur_ep in found_results: found_results[provider_id][cur_ep] += search_results[ cur_ep] else: found_results[provider_id][cur_ep] = search_results[ cur_ep] break elif not cur_provider.search_fallback or search_count == 2: break search_mode = '%sonly' % ('ep', 'sp')['ep' in search_mode] logger.log(u'Falling back to %s search ...' % ('season pack', 'episode')['ep' in search_mode]) # skip to next provider if we have no results to process if not len(found_results[provider_id]): continue any_qualities, best_qualities = Quality.splitQuality(show.quality) # pick the best season NZB best_season_result = None if SEASON_RESULT in found_results[provider_id]: best_season_result = pick_best_result( found_results[provider_id][SEASON_RESULT], show, any_qualities + best_qualities) highest_quality_overall = 0 for cur_episode in found_results[provider_id]: for cur_result in found_results[provider_id][cur_episode]: if Quality.UNKNOWN != cur_result.quality and highest_quality_overall < cur_result.quality: highest_quality_overall = cur_result.quality logger.log( u'%s is the highest quality of any match' % Quality.qualityStrings[highest_quality_overall], logger.DEBUG) # see if every episode is wanted if best_season_result: # get the quality of the season nzb season_qual = best_season_result.quality logger.log( u'%s is the quality of the season %s' % (Quality.qualityStrings[season_qual], best_season_result.provider.providerType), logger.DEBUG) my_db = db.DBConnection() sql = 'SELECT episode FROM tv_episodes WHERE showid = %s AND (season IN (%s))' %\ (show.indexerid, ','.join([str(x.season) for x in episodes])) ep_nums = [int(x['episode']) for x in my_db.select(sql)] logger.log(u'Executed query: [%s]' % sql) logger.log(u'Episode list: %s' % ep_nums, logger.DEBUG) all_wanted = True any_wanted = False for ep_num in ep_nums: for season in set([x.season for x in episodes]): if not show.wantEpisode(season, ep_num, season_qual): all_wanted = False else: any_wanted = True # if we need every ep in the season and there's nothing better then just download this and # be done with it (unless single episodes are preferred) if all_wanted and highest_quality_overall == best_season_result.quality: logger.log( u'Every episode in this season is needed, downloading the whole %s %s' % (best_season_result.provider.providerType, best_season_result.name)) ep_objs = [] for ep_num in ep_nums: for season in set([x.season for x in episodes]): ep_objs.append(show.getEpisode(season, ep_num)) best_season_result.episodes = ep_objs return [best_season_result] elif not any_wanted: logger.log( u'No episodes from this season are wanted at this quality, ignoring the result of ' + best_season_result.name, logger.DEBUG) else: if GenericProvider.NZB == best_season_result.provider.providerType: logger.log( u'Breaking apart the NZB and adding the individual ones to our results', logger.DEBUG) # if not, break it apart and add them as the lowest priority results individual_results = nzbSplitter.splitResult( best_season_result) individual_results = filter( lambda r: show_name_helpers.pass_wordlist_checks( r.name, parse=False, indexer_lookup=False) and r. show == show, individual_results) for cur_result in individual_results: if 1 == len(cur_result.episodes): ep_num = cur_result.episodes[0].episode elif 1 < len(cur_result.episodes): ep_num = MULTI_EP_RESULT if ep_num in found_results[provider_id]: found_results[provider_id][ep_num].append( cur_result) else: found_results[provider_id][ep_num] = [cur_result] # If this is a torrent all we can do is leech the entire torrent, # user will have to select which eps not do download in his torrent client else: # Season result from Torrent Provider must be a full-season torrent, creating multi-ep result for it logger.log( u'Adding multi episode result for full season torrent. In your torrent client, set ' + u'the episodes that you do not want to "don\'t download"' ) ep_objs = [] for ep_num in ep_nums: for season in set([x.season for x in episodes]): ep_objs.append(show.getEpisode(season, ep_num)) best_season_result.episodes = ep_objs ep_num = MULTI_EP_RESULT if ep_num in found_results[provider_id]: found_results[provider_id][ep_num].append( best_season_result) else: found_results[provider_id][ep_num] = [ best_season_result ] # go through multi-ep results and see if we really want them or not, get rid of the rest multi_results = {} if MULTI_EP_RESULT in found_results[provider_id]: for multi_result in found_results[provider_id][MULTI_EP_RESULT]: logger.log( u'Checking usefulness of multi episode result [%s]' % multi_result.name, logger.DEBUG) if sickbeard.USE_FAILED_DOWNLOADS and failed_history.has_failed( multi_result.name, multi_result.size, multi_result.provider.name): logger.log( u'Rejecting previously failed multi episode result [%s]' % multi_result.name) continue # see how many of the eps that this result covers aren't covered by single results needed_eps = [] not_needed_eps = [] for ep_obj in multi_result.episodes: ep_num = ep_obj.episode # if we have results for the episode if ep_num in found_results[provider_id] and 0 < len( found_results[provider_id][ep_num]): needed_eps.append(ep_num) else: not_needed_eps.append(ep_num) logger.log( u'Single episode check result is... needed episodes: %s, not needed episodes: %s' % (needed_eps, not_needed_eps), logger.DEBUG) if not not_needed_eps: logger.log( u'All of these episodes were covered by single episode results, ' + 'ignoring this multi episode result', logger.DEBUG) continue # check if these eps are already covered by another multi-result multi_needed_eps = [] multi_not_needed_eps = [] for ep_obj in multi_result.episodes: ep_num = ep_obj.episode if ep_num in multi_results: multi_not_needed_eps.append(ep_num) else: multi_needed_eps.append(ep_num) logger.log( u'Multi episode check result is... multi needed episodes: ' + '%s, multi not needed episodes: %s' % (multi_needed_eps, multi_not_needed_eps), logger.DEBUG) if not multi_needed_eps: logger.log( u'All of these episodes were covered by another multi episode nzb, ' + 'ignoring this multi episode result', logger.DEBUG) continue # if we're keeping this multi-result then remember it for ep_obj in multi_result.episodes: multi_results[ep_obj.episode] = multi_result # don't bother with the single result if we're going to get it with a multi result for ep_obj in multi_result.episodes: ep_num = ep_obj.episode if ep_num in found_results[provider_id]: logger.log( u'A needed multi episode result overlaps with a single episode result for episode ' + '#%s, removing the single episode results from the list' % ep_num, logger.DEBUG) del found_results[provider_id][ep_num] # of all the single ep results narrow it down to the best one for each episode final_results += set(multi_results.values()) quality_list = use_quality_list and ( None, best_qualities)[any(best_qualities)] or None for cur_ep in found_results[provider_id]: if cur_ep in (MULTI_EP_RESULT, SEASON_RESULT): continue if 0 == len(found_results[provider_id][cur_ep]): continue best_result = pick_best_result(found_results[provider_id][cur_ep], show, quality_list) # if all results were rejected move on to the next episode if not best_result: continue # filter out possible bad torrents from providers if 'torrent' == best_result.resultType: if not best_result.url.startswith( 'magnet') and None is not best_result.get_data_func: best_result.url = best_result.get_data_func( best_result.url) best_result.get_data_func = None # consume only once if not best_result.url: continue if best_result.url.startswith('magnet'): if 'blackhole' != sickbeard.TORRENT_METHOD: best_result.content = None else: cache_file = ek.ek( os.path.join, sickbeard.CACHE_DIR or helpers._getTempDir(), '%s.torrent' % (helpers.sanitizeFileName(best_result.name))) if not helpers.download_file( best_result.url, cache_file, session=best_result.provider.session): continue try: with open(cache_file, 'rb') as fh: td = fh.read() setattr(best_result, 'cache_file', cache_file) except (StandardError, Exception): continue if getattr(best_result.provider, 'chk_td', None): name = None try: hdr = re.findall('(\w+(\d+):)', td[0:6])[0] x, v = len(hdr[0]), int(hdr[1]) while x < len(td): y = x + v name = 'name' == td[x:y] w = re.findall('((?:i-?\d+e|e+|d|l+)*(\d+):)', td[y:y + 32])[0] x, v = y + len(w[0]), int(w[1]) if name: name = td[x:x + v] break except (StandardError, Exception): continue if name: if not pass_show_wordlist_checks(name, show): continue if not show_name_helpers.pass_wordlist_checks( name, indexer_lookup=False): logger.log( 'Ignored: %s (debug log has detail)' % name) continue best_result.name = name if 'blackhole' != sickbeard.TORRENT_METHOD: best_result.content = td # add result if its not a duplicate and found = False for i, result in enumerate(final_results): for best_result_ep in best_result.episodes: if best_result_ep in result.episodes: if best_result.quality > result.quality: final_results.pop(i) else: found = True if not found: final_results += [best_result] # check that we got all the episodes we wanted first before doing a match and snatch wanted_ep_count = 0 for wanted_ep in episodes: for result in final_results: if wanted_ep in result.episodes and is_final_result(result): wanted_ep_count += 1 # make sure we search every provider for results unless we found everything we wanted if len(episodes) == wanted_ep_count: break if not len(provider_list): logger.log( 'No NZB/Torrent providers in Media Providers/Options are allowed for active searching', logger.WARNING) elif not search_done: logger.log( 'Failed active search of %s enabled provider%s. More info in debug log.' % (len(provider_list), helpers.maybe_plural(len(provider_list))), logger.ERROR) elif not any(final_results): logger.log('No suitable candidates') return final_results
def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_list=None): propers = {} # for each provider get a list of the orig_thread_name = threading.currentThread().name providers = [ x for x in sickbeard.providers.sortedProviderList() if x.is_active() ] for cur_provider in providers: if not recent_anime and cur_provider.anime_only: continue if None is not proper_list: found_propers = proper_list.get(cur_provider.get_id(), []) if not found_propers: continue else: threading.currentThread( ).name = orig_thread_name + ' :: [' + cur_provider.name + ']' logger.log(u'Searching for new PROPER releases') try: found_propers = cur_provider.find_propers( search_date=aired_since_shows, shows=recent_shows, anime=recent_anime) except exceptions.AuthException as e: logger.log(u'Authentication error: ' + ex(e), logger.ERROR) continue except Exception as e: logger.log( u'Error while searching ' + cur_provider.name + ', skipping: ' + ex(e), logger.ERROR) logger.log(traceback.format_exc(), logger.ERROR) continue finally: threading.currentThread().name = orig_thread_name # if they haven't been added by a different provider than add the proper to the list count = 0 for x in found_propers: name = _generic_name(x.name) if name not in propers: try: np = NameParser(False, try_scene_exceptions=True, showObj=x.parsed_show, indexer_lookup=False) parse_result = np.parse(x.name) if parse_result.series_name and parse_result.episode_numbers and \ (parse_result.show.indexer, parse_result.show.indexerid) in recent_shows + recent_anime: cur_size = getattr(x, 'size', None) if failed_history.has_failed(x.name, cur_size, cur_provider.name): continue logger.log(u'Found new proper: ' + x.name, logger.DEBUG) x.show = parse_result.show.indexerid x.provider = cur_provider x.is_repack, x.properlevel = Quality.get_proper_level( parse_result.extra_info_no_name(), parse_result.version, parse_result.is_anime, check_is_repack=True) x.is_internal = parse_result.extra_info_no_name() and \ re.search(r'\binternal\b', parse_result.extra_info_no_name(), flags=re.I) x.codec = _get_codec(parse_result.extra_info_no_name()) propers[name] = x count += 1 except (InvalidNameException, InvalidShowException): continue except (StandardError, Exception): continue cur_provider.log_result('Propers', count, '%s' % cur_provider.name) # take the list of unique propers and get it sorted by sorted_propers = sorted(propers.values(), key=operator.attrgetter('properlevel', 'date'), reverse=True) verified_propers = set() for cur_proper in sorted_propers: np = NameParser(False, try_scene_exceptions=True, showObj=cur_proper.parsed_show, indexer_lookup=False) try: parse_result = np.parse(cur_proper.name) except (StandardError, Exception): continue # set the indexerid in the db to the show's indexerid cur_proper.indexerid = parse_result.show.indexerid # set the indexer in the db to the show's indexer cur_proper.indexer = parse_result.show.indexer # populate our Proper instance cur_proper.season = parse_result.season_number if None is not parse_result.season_number else 1 cur_proper.episode = parse_result.episode_numbers[0] cur_proper.release_group = parse_result.release_group cur_proper.version = parse_result.version cur_proper.extra_info = parse_result.extra_info cur_proper.extra_info_no_name = parse_result.extra_info_no_name cur_proper.quality = Quality.nameQuality(cur_proper.name, parse_result.is_anime) cur_proper.is_anime = parse_result.is_anime # only get anime proper if it has release group and version if parse_result.is_anime: if not cur_proper.release_group and -1 == cur_proper.version: logger.log( u'Proper %s doesn\'t have a release group and version, ignoring it' % cur_proper.name, logger.DEBUG) continue if not show_name_helpers.pass_wordlist_checks( cur_proper.name, parse=False, indexer_lookup=False): logger.log( u'Proper %s isn\'t a valid scene release that we want, ignoring it' % cur_proper.name, logger.DEBUG) continue re_extras = dict(re_prefix='.*', re_suffix='.*') result = show_name_helpers.contains_any( cur_proper.name, parse_result.show.rls_ignore_words, **re_extras) if None is not result and result: logger.log(u'Ignored: %s for containing ignore word' % cur_proper.name) continue result = show_name_helpers.contains_any( cur_proper.name, parse_result.show.rls_require_words, **re_extras) if None is not result and not result: logger.log( u'Ignored: %s for not containing any required word match' % cur_proper.name) continue # check if we actually want this proper (if it's the right quality) my_db = db.DBConnection() sql_results = my_db.select( 'SELECT release_group, status, version, release_name FROM tv_episodes WHERE showid = ? AND indexer = ? ' + 'AND season = ? AND episode = ?', [ cur_proper.indexerid, cur_proper.indexer, cur_proper.season, cur_proper.episode ]) if not sql_results: continue # only keep the proper if we have already retrieved the same quality ep (don't get better/worse ones) # don't take proper of the same level we already downloaded old_status, old_quality = Quality.splitCompositeStatus( int(sql_results[0]['status'])) cur_proper.is_repack, cur_proper.proper_level = Quality.get_proper_level( cur_proper.extra_info_no_name(), cur_proper.version, cur_proper.is_anime, check_is_repack=True) old_release_group = sql_results[0]['release_group'] # check if we want this release: same quality as current, current has correct status # restrict other release group releases to proper's if old_status not in SNATCHED_ANY + [DOWNLOADED, ARCHIVED] \ or cur_proper.quality != old_quality \ or (cur_proper.is_repack and cur_proper.release_group != old_release_group): continue np = NameParser(False, try_scene_exceptions=True, showObj=parse_result.show, indexer_lookup=False) try: extra_info = np.parse( sql_results[0]['release_name']).extra_info_no_name() except (StandardError, Exception): extra_info = None old_proper_level, old_is_internal, old_codec, old_extra_no_name, old_name = \ get_old_proper_level(parse_result.show, cur_proper.indexer, cur_proper.indexerid, cur_proper.season, parse_result.episode_numbers, old_status, cur_proper.quality, extra_info, cur_proper.version, cur_proper.is_anime) old_name = (old_name, sql_results[0]['release_name'])[old_name in ('', None)] if cur_proper.proper_level < old_proper_level: continue elif cur_proper.proper_level == old_proper_level: if '264' == cur_proper.codec and 'xvid' == old_codec: pass elif old_is_internal and not cur_proper.is_internal: pass else: continue log_same_grp = 'Skipping proper from release group: [%s], does not match existing release group: [%s] for [%s]'\ % (cur_proper.release_group, old_release_group, cur_proper.name) is_web = (old_quality in (Quality.HDWEBDL, Quality.FULLHDWEBDL, Quality.UHD4KWEB) or (old_quality == Quality.SDTV and re.search(r'\Wweb.?(dl|rip|.[hx]26[45])\W', str(sql_results[0]['release_name']), re.I))) if is_web: old_webdl_type = get_webdl_type(old_extra_no_name, old_name) new_webdl_type = get_webdl_type(cur_proper.extra_info_no_name(), cur_proper.name) if old_webdl_type != new_webdl_type: logger.log( 'Skipping proper webdl source: [%s], does not match existing webdl source: [%s] for [%s]' % (old_webdl_type, new_webdl_type, cur_proper.name), logger.DEBUG) continue # for webldls, prevent propers from different groups if sickbeard.PROPERS_WEBDL_ONEGRP and is_web and cur_proper.release_group != old_release_group: logger.log(log_same_grp, logger.DEBUG) continue # check if we actually want this proper (if it's the right release group and a higher version) if parse_result.is_anime: old_version = int(sql_results[0]['version']) if -1 < old_version < cur_proper.version: logger.log(u'Found new anime v%s to replace existing v%s' % (cur_proper.version, old_version)) else: continue if cur_proper.release_group != old_release_group: logger.log(log_same_grp, logger.DEBUG) continue # if the show is in our list and there hasn't been a proper already added for that particular episode # then add it to our list of propers if cur_proper.indexerid != -1: if (cur_proper.indexerid, cur_proper.indexer, cur_proper.season, cur_proper.episode) not in map( operator.attrgetter('indexerid', 'indexer', 'season', 'episode'), verified_propers): logger.log(u'Found a proper that may be useful: %s' % cur_proper.name) verified_propers.add(cur_proper) else: rp = set() for vp in verified_propers: if vp.indexer == cur_proper.indexer and vp.indexerid == cur_proper.indexerid and \ vp.season == cur_proper.season and vp.episode == cur_proper.episode and \ vp.proper_level < cur_proper.proper_level: rp.add(vp) if rp: verified_propers = verified_propers - rp logger.log(u'Found a proper that may be useful: %s' % cur_proper.name) verified_propers.add(cur_proper) return list(verified_propers)
def pick_best_result(results, show, quality_list=None, filter_rls=False): logger.log( u'Picking the best result out of %s' % [x.name for x in results], logger.DEBUG) # find the best result for the current episode best_result = None best_fallback_result = None scene_only = scene_or_contain = scene_loose = scene_loose_active = scene_rej_nuked = scene_nuked_active = False if filter_rls: try: provider = getattr(results[0], 'provider', None) scene_only = getattr(provider, 'scene_only', False) scene_or_contain = getattr(provider, 'scene_or_contain', '') recent_task = 'RECENT' in filter_rls scene_loose = getattr(provider, 'scene_loose', False) and recent_task scene_loose_active = getattr(provider, 'scene_loose_active', False) and not recent_task scene_rej_nuked = getattr(provider, 'scene_rej_nuked', False) scene_nuked_active = getattr(provider, 'scene_nuked_active', False) and not recent_task except (StandardError, Exception): filter_rls = False addendum = '' for cur_result in results: if show.is_anime and not show.release_groups.is_valid(cur_result): continue if quality_list and cur_result.quality not in quality_list: logger.log( u'Rejecting unwanted quality %s for [%s]' % (Quality.qualityStrings[cur_result.quality], cur_result.name), logger.DEBUG) continue if not pass_show_wordlist_checks(cur_result.name, show): continue cur_size = getattr(cur_result, 'size', None) if sickbeard.USE_FAILED_DOWNLOADS and None is not cur_size and failed_history.has_failed( cur_result.name, cur_size, cur_result.provider.name): logger.log(u'Rejecting previously failed [%s]' % cur_result.name) continue if filter_rls and any([ scene_only, scene_loose, scene_loose_active, scene_rej_nuked, scene_nuked_active ]): if show.is_anime: addendum = u'anime (skipping scene/nuke filter) ' else: scene_contains = False if scene_only and scene_or_contain: re_extras = dict(re_prefix='.*', re_suffix='.*') r = show_name_helpers.contains_any(cur_result.name, scene_or_contain, **re_extras) if None is not r and r: scene_contains = True if scene_contains and not scene_rej_nuked: logger.log( u'Considering title match to \'or contain\' [%s]' % cur_result.name, logger.DEBUG) reject = False else: reject, url = can_reject(cur_result.name) if reject: if isinstance(reject, basestring): if scene_rej_nuked and not scene_nuked_active: logger.log( u'Rejecting nuked release. Nuke reason [%s] source [%s]' % (reject, url), logger.DEBUG) elif scene_nuked_active: best_fallback_result = best_candidate( best_fallback_result, cur_result) else: logger.log( u'Considering nuked release. Nuke reason [%s] source [%s]' % (reject, url), logger.DEBUG) reject = False elif scene_contains or any( [scene_loose, scene_loose_active]): best_fallback_result = best_candidate( best_fallback_result, cur_result) else: logger.log( u'Rejecting as not scene release listed at any [%s]' % url, logger.DEBUG) if reject: continue best_result = best_candidate(best_result, cur_result) if best_result and scene_only and not show.is_anime: addendum = u'scene release filtered ' elif not best_result and best_fallback_result: addendum = u'non scene release filtered ' best_result = best_fallback_result if best_result: logger.log(u'Picked as the best %s[%s]' % (addendum, best_result.name), logger.DEBUG) else: logger.log(u'No result picked.', logger.DEBUG) return best_result
def _get_proper_list(aired_since_shows, recent_shows, recent_anime, proper_list=None): propers = {} my_db = db.DBConnection() # for each provider get a list of arbitrary Propers orig_thread_name = threading.currentThread().name providers = filter(lambda p: p.is_active(), sickbeard.providers.sortedProviderList()) for cur_provider in providers: if not recent_anime and cur_provider.anime_only: continue if None is not proper_list: found_propers = proper_list.get(cur_provider.get_id(), []) if not found_propers: continue else: threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name) logger.log('Searching for new PROPER releases') try: found_propers = cur_provider.find_propers( search_date=aired_since_shows, shows=recent_shows, anime=recent_anime) except exceptions.AuthException as e: logger.log('Authentication error: %s' % ex(e), logger.ERROR) continue except Exception as e: logger.log( 'Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR) logger.log(traceback.format_exc(), logger.ERROR) continue finally: threading.currentThread().name = orig_thread_name # if they haven't been added by a different provider than add the Proper to the list for cur_proper in found_propers: name = _generic_name(cur_proper.name) if name in propers: continue try: np = NameParser(False, try_scene_exceptions=True, showObj=cur_proper.parsed_show, indexer_lookup=False) parse_result = np.parse(cur_proper.name) except (InvalidNameException, InvalidShowException, Exception): continue # get the show object cur_proper.parsed_show = (cur_proper.parsed_show or helpers.findCertainShow( sickbeard.showList, parse_result.show.indexerid)) if None is cur_proper.parsed_show: logger.log( 'Skip download; cannot find show with indexerid [%s]' % cur_proper.indexerid, logger.ERROR) continue cur_proper.indexer = cur_proper.parsed_show.indexer cur_proper.indexerid = cur_proper.parsed_show.indexerid if not (-1 != cur_proper.indexerid and parse_result.series_name and parse_result.episode_numbers and (cur_proper.indexer, cur_proper.indexerid) in recent_shows + recent_anime): continue # only get anime Proper if it has release group and version if parse_result.is_anime and not parse_result.release_group and -1 == parse_result.version: logger.log( 'Ignored Proper with no release group and version in name [%s]' % cur_proper.name, logger.DEBUG) continue if not show_name_helpers.pass_wordlist_checks( cur_proper.name, parse=False, indexer_lookup=False): logger.log('Ignored unwanted Proper [%s]' % cur_proper.name, logger.DEBUG) continue re_x = dict(re_prefix='.*', re_suffix='.*') result = show_name_helpers.contains_any( cur_proper.name, cur_proper.parsed_show.rls_ignore_words, **re_x) if None is not result and result: logger.log( 'Ignored Proper containing ignore word [%s]' % cur_proper.name, logger.DEBUG) continue result = show_name_helpers.contains_any( cur_proper.name, cur_proper.parsed_show.rls_require_words, **re_x) if None is not result and not result: logger.log( 'Ignored Proper for not containing any required word [%s]' % cur_proper.name, logger.DEBUG) continue cur_size = getattr(cur_proper, 'size', None) if failed_history.has_failed(cur_proper.name, cur_size, cur_provider.name): continue cur_proper.season = parse_result.season_number if None is not parse_result.season_number else 1 cur_proper.episode = parse_result.episode_numbers[0] # check if we actually want this Proper (if it's the right quality) sql_results = my_db.select( 'SELECT release_group, status, version, release_name' ' FROM tv_episodes' ' WHERE showid = ? AND indexer = ? AND season = ? AND episode = ?' ' LIMIT 1', [ cur_proper.indexerid, cur_proper.indexer, cur_proper.season, cur_proper.episode ]) if not sql_results: continue # only keep the Proper if we already retrieved the same quality ep (don't get better/worse ones) # check if we want this release: same quality as current, current has correct status # restrict other release group releases to Proper's old_status, old_quality = Quality.splitCompositeStatus( int(sql_results[0]['status'])) cur_proper.quality = Quality.nameQuality(cur_proper.name, parse_result.is_anime) cur_proper.is_repack, cur_proper.properlevel = Quality.get_proper_level( parse_result.extra_info_no_name(), parse_result.version, parse_result.is_anime, check_is_repack=True) cur_proper.proper_level = cur_proper.properlevel # local non global value old_release_group = sql_results[0]['release_group'] try: same_release_group = parse_result.release_group.lower( ) == old_release_group.lower() except (StandardError, Exception): same_release_group = parse_result.release_group == old_release_group if old_status not in SNATCHED_ANY + [DOWNLOADED, ARCHIVED] \ or cur_proper.quality != old_quality \ or (cur_proper.is_repack and not same_release_group): continue np = NameParser(False, try_scene_exceptions=True, showObj=cur_proper.parsed_show, indexer_lookup=False) try: extra_info = np.parse( sql_results[0]['release_name']).extra_info_no_name() except (StandardError, Exception): extra_info = None # don't take Proper of the same level we already downloaded old_proper_level, old_is_internal, old_codec, old_extra_no_name, old_name = \ get_old_proper_level(cur_proper.parsed_show, cur_proper.indexer, cur_proper.indexerid, cur_proper.season, parse_result.episode_numbers, old_status, cur_proper.quality, extra_info, parse_result.version, parse_result.is_anime) cur_proper.codec = _get_codec(parse_result.extra_info_no_name()) if cur_proper.proper_level < old_proper_level: continue cur_proper.is_internal = (parse_result.extra_info_no_name() and re.search( r'\binternal\b', parse_result.extra_info_no_name(), flags=re.I)) if cur_proper.proper_level == old_proper_level: if (('264' == cur_proper.codec and 'xvid' == old_codec) or (old_is_internal and not cur_proper.is_internal)): pass continue is_web = (old_quality in (Quality.HDWEBDL, Quality.FULLHDWEBDL, Quality.UHD4KWEB) or (old_quality == Quality.SDTV and re.search( r'\Wweb.?(dl|rip|.[hx]26[45])\W', str(sql_results[0]['release_name']), re.I))) if is_web: old_name = (old_name, sql_results[0]['release_name'])[old_name in ('', None)] old_webdl_type = get_webdl_type(old_extra_no_name, old_name) new_webdl_type = get_webdl_type( parse_result.extra_info_no_name(), cur_proper.name) if old_webdl_type != new_webdl_type: logger.log( 'Ignored Proper webdl source [%s], does not match existing webdl source [%s] for [%s]' % (old_webdl_type, new_webdl_type, cur_proper.name), logger.DEBUG) continue # for webdls, prevent Propers from different groups log_same_grp = 'Ignored Proper from release group [%s] does not match existing group [%s] for [%s]' \ % (parse_result.release_group, old_release_group, cur_proper.name) if sickbeard.PROPERS_WEBDL_ONEGRP and is_web and not same_release_group: logger.log(log_same_grp, logger.DEBUG) continue # check if we actually want this Proper (if it's the right release group and a higher version) if parse_result.is_anime: old_version = int(sql_results[0]['version']) if not (-1 < old_version < parse_result.version): continue if not same_release_group: logger.log(log_same_grp, logger.DEBUG) continue found_msg = 'Found anime Proper v%s to replace v%s' % ( parse_result.version, old_version) else: found_msg = 'Found Proper [%s]' % cur_proper.name # make sure the episode has been downloaded before history_limit = datetime.datetime.today() - datetime.timedelta( days=30) history_results = my_db.select( 'SELECT resource FROM history' ' WHERE showid = ?' ' AND season = ? AND episode = ? AND quality = ? AND date >= ?' ' AND (%s)' % ' OR '.join('action LIKE "%%%02d"' % x for x in SNATCHED_ANY + [DOWNLOADED, ARCHIVED]), [ cur_proper.indexerid, cur_proper.season, cur_proper.episode, cur_proper.quality, history_limit.strftime(history.dateFormat) ]) # skip if the episode has never downloaded, because a previous quality is required to match the Proper if not len(history_results): logger.log( 'Ignored Proper cannot find a recent history item for [%s]' % cur_proper.name, logger.DEBUG) continue # make sure that none of the existing history downloads are the same Proper as the download candidate clean_proper_name = _generic_name( helpers.remove_non_release_groups( cur_proper.name, cur_proper.parsed_show.is_anime)) is_same = False for hitem in history_results: # if the result exists in history already we need to skip it if clean_proper_name == _generic_name( helpers.remove_non_release_groups( ek.ek(os.path.basename, hitem['resource']))): is_same = True break if is_same: logger.log('Ignored Proper already in history [%s]' % cur_proper.name) continue logger.log(found_msg, logger.DEBUG) # finish populating the Proper instance # cur_proper.show = cur_proper.parsed_show.indexerid cur_proper.provider = cur_provider cur_proper.extra_info = parse_result.extra_info cur_proper.extra_info_no_name = parse_result.extra_info_no_name cur_proper.release_group = parse_result.release_group cur_proper.is_anime = parse_result.is_anime cur_proper.version = parse_result.version propers[name] = cur_proper cur_provider.log_result('Propers', len(propers), '%s' % cur_provider.name) return propers.values()
def search_providers(show, episodes, manual_search=False, torrent_only=False, try_other_searches=False, old_status=None, scheduled=False): found_results = {} final_results = [] search_done = False orig_thread_name = threading.currentThread().name use_quality_list = None if any([episodes]): old_status = old_status or failed_history.find_old_status(episodes[0]) or episodes[0].status if old_status: status, quality = Quality.splitCompositeStatus(old_status) use_quality_list = (status not in ( common.WANTED, common.FAILED, common.UNAIRED, common.SKIPPED, common.IGNORED, common.UNKNOWN)) provider_list = [x for x in sickbeard.providers.sortedProviderList() if x.is_active() and x.enable_backlog and (not torrent_only or x.providerType == GenericProvider.TORRENT) and (not scheduled or x.enable_scheduled_backlog)] for cur_provider in provider_list: if cur_provider.anime_only and not show.is_anime: logger.log(u'%s is not an anime, skipping' % show.name, logger.DEBUG) continue threading.currentThread().name = '%s :: [%s]' % (orig_thread_name, cur_provider.name) provider_id = cur_provider.get_id() found_results[provider_id] = {} search_count = 0 search_mode = getattr(cur_provider, 'search_mode', 'eponly') while True: search_count += 1 if 'eponly' == search_mode: logger.log(u'Performing episode search for %s' % show.name) else: logger.log(u'Performing season pack search for %s' % show.name) try: cur_provider.cache._clearCache() search_results = cur_provider.find_search_results(show, episodes, search_mode, manual_search, try_other_searches=try_other_searches) if any(search_results): logger.log(', '.join(['%s %s candidate%s' % ( len(v), (('multiep', 'season')[SEASON_RESULT == k], 'episode')['ep' in search_mode], helpers.maybe_plural(len(v))) for (k, v) in search_results.iteritems()])) except exceptions.AuthException as e: logger.log(u'Authentication error: %s' % ex(e), logger.ERROR) break except Exception as e: logger.log(u'Error while searching %s, skipping: %s' % (cur_provider.name, ex(e)), logger.ERROR) logger.log(traceback.format_exc(), logger.ERROR) break finally: threading.currentThread().name = orig_thread_name search_done = True if len(search_results): # make a list of all the results for this provider for cur_ep in search_results: # skip non-tv crap search_results[cur_ep] = filter( lambda ep_item: show_name_helpers.pass_wordlist_checks( ep_item.name, parse=False, indexer_lookup=False) and ep_item.show == show, search_results[cur_ep]) if cur_ep in found_results: found_results[provider_id][cur_ep] += search_results[cur_ep] else: found_results[provider_id][cur_ep] = search_results[cur_ep] break elif not getattr(cur_provider, 'search_fallback', False) or 2 == search_count: break search_mode = '%sonly' % ('ep', 'sp')['ep' in search_mode] logger.log(u'Falling back to %s search ...' % ('season pack', 'episode')['ep' in search_mode]) # skip to next provider if we have no results to process if not len(found_results[provider_id]): continue any_qualities, best_qualities = Quality.splitQuality(show.quality) # pick the best season NZB best_season_result = None if SEASON_RESULT in found_results[provider_id]: best_season_result = pick_best_result(found_results[provider_id][SEASON_RESULT], show, any_qualities + best_qualities) highest_quality_overall = 0 for cur_episode in found_results[provider_id]: for cur_result in found_results[provider_id][cur_episode]: if Quality.UNKNOWN != cur_result.quality and highest_quality_overall < cur_result.quality: highest_quality_overall = cur_result.quality logger.log(u'%s is the highest quality of any match' % Quality.qualityStrings[highest_quality_overall], logger.DEBUG) # see if every episode is wanted if best_season_result: # get the quality of the season nzb season_qual = best_season_result.quality logger.log(u'%s is the quality of the season %s' % (Quality.qualityStrings[season_qual], best_season_result.provider.providerType), logger.DEBUG) my_db = db.DBConnection() sql = 'SELECT season, episode FROM tv_episodes WHERE showid = %s AND (season IN (%s))' %\ (show.indexerid, ','.join([str(x.season) for x in episodes])) ep_nums = [(int(x['season']), int(x['episode'])) for x in my_db.select(sql)] logger.log(u'Executed query: [%s]' % sql) logger.log(u'Episode list: %s' % ep_nums, logger.DEBUG) all_wanted = True any_wanted = False for ep_num in ep_nums: if not show.wantEpisode(ep_num[0], ep_num[1], season_qual): all_wanted = False else: any_wanted = True # if we need every ep in the season and there's nothing better then just download this and # be done with it (unless single episodes are preferred) if all_wanted and highest_quality_overall == best_season_result.quality: logger.log(u'Every episode in this season is needed, downloading the whole %s %s' % (best_season_result.provider.providerType, best_season_result.name)) ep_objs = [] for ep_num in ep_nums: ep_objs.append(show.getEpisode(ep_num[0], ep_num[1])) best_season_result.episodes = ep_objs return [best_season_result] elif not any_wanted: logger.log(u'No episodes from this season are wanted at this quality, ignoring the result of ' + best_season_result.name, logger.DEBUG) else: if GenericProvider.NZB == best_season_result.provider.providerType: logger.log(u'Breaking apart the NZB and adding the individual ones to our results', logger.DEBUG) # if not, break it apart and add them as the lowest priority results individual_results = nzbSplitter.splitResult(best_season_result) individual_results = filter( lambda r: show_name_helpers.pass_wordlist_checks( r.name, parse=False, indexer_lookup=False) and r.show == show, individual_results) for cur_result in individual_results: if 1 == len(cur_result.episodes): ep_num = cur_result.episodes[0].episode elif 1 < len(cur_result.episodes): ep_num = MULTI_EP_RESULT if ep_num in found_results[provider_id]: found_results[provider_id][ep_num].append(cur_result) else: found_results[provider_id][ep_num] = [cur_result] # If this is a torrent all we can do is leech the entire torrent, # user will have to select which eps not do download in his torrent client else: # Season result from Torrent Provider must be a full-season torrent, creating multi-ep result for it logger.log(u'Adding multi episode result for full season torrent. In your torrent client, set ' + u'the episodes that you do not want to "don\'t download"') ep_objs = [] for ep_num in ep_nums: ep_objs.append(show.getEpisode(ep_num[0], ep_num[1])) best_season_result.episodes = ep_objs ep_num = MULTI_EP_RESULT if ep_num in found_results[provider_id]: found_results[provider_id][ep_num].append(best_season_result) else: found_results[provider_id][ep_num] = [best_season_result] # go through multi-ep results and see if we really want them or not, get rid of the rest multi_results = {} if MULTI_EP_RESULT in found_results[provider_id]: for multi_result in found_results[provider_id][MULTI_EP_RESULT]: logger.log(u'Checking usefulness of multi episode result [%s]' % multi_result.name, logger.DEBUG) if sickbeard.USE_FAILED_DOWNLOADS and failed_history.has_failed(multi_result.name, multi_result.size, multi_result.provider.name): logger.log(u'Rejecting previously failed multi episode result [%s]' % multi_result.name) continue # see how many of the eps that this result covers aren't covered by single results needed_eps = [] not_needed_eps = [] for ep_obj in multi_result.episodes: ep_num = ep_obj.episode # if we have results for the episode if ep_num in found_results[provider_id] and 0 < len(found_results[provider_id][ep_num]): needed_eps.append(ep_num) else: not_needed_eps.append(ep_num) logger.log(u'Single episode check result is... needed episodes: %s, not needed episodes: %s' % (needed_eps, not_needed_eps), logger.DEBUG) if not not_needed_eps: logger.log(u'All of these episodes were covered by single episode results, ' + 'ignoring this multi episode result', logger.DEBUG) continue # check if these eps are already covered by another multi-result multi_needed_eps = [] multi_not_needed_eps = [] for ep_obj in multi_result.episodes: ep_num = ep_obj.episode if ep_num in multi_results: multi_not_needed_eps.append(ep_num) else: multi_needed_eps.append(ep_num) logger.log(u'Multi episode check result is... multi needed episodes: ' + '%s, multi not needed episodes: %s' % (multi_needed_eps, multi_not_needed_eps), logger.DEBUG) if not multi_needed_eps: logger.log(u'All of these episodes were covered by another multi episode nzb, ' + 'ignoring this multi episode result', logger.DEBUG) continue # if we're keeping this multi-result then remember it for ep_obj in multi_result.episodes: multi_results[ep_obj.episode] = multi_result # don't bother with the single result if we're going to get it with a multi result for ep_obj in multi_result.episodes: ep_num = ep_obj.episode if ep_num in found_results[provider_id]: logger.log(u'A needed multi episode result overlaps with a single episode result for episode ' + '#%s, removing the single episode results from the list' % ep_num, logger.DEBUG) del found_results[provider_id][ep_num] # of all the single ep results narrow it down to the best one for each episode final_results += set(multi_results.values()) quality_list = use_quality_list and (None, best_qualities)[any(best_qualities)] or None for cur_ep in found_results[provider_id]: if cur_ep in (MULTI_EP_RESULT, SEASON_RESULT): continue if 0 == len(found_results[provider_id][cur_ep]): continue best_result = pick_best_result(found_results[provider_id][cur_ep], show, quality_list, filter_rls=orig_thread_name) # if all results were rejected move on to the next episode if not best_result: continue # filter out possible bad torrents from providers if 'torrent' == best_result.resultType: if not best_result.url.startswith('magnet') and None is not best_result.get_data_func: best_result.url = best_result.get_data_func(best_result.url) best_result.get_data_func = None # consume only once if not best_result.url: continue if best_result.url.startswith('magnet'): if 'blackhole' != sickbeard.TORRENT_METHOD: best_result.content = None else: cache_file = ek.ek(os.path.join, sickbeard.CACHE_DIR or helpers._getTempDir(), '%s.torrent' % (helpers.sanitizeFileName(best_result.name))) if not helpers.download_file(best_result.url, cache_file, session=best_result.provider.session): continue try: with open(cache_file, 'rb') as fh: td = fh.read() setattr(best_result, 'cache_file', cache_file) except (StandardError, Exception): continue if getattr(best_result.provider, 'chk_td', None): name = None try: hdr = re.findall('(\w+(\d+):)', td[0:6])[0] x, v = len(hdr[0]), int(hdr[1]) while x < len(td): y = x + v name = 'name' == td[x: y] w = re.findall('((?:i-?\d+e|e+|d|l+)*(\d+):)', td[y: y + 32])[0] x, v = y + len(w[0]), int(w[1]) if name: name = td[x: x + v] break except (StandardError, Exception): continue if name: if not pass_show_wordlist_checks(name, show): continue if not show_name_helpers.pass_wordlist_checks(name, indexer_lookup=False): logger.log('Ignored: %s (debug log has detail)' % name) continue best_result.name = name if 'blackhole' != sickbeard.TORRENT_METHOD: best_result.content = td # add result if its not a duplicate and found = False for i, result in enumerate(final_results): for best_result_ep in best_result.episodes: if best_result_ep in result.episodes: if best_result.quality > result.quality: final_results.pop(i) else: found = True if not found: final_results += [best_result] # check that we got all the episodes we wanted first before doing a match and snatch wanted_ep_count = 0 for wanted_ep in episodes: for result in final_results: if wanted_ep in result.episodes and is_final_result(result): wanted_ep_count += 1 # make sure we search every provider for results unless we found everything we wanted if len(episodes) == wanted_ep_count: break if not len(provider_list): logger.log('No NZB/Torrent providers in Media Providers/Options are allowed for active searching', logger.WARNING) elif not search_done: logger.log('Failed active search of %s enabled provider%s. More info in debug log.' % ( len(provider_list), helpers.maybe_plural(len(provider_list))), logger.ERROR) elif not any(final_results): logger.log('No suitable candidates') return final_results
def pick_best_result(results, show, quality_list=None, filter_rls=False): logger.log(u'Picking the best result out of %s' % [x.name for x in results], logger.DEBUG) # find the best result for the current episode best_result = None best_fallback_result = None scene_only = scene_or_contain = scene_loose = scene_loose_active = scene_rej_nuked = scene_nuked_active = False if filter_rls: try: provider = getattr(results[0], 'provider', None) scene_only = getattr(provider, 'scene_only', False) scene_or_contain = getattr(provider, 'scene_or_contain', '') recent_task = 'RECENT' in filter_rls scene_loose = getattr(provider, 'scene_loose', False) and recent_task scene_loose_active = getattr(provider, 'scene_loose_active', False) and not recent_task scene_rej_nuked = getattr(provider, 'scene_rej_nuked', False) scene_nuked_active = getattr(provider, 'scene_nuked_active', False) and not recent_task except (StandardError, Exception): filter_rls = False addendum = '' for cur_result in results: if show.is_anime and not show.release_groups.is_valid(cur_result): continue if quality_list and cur_result.quality not in quality_list: logger.log(u'Rejecting unwanted quality %s for [%s]' % ( Quality.qualityStrings[cur_result.quality], cur_result.name), logger.DEBUG) continue if not pass_show_wordlist_checks(cur_result.name, show): continue cur_size = getattr(cur_result, 'size', None) if sickbeard.USE_FAILED_DOWNLOADS and None is not cur_size and failed_history.has_failed( cur_result.name, cur_size, cur_result.provider.name): logger.log(u'Rejecting previously failed [%s]' % cur_result.name) continue if filter_rls and any([scene_only, scene_loose, scene_loose_active, scene_rej_nuked, scene_nuked_active]): if show.is_anime: addendum = u'anime (skipping scene/nuke filter) ' else: scene_contains = False if scene_only and scene_or_contain: re_extras = dict(re_prefix='.*', re_suffix='.*') r = show_name_helpers.contains_any(cur_result.name, scene_or_contain, **re_extras) if None is not r and r: scene_contains = True if scene_contains and not scene_rej_nuked: logger.log(u'Considering title match to \'or contain\' [%s]' % cur_result.name, logger.DEBUG) reject = False else: reject, url = can_reject(cur_result.name) if reject: if isinstance(reject, basestring): if scene_rej_nuked and not scene_nuked_active: logger.log(u'Rejecting nuked release. Nuke reason [%s] source [%s]' % (reject, url), logger.DEBUG) elif scene_nuked_active: best_fallback_result = best_candidate(best_fallback_result, cur_result) else: logger.log(u'Considering nuked release. Nuke reason [%s] source [%s]' % (reject, url), logger.DEBUG) reject = False elif scene_contains or any([scene_loose, scene_loose_active]): best_fallback_result = best_candidate(best_fallback_result, cur_result) else: logger.log(u'Rejecting as not scene release listed at any [%s]' % url, logger.DEBUG) if reject: continue best_result = best_candidate(best_result, cur_result) if best_result and scene_only and not show.is_anime: addendum = u'scene release filtered ' elif not best_result and best_fallback_result: addendum = u'non scene release filtered ' best_result = best_fallback_result if best_result: logger.log(u'Picked as the best %s[%s]' % (addendum, best_result.name), logger.DEBUG) else: logger.log(u'No result picked.', logger.DEBUG) return best_result