def testConvertMovie(self):
MovieIdCache.delete().execute()
id = infos.convertId("imdb", "tmdb", "0169547")
self.assertEqual("14", id)
#This time from cache
id = infos.convertId("imdb", "tmdb", "0169547")
self.assertEqual("14", id)
MovieIdCache.delete().execute()
id = infos.convertId("tmdb", "imdb", "14")
self.assertEqual("0169547", id)
def testConvertMovie(self):
MovieIdCache.delete().execute()
id = infos.convertId("imdb", "tmdb", "0169547")
self.assertEqual("14", id)
#This time from cache
id = infos.convertId("imdb", "tmdb", "0169547")
self.assertEqual("14", id)
MovieIdCache.delete().execute()
id = infos.convertId("tmdb", "imdb", "14")
self.assertEqual("0169547", id)
def testGetMovieTitle(self):
MovieIdCache.delete().execute()
title = infos.convertId("imdb", "title", "0169547")
self.assertEqual("American Beauty", title)
# This time from cache
title = infos.convertId("imdb", "title", "0169547")
self.assertEqual("American Beauty", title)
MovieIdCache.delete().execute()
title = infos.convertId("tmdb", "title", "14")
self.assertEqual("American Beauty", title)
# This time from cache
title = infos.convertId("tmdb", "title", "14")
self.assertEqual("American Beauty", title)
def testGetMovieTitle(self):
MovieIdCache.delete().execute()
title = infos.convertId("imdb", "title", "0169547")
self.assertEqual("American Beauty", title)
# This time from cache
title = infos.convertId("imdb", "title", "0169547")
self.assertEqual("American Beauty", title)
MovieIdCache.delete().execute()
title = infos.convertId("tmdb", "title", "14")
self.assertEqual("American Beauty", title)
# This time from cache
title = infos.convertId("tmdb", "title", "14")
self.assertEqual("American Beauty", title)
def testConvertTv(self):
TvIdCache.delete().execute()
id = infos.convertId("tvdb", "tvrage", "299350")
self.assertEqual("47566", id)
#This time from cache
id = infos.convertId("tvdb", "tvrage", "299350")
self.assertEqual("47566", id)
TvIdCache.delete().execute()
id = infos.convertId("tvdb", "tvmaze", "299350")
self.assertEqual("3036", id)
#This time from cache
id = infos.convertId("tvdb", "tvmaze", "299350")
self.assertEqual("3036", id)
TvIdCache.delete().execute()
id = infos.convertId("tvrage", "tvdb", "47566")
self.assertEqual("299350", id)
TvIdCache.delete().execute()
id = infos.convertId("tvrage", "tvmaze", "47566")
self.assertEqual("3036", id)
TvIdCache.delete().execute()
id = infos.convertId("tvmaze", "tvrage", "3036")
self.assertEqual("47566", id)
TvIdCache.delete().execute()
id = infos.convertId("tvmaze", "tvdb", "3036")
self.assertEqual("299350", id)
def testConvertTv(self):
TvIdCache.delete().execute()
id = infos.convertId("tvdb", "tvrage", "299350")
self.assertEqual("47566", id)
#This time from cache
id = infos.convertId("tvdb", "tvrage", "299350")
self.assertEqual("47566", id)
TvIdCache.delete().execute()
id = infos.convertId("tvdb", "tvmaze", "299350")
self.assertEqual("3036", id)
#This time from cache
id = infos.convertId("tvdb", "tvmaze", "299350")
self.assertEqual("3036", id)
TvIdCache.delete().execute()
id = infos.convertId("tvrage", "tvdb", "47566")
self.assertEqual("299350", id)
TvIdCache.delete().execute()
id = infos.convertId("tvrage", "tvmaze", "47566")
self.assertEqual("3036", id)
TvIdCache.delete().execute()
id = infos.convertId("tvmaze", "tvrage", "3036")
self.assertEqual("47566", id)
TvIdCache.delete().execute()
id = infos.convertId("tvmaze", "tvdb", "3036")
self.assertEqual("299350", id)
def internalapi_moviesearch(args):
logger.debug("Movie search request with args %s" % args)
indexers = urllib.unquote(
args["indexers"]) if args["indexers"] is not None else None
search_request = SearchRequest(type="movie",
query=args["query"],
offset=args["offset"],
category=args["category"],
minsize=args["minsize"],
maxsize=args["maxsize"],
minage=args["minage"],
maxage=args["maxage"],
indexers=indexers)
if args["imdbid"]:
search_request.identifier_key = "imdbid"
search_request.identifier_value = args["imdbid"]
elif args["tmdbid"]:
logger.debug("Need to get IMDB id from TMDB id %s" % args["tmdbid"])
imdbid = infos.convertId("tmdb", "imdb", args["tmdbid"])
if imdbid is None:
raise AttributeError("Unable to convert TMDB id %s" %
args["tmdbid"])
search_request.identifier_key = "imdbid"
search_request.identifier_value = imdbid
return startSearch(search_request)
def internalapi_moviesearch(args):
logger.debug("Movie search request with args %s" % args)
indexers = urllib.unquote(args["indexers"]) if args["indexers"] is not None else None
search_request = SearchRequest(type="movie", query=args["query"], offset=args["offset"], category=args["category"], minsize=args["minsize"], maxsize=args["maxsize"], minage=args["minage"], maxage=args["maxage"], indexers=indexers)
if args["imdbid"]:
search_request.identifier_key = "imdbid"
search_request.identifier_value = args["imdbid"]
elif args["tmdbid"]:
logger.debug("Need to get IMDB id from TMDB id %s" % args["tmdbid"])
imdbid = infos.convertId("tmdb", "imdb", args["tmdbid"])
if imdbid is None:
raise AttributeError("Unable to convert TMDB id %s" % args["tmdbid"])
search_request.identifier_key = "imdbid"
search_request.identifier_value = imdbid
return startSearch(search_request)
def testGetTvTitleDoesntExist(self):
TvIdCache.delete().execute()
id = infos.convertId("tvdb", "title", "299350000")
self.assertIsNone(id)
def pick_indexers(search_request, internal=True):
# type: (nzbhydra.search.SearchRequest, bool) -> List[nzbhydra.search_modules.SearchModule]
query_supplied = True if search_request.query else False
queryCanBeGenerated = None #Store if we can generate a query from IDs. Initiall true but when we need this the first time and query generation fails we set it to false
picked_indexers = []
selected_indexers = search_request.indexers.split(
"|") if search_request.indexers is not None else None
for p in indexers.enabled_indexers:
if not p.settings.enabled:
logger.debug("Did not pick %s because it is disabled" % p)
continue
if internal and p.settings.accessType == "external":
logger.debug(
"Did not pick %s because it is only enabled for external searches"
% p)
continue
if not internal and p.settings.accessType == "internal":
logger.debug(
"Did not pick %s because it is only enabled for internal searches"
% p)
continue
if selected_indexers and p.name not in selected_indexers:
logger.debug(
"Did not pick %s because it was not selected by the user" % p)
continue
try:
status = p.indexer.status.get()
if status.disabled_until > arrow.utcnow(
) and not config.settings.searching.ignoreTemporarilyDisabled:
logger.info(
"Did not pick %s because it is disabled temporarily due to an error: %s"
% (p, status.reason))
continue
except IndexerStatus.DoesNotExist:
pass
if p.settings.hitLimit > 0:
if p.settings.hitLimitResetTime:
hitLimitResetTime = arrow.get(p.settings.hitLimitResetTime)
comparisonTime = arrow.now().replace(
hour=hitLimitResetTime.hour,
minute=hitLimitResetTime.minute,
second=0)
if comparisonTime > arrow.now():
comparisonTime = arrow.get(
comparisonTime.datetime - timedelta(days=1)
) #Arrow is too dumb to properly subtract 1 day (throws an error on every first of the month)
else:
comparisonTime = arrow.now().replace(hour=0,
minute=0,
second=0)
apiHits = IndexerApiAccess().select().where(
(IndexerApiAccess.indexer == p.indexer)
& (IndexerApiAccess.time > comparisonTime)
& IndexerApiAccess.response_successful).count()
if apiHits > p.settings.hitLimit:
logger.info(
"Did not pick %s because its API hit limit of %d was reached"
% (p, p.settings.hitLimit))
continue
else:
logger.debug(
"%s has had %d of a maximum of %d API hits since %02d:%02d"
% (p, apiHits, p.settings.hitLimit, comparisonTime.hour,
comparisonTime.minute))
if (query_supplied or search_request.identifier_key
is not None) and not p.supports_queries:
logger.debug(
"Did not pick %s because a query was supplied but the indexer does not support queries"
% p)
continue
# Here on we check if we could supply the indexer with generated/retrieved data like the title of a series
if not query_supplied and p.needs_queries and search_request.identifier_key is None:
logger.debug(
"Did not pick %s because no query was supplied but the indexer needs queries"
% p)
continue
# If we can theoretically do that we must try to actually get the title, otherwise the indexer won't be able to search
allow_query_generation = (
config.InternalExternalSelection.internal
in config.settings.searching.generate_queries
and internal) or (config.InternalExternalSelection.external
in config.settings.searching.generate_queries
and not internal)
if search_request.identifier_key is not None and not canUseIdKey(
p, search_request.identifier_key):
if not (allow_query_generation and p.generate_queries):
logger.debug(
"Did not pick %s because search will be done by an identifier and the indexer or system wide settings don't allow query generation"
% p)
continue
else:
if queryCanBeGenerated is None:
try:
title = infos.convertId(
search_request.identifier_key, "title",
search_request.identifier_value)
if title:
search_request.title = title
queryCanBeGenerated = True
else:
queryCanBeGenerated = False
except:
queryCanBeGenerated = False
logger.debug(
"Unable to get title for supplied ID. Indexers that don't support the ID will be skipped"
)
if not queryCanBeGenerated:
logger.debug(
"Did not pick %s because search will be done by an identifier and retrieval of the title for query generation failed"
% p)
continue
logger.debug("Picked %s" % p)
picked_indexers.append(p)
return picked_indexers
def search(search_request):
logger.info("Starting new search: %s" % search_request)
if search_request.maxage is None and config.settings.searching.maxAge:
search_request.maxage = config.settings.searching.maxAge
logger.info("Will ignore results older than %d days" % search_request.maxage)
# Clean up cache
for k in list(pseudo_cache.keys()):
if pseudo_cache[k]["last_access"].replace(minutes=+5) < arrow.utcnow():
pseudo_cache.pop(k)
# Clean up old search results. We do this here because we don't have any background jobs and this is the function most regularly called
keepFor = config.settings.main.keepSearchResultsForDays
oldSearchResultsCount = countOldSearchResults(keepFor)
if oldSearchResultsCount > 0:
logger.info("Deleting %d search results from database that are older than %d days" % (oldSearchResultsCount, keepFor))
SearchResult.delete().where(SearchResult.firstFound < (datetime.date.today() - datetime.timedelta(days=keepFor))).execute()
else:
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug("%d search results stored in database" % SearchResult.select().count())
limit = search_request.limit
external_offset = int(search_request.offset)
search_hash = search_request.search_hash
categoryResult = categories.getCategoryByAnyInput(search_request.category)
search_request.category = categoryResult
if search_hash not in pseudo_cache.keys() or search_request.offset == 0: # If it's a new search (which starts with offset 0) do it again instead of using the cached results
logger.debug("Didn't find this query in cache or want to do a new search")
cache_entry = {"results": [], "indexer_infos": {}, "total": 0, "last_access": arrow.utcnow(), "offset": 0, "rejected": SearchModule.getRejectedCountDict(), "usedFallback": False}
category = categoryResult.category
indexers_to_call = pick_indexers(search_request)
for p in indexers_to_call:
cache_entry["indexer_infos"][p] = {"has_more": True, "search_request": search_request, "total_included": False}
dbsearch = Search(internal=search_request.internal, query=search_request.query, category=categoryResult.category.pretty, identifier_key=search_request.identifier_key, identifier_value=search_request.identifier_value, season=search_request.season, episode=search_request.episode,
type=search_request.type, title=search_request.title, author=search_request.author, username=search_request.username)
saveSearch(dbsearch)
# dbsearch.save()
cache_entry["dbsearch"] = dbsearch
# Find ignored words and parse query for ignored words
search_request.forbiddenWords = []
search_request.requiredWords = []
applyRestrictionsGlobal = config.settings.searching.applyRestrictions == "both" or (config.settings.searching.applyRestrictions == "internal" and search_request.internal) or (config.settings.searching.applyRestrictions == "external" and not search_request.internal)
applyRestrictionsCategory = category.applyRestrictions == "both" or (category.applyRestrictions == "internal" and search_request.internal) or (search_request.category.category.applyRestrictions == "external" and not search_request.internal)
if config.settings.searching.forbiddenWords and applyRestrictionsGlobal:
logger.debug("Using configured global forbidden words: %s" % config.settings.searching.forbiddenWords)
search_request.forbiddenWords.extend([x.lower().strip() for x in list(filter(bool, config.settings.searching.forbiddenWords.split(",")))])
if config.settings.searching.requiredWords and applyRestrictionsGlobal:
logger.debug("Using configured global required words: %s" % config.settings.searching.requiredWords)
search_request.requiredWords.extend([x.lower().strip() for x in list(filter(bool, config.settings.searching.requiredWords.split(",")))])
if category.forbiddenWords and applyRestrictionsCategory:
logger.debug("Using configured forbidden words for category %s: %s" % (category.pretty, category.forbiddenWords))
search_request.forbiddenWords.extend([x.lower().strip() for x in list(filter(bool, category.forbiddenWords.split(",")))])
if category.requiredWords and applyRestrictionsCategory:
logger.debug("Using configured required words for category %s: %s" % (category.pretty, category.requiredWords))
search_request.requiredWords.extend([x.lower().strip() for x in list(filter(bool, category.requiredWords.split(",")))])
if search_request.query:
forbiddenWords = [str(x[1]) for x in re.findall(r"[\s|\b](\-\-|!)(?P<term>\w+)", search_request.query)]
if len(forbiddenWords) > 0:
logger.debug("Query before removing NOT terms: %s" % search_request.query)
search_request.query = re.sub(r"[\s|\b](\-\-|!)(?P<term>\w+)", "", search_request.query)
logger.debug("Query after removing NOT terms: %s" % search_request.query)
logger.debug("Found NOT terms: %s" % ",".join(forbiddenWords))
search_request.forbiddenWords.extend(forbiddenWords)
cache_entry["forbiddenWords"] = search_request.forbiddenWords
cache_entry["requiredWords"] = search_request.requiredWords
cache_entry["query"] = search_request.query
pseudo_cache[search_hash] = cache_entry
else:
cache_entry = pseudo_cache[search_hash]
indexers_to_call = [indexer for indexer, info in cache_entry["indexer_infos"].items() if info["has_more"]]
dbsearch = cache_entry["dbsearch"]
search_request.forbiddenWords = cache_entry["forbiddenWords"]
search_request.requiredWords = cache_entry["requiredWords"]
search_request.query = cache_entry["query"]
logger.debug("Found search in cache")
logger.debug("Will search at indexers as long as we don't have enough results for the current offset+limit and any indexer has more results.")
if search_request.loadAll:
logger.debug("Requested to load all results. Will continue to search until all indexers are exhausted")
while (len(cache_entry["results"]) < external_offset + limit or search_request.loadAll) and len(indexers_to_call) > 0:
if len(cache_entry["results"]) < external_offset + limit:
logger.debug("We want %d results but have only %d so far" % ((external_offset + limit), len(cache_entry["results"])))
elif search_request.loadAll:
logger.debug("All results requested. Continuing to search.")
logger.debug("%d indexers still have results" % len(indexers_to_call))
search_request.offset = cache_entry["offset"]
logger.debug("Searching indexers with offset %d" % search_request.offset)
result = search_and_handle_db(dbsearch, {x: search_request for x in indexers_to_call})
logger.debug("All search calls to indexers completed")
search_results = []
indexers_to_call = []
waslocked = False
before = arrow.now()
if databaseLock.locked():
logger.debug("Database accesses locked by other search. Will wait for our turn.")
waslocked = True
databaseLock.acquire()
if waslocked:
after = arrow.now()
took = (after - before).seconds * 1000 + (after - before).microseconds / 1000
logger.debug("Waited %dms for database lock" % took)
for indexer, queries_execution_result in result["results"].items():
with db.atomic():
logger.info("%s returned %d results" % (indexer, len(queries_execution_result.results)))
for result in queries_execution_result.results:
if result.title is None or result.link is None or result.indexerguid is None:
logger.info("Skipping result with missing data: %s" % result)
continue
try:
searchResultId = hashlib.sha1(str(indexer.indexer.id) + result.indexerguid).hexdigest()
tryGetOrCreateSearchResultDbEntry(searchResultId, indexer.indexer.id, result)
result.searchResultId = searchResultId
search_results.append(result)
except (IntegrityError, OperationalError) as e:
logger.error("Error while trying to save search result to database. Skipping it. Error: %s" % e)
cache_entry["indexer_infos"][indexer].update(
{"did_search": queries_execution_result.didsearch, "indexer": indexer.name, "search_request": search_request, "has_more": queries_execution_result.has_more, "total": queries_execution_result.total, "total_known": queries_execution_result.total_known,
"indexer_search": queries_execution_result.indexerSearchEntry, "rejected": queries_execution_result.rejected, "processed_results": queries_execution_result.loaded_results})
if queries_execution_result.has_more:
indexers_to_call.append(indexer)
logger.debug("%s still has more results so we could use it the next round" % indexer)
if queries_execution_result.total_known:
if not cache_entry["indexer_infos"][indexer]["total_included"]:
cache_entry["total"] += queries_execution_result.total
logger.debug("%s reports %d total results. We'll include in the total this time only" % (indexer, queries_execution_result.total))
cache_entry["indexer_infos"][indexer]["total_included"] = True
elif queries_execution_result.has_more:
logger.debug("%s doesn't report an exact number of results so let's just add another 100 to the total" % indexer)
cache_entry["total"] += 100
for rejectKey in cache_entry["rejected"].keys():
if rejectKey in cache_entry["indexer_infos"][indexer]["rejected"].keys():
cache_entry["rejected"][rejectKey] += cache_entry["indexer_infos"][indexer]["rejected"][rejectKey]
databaseLock.release()
logger.debug("Searching for duplicates")
numberResultsBeforeDuplicateRemoval = len(search_results)
grouped_by_sameness, uniqueResultsPerIndexer = find_duplicates(search_results)
allresults = []
for group in grouped_by_sameness:
if search_request.internal:
for i in group:
# We give each group of results a unique value by which they can be identified later
i.hash = hash(group[0].details_link)
allresults.append(i)
else:
# We sort by age first and then by indexerscore so the newest result with the highest indexer score is chosen
group = sorted(group, key=lambda x: x.epoch, reverse=True)
group = sorted(group, key=lambda x: x.indexerscore, reverse=True)
allresults.append(group[0])
search_results = allresults
with databaseLock:
for indexer, result_infos in cache_entry["indexer_infos"].iteritems():
if indexer.name in uniqueResultsPerIndexer.keys(): # If the search failed it isn't contained in the duplicates list
uniqueResultsCount = uniqueResultsPerIndexer[result_infos["indexer"]]
processedResults = result_infos["processed_results"]
logger.debug("Indexer %s had a unique results share of %d%% (%d of %d total results were only provided by this indexer)" % (indexer.name, 100 / (numberResultsBeforeDuplicateRemoval / uniqueResultsCount), uniqueResultsCount, numberResultsBeforeDuplicateRemoval))
result_infos["indexer_search"].uniqueResults = uniqueResultsCount
result_infos["indexer_search"].processedResults = processedResults
result_infos["indexer_search"].save()
if not search_request.internal:
countAfter = len(search_results)
countRemoved = numberResultsBeforeDuplicateRemoval - countAfter
logger.info("Removed %d duplicates from %d results" % (countRemoved, numberResultsBeforeDuplicateRemoval))
search_results = sorted(search_results, key=lambda x: x.epoch, reverse=True)
cache_entry["results"].extend(search_results)
cache_entry["offset"] += limit
if len(indexers_to_call) == 0 and len(cache_entry["results"]) == 0 and search_request.identifier_key is not None and not cache_entry["usedFallback"] and ("internal" if search_request.internal else "external") in config.settings.searching.idFallbackToTitle:
logger.debug("No results found using ID based search. Getting title from ID to fall back")
title = infos.convertId(search_request.identifier_key, "title", search_request.identifier_value)
if title:
logger.info("Repeating search with title based query as fallback")
search_request.title = title
indexers_to_call = [indexer for indexer, _ in cache_entry["indexer_infos"].items()]
cache_entry["usedFallback"] = True
else:
logger.info("Unable to find title for ID")
if len(indexers_to_call) == 0:
logger.info("All indexers exhausted")
elif len(cache_entry["results"]) >= external_offset + limit:
logger.debug("Loaded a total of %d results which is enough for the %d requested. Stopping search." % (len(cache_entry["results"]), (external_offset + limit)))
if search_request.internal:
logger.debug("We have %d cached results and return them all because we search internally" % len(cache_entry["results"]))
nzb_search_results = copy.deepcopy(cache_entry["results"][external_offset:])
else:
logger.debug("We have %d cached results and return %d-%d of %d total available accounting for the limit set for the API search" % (len(cache_entry["results"]), external_offset, external_offset + limit, cache_entry["total"]))
nzb_search_results = copy.deepcopy(cache_entry["results"][external_offset:(external_offset + limit)])
cache_entry["last_access"] = arrow.utcnow()
for k, v in cache_entry["rejected"].items():
if v > 0:
logger.info("Rejected %d results %s" % (v, k))
logger.info("Returning %d results" % len(nzb_search_results))
return {"results": nzb_search_results, "indexer_infos": cache_entry["indexer_infos"], "dbsearchid": cache_entry["dbsearch"].id, "total": cache_entry["total"], "offset": external_offset, "rejected": cache_entry["rejected"].items()}
def testGetMovieTitleDoesNotExist(self):
MovieIdCache.delete().execute()
title = infos.convertId("imdb", "title", "016954739339")
self.assertIsNone(title)
def search(search_request):
logger.info("Starting new search: %s" % search_request)
if search_request.maxage is None and config.settings.searching.maxAge:
search_request.maxage = config.settings.searching.maxAge
logger.info("Will ignore results older than %d days" % search_request.maxage)
# Clean up cache
for k in list(pseudo_cache.keys()):
if pseudo_cache[k]["last_access"].replace(minutes=+5) < arrow.utcnow():
pseudo_cache.pop(k)
# Clean up old search results. We do this here because we don't have any background jobs and this is the function most regularly called
keepFor = config.settings.main.keepSearchResultsForDays
oldSearchResultsCount = countOldSearchResults(keepFor)
if oldSearchResultsCount > 0:
logger.info("Deleting %d search results from database that are older than %d days" % (oldSearchResultsCount, keepFor))
SearchResult.delete().where(SearchResult.firstFound < (datetime.date.today() - datetime.timedelta(days=keepFor))).execute()
else:
if logger.getEffectiveLevel() == logging.DEBUG:
logger.debug("%d search results stored in database" % SearchResult.select().count())
limit = search_request.limit
external_offset = int(search_request.offset)
search_hash = search_request.search_hash
categoryResult = categories.getCategoryByAnyInput(search_request.category)
search_request.category = categoryResult
if search_hash not in pseudo_cache.keys() or search_request.offset == 0: # If it's a new search (which starts with offset 0) do it again instead of using the cached results
logger.debug("Didn't find this query in cache or want to do a new search")
cache_entry = {"results": [], "indexer_infos": {}, "total": 0, "last_access": arrow.utcnow(), "offset": 0, "rejected": SearchModule.getRejectedCountDict(), "usedFallback": False, "offsetFallback": 0}
category = categoryResult.category
indexers_to_call = pick_indexers(search_request)
for p in indexers_to_call:
cache_entry["indexer_infos"][p] = {"has_more": True, "search_request": search_request, "total_included": False}
dbsearch = Search(internal=search_request.internal, query=search_request.query, category=categoryResult.category.pretty, identifier_key=search_request.identifier_key, identifier_value=search_request.identifier_value, season=search_request.season, episode=search_request.episode,
type=search_request.type, title=search_request.title, author=search_request.author, username=search_request.username)
saveSearch(dbsearch)
# dbsearch.save()
cache_entry["dbsearch"] = dbsearch
# Find ignored words and parse query for ignored words
search_request.forbiddenWords = []
search_request.requiredWords = []
applyRestrictionsGlobal = config.settings.searching.applyRestrictions == "both" or (config.settings.searching.applyRestrictions == "internal" and search_request.internal) or (config.settings.searching.applyRestrictions == "external" and not search_request.internal)
applyRestrictionsCategory = category.applyRestrictions == "both" or (category.applyRestrictions == "internal" and search_request.internal) or (search_request.category.category.applyRestrictions == "external" and not search_request.internal)
if config.settings.searching.forbiddenWords and applyRestrictionsGlobal:
logger.debug("Using configured global forbidden words: %s" % config.settings.searching.forbiddenWords)
search_request.forbiddenWords.extend([x.lower().strip() for x in list(filter(bool, config.settings.searching.forbiddenWords.split(",")))])
if config.settings.searching.requiredWords and applyRestrictionsGlobal:
logger.debug("Using configured global required words: %s" % config.settings.searching.requiredWords)
search_request.requiredWords.extend([x.lower().strip() for x in list(filter(bool, config.settings.searching.requiredWords.split(",")))])
if category.forbiddenWords and applyRestrictionsCategory:
logger.debug("Using configured forbidden words for category %s: %s" % (category.pretty, category.forbiddenWords))
search_request.forbiddenWords.extend([x.lower().strip() for x in list(filter(bool, category.forbiddenWords.split(",")))])
if category.requiredWords and applyRestrictionsCategory:
logger.debug("Using configured required words for category %s: %s" % (category.pretty, category.requiredWords))
search_request.requiredWords.extend([x.lower().strip() for x in list(filter(bool, category.requiredWords.split(",")))])
if search_request.query:
forbiddenWords = [str(x[1]) for x in re.findall(r"[\s|\b](\-\-|!)(?P<term>\w+)", search_request.query)]
if len(forbiddenWords) > 0:
logger.debug("Query before removing NOT terms: %s" % search_request.query)
search_request.query = re.sub(r"[\s|\b](\-\-|!)(?P<term>\w+)", "", search_request.query)
logger.debug("Query after removing NOT terms: %s" % search_request.query)
logger.debug("Found NOT terms: %s" % ",".join(forbiddenWords))
search_request.forbiddenWords.extend(forbiddenWords)
cache_entry["forbiddenWords"] = search_request.forbiddenWords
cache_entry["requiredWords"] = search_request.requiredWords
cache_entry["query"] = search_request.query
pseudo_cache[search_hash] = cache_entry
else:
cache_entry = pseudo_cache[search_hash]
indexers_to_call = [indexer for indexer, info in cache_entry["indexer_infos"].items() if info["has_more"]]
dbsearch = cache_entry["dbsearch"]
search_request.forbiddenWords = cache_entry["forbiddenWords"]
search_request.requiredWords = cache_entry["requiredWords"]
search_request.query = cache_entry["query"]
logger.debug("Found search in cache")
logger.debug("Will search at indexers as long as we don't have enough results for the current offset+limit and any indexer has more results.")
if search_request.loadAll:
logger.debug("Requested to load all results. Will continue to search until all indexers are exhausted")
while (len(cache_entry["results"]) < external_offset + limit or search_request.loadAll) and len(indexers_to_call) > 0:
if len(cache_entry["results"]) < external_offset + limit:
logger.debug("We want %d results but have only %d so far" % ((external_offset + limit), len(cache_entry["results"])))
elif search_request.loadAll:
logger.debug("All results requested. Continuing to search.")
logger.debug("%d indexers still have results" % len(indexers_to_call))
if cache_entry["usedFallback"]:
search_request.offset = cache_entry["offsetFallback"]
else:
search_request.offset = cache_entry["offset"]
logger.debug("Searching indexers with offset %d" % search_request.offset)
result = search_and_handle_db(dbsearch, {x: search_request for x in indexers_to_call})
logger.debug("All search calls to indexers completed")
search_results = []
indexers_to_call = []
waslocked = False
before = arrow.now()
if databaseLock.locked():
logger.debug("Database accesses locked by other search. Will wait for our turn.")
waslocked = True
databaseLock.acquire()
if waslocked:
after = arrow.now()
took = (after - before).seconds * 1000 + (after - before).microseconds / 1000
logger.debug("Waited %dms for database lock" % took)
for indexer, queries_execution_result in result["results"].items():
with db.atomic():
logger.info("%s returned %d results" % (indexer, len(queries_execution_result.results)))
for result in queries_execution_result.results:
if result.title is None or result.link is None or result.indexerguid is None:
logger.info("Skipping result with missing data: %s" % result)
continue
try:
searchResultId = hashlib.sha1(str(indexer.indexer.id) + result.indexerguid.encode('ascii', 'ignore').decode("utf-8")).hexdigest()
tryGetOrCreateSearchResultDbEntry(searchResultId, indexer.indexer.id, result)
result.searchResultId = searchResultId
search_results.append(result)
except (IntegrityError, OperationalError) as e:
logger.error("Error while trying to save search result to database. Skipping it. Error: %s" % e)
cache_entry["indexer_infos"][indexer].update(
{"did_search": queries_execution_result.didsearch, "indexer": indexer.name, "search_request": search_request, "has_more": queries_execution_result.has_more, "total": queries_execution_result.total, "total_known": queries_execution_result.total_known,
"indexer_search": queries_execution_result.indexerSearchEntry, "rejected": queries_execution_result.rejected, "processed_results": queries_execution_result.loaded_results})
if queries_execution_result.has_more:
indexers_to_call.append(indexer)
logger.debug("%s still has more results so we could use it the next round" % indexer)
if queries_execution_result.total_known:
if not cache_entry["indexer_infos"][indexer]["total_included"]:
cache_entry["total"] += queries_execution_result.total
logger.debug("%s reports %d total results. We'll include in the total this time only" % (indexer, queries_execution_result.total))
cache_entry["indexer_infos"][indexer]["total_included"] = True
elif queries_execution_result.has_more:
logger.debug("%s doesn't report an exact number of results so let's just add another 100 to the total" % indexer)
cache_entry["total"] += 100
for rejectKey in cache_entry["rejected"].keys():
if rejectKey in cache_entry["indexer_infos"][indexer]["rejected"].keys():
cache_entry["rejected"][rejectKey] += cache_entry["indexer_infos"][indexer]["rejected"][rejectKey]
databaseLock.release()
logger.debug("Searching for duplicates")
numberResultsBeforeDuplicateRemoval = len(search_results)
grouped_by_sameness, uniqueResultsPerIndexer = find_duplicates(search_results)
allresults = []
for group in grouped_by_sameness:
if search_request.internal:
for i in group:
# We give each group of results a unique value by which they can be identified later
i.hash = hash(group[0].details_link)
allresults.append(i)
else:
# We sort by age first and then by indexerscore so the newest result with the highest indexer score is chosen
group = sorted(group, key=lambda x: x.epoch, reverse=True)
group = sorted(group, key=lambda x: x.indexerscore, reverse=True)
allresults.append(group[0])
search_results = allresults
with databaseLock:
for indexer, result_infos in cache_entry["indexer_infos"].iteritems():
if indexer.name in uniqueResultsPerIndexer.keys(): # If the search failed it isn't contained in the duplicates list
uniqueResultsCount = uniqueResultsPerIndexer[result_infos["indexer"]]
processedResults = result_infos["processed_results"]
logger.debug("Indexer %s had a unique results share of %d%% (%d of %d total results were only provided by this indexer)" % (indexer.name, 100 / (numberResultsBeforeDuplicateRemoval / uniqueResultsCount), uniqueResultsCount, numberResultsBeforeDuplicateRemoval))
result_infos["indexer_search"].uniqueResults = uniqueResultsCount
result_infos["indexer_search"].processedResults = processedResults
result_infos["indexer_search"].save()
if not search_request.internal:
countAfter = len(search_results)
countRemoved = numberResultsBeforeDuplicateRemoval - countAfter
logger.info("Removed %d duplicates from %d results" % (countRemoved, numberResultsBeforeDuplicateRemoval))
search_results = sorted(search_results, key=lambda x: x.epoch, reverse=True)
cache_entry["results"].extend(search_results)
if cache_entry["usedFallback"]:
cache_entry["offsetFallback"] += limit
else:
cache_entry["offset"] += limit
if len(indexers_to_call) == 0 and not cache_entry["usedFallback"] and search_request.identifier_key is not None and ("internal" if search_request.internal else "external") in config.settings.searching.idFallbackToTitle:
call_fallback = False
if config.settings.searching.idFallbackToTitlePerIndexer:
indexers_to_call_fallback = [indexer for indexer, _ in cache_entry["indexer_infos"].items() if cache_entry["indexer_infos"][indexer]["processed_results"] == 0]
if len(indexers_to_call_fallback) > 0:
logger.info("One or more indexers found no results found using ID based search. Getting title from ID to fall back")
logger.info("Fallback indexers: %s" % (str(indexers_to_call_fallback)))
call_fallback = True
else:
if len(cache_entry["results"]) == 0:
logger.info("No results found using ID based search. Getting title from ID to fall back")
call_fallback = True
if call_fallback:
title = infos.convertId(search_request.identifier_key, "title", search_request.identifier_value)
if title:
logger.info("Repeating search with title based query as fallback")
logger.info("Fallback title: %s" % (title))
# Add title and remove identifier key/value from search
search_request.title = title
search_request.identifier_key = None
search_request.identifier_value = None
if config.settings.searching.idFallbackToTitlePerIndexer:
indexers_to_call = indexers_to_call_fallback
else:
indexers_to_call = [indexer for indexer, _ in cache_entry["indexer_infos"].items()]
cache_entry["usedFallback"] = True
else:
logger.info("Unable to find title for ID")
if len(indexers_to_call) == 0:
logger.info("All indexers exhausted")
elif len(cache_entry["results"]) >= external_offset + limit:
logger.debug("Loaded a total of %d results which is enough for the %d requested. Stopping search." % (len(cache_entry["results"]), (external_offset + limit)))
if search_request.internal:
logger.debug("We have %d cached results and return them all because we search internally" % len(cache_entry["results"]))
nzb_search_results = copy.deepcopy(cache_entry["results"][external_offset:])
else:
logger.debug("We have %d cached results and return %d-%d of %d total available accounting for the limit set for the API search" % (len(cache_entry["results"]), external_offset, external_offset + limit, cache_entry["total"]))
nzb_search_results = copy.deepcopy(cache_entry["results"][external_offset:(external_offset + limit)])
cache_entry["last_access"] = arrow.utcnow()
for k, v in cache_entry["rejected"].items():
if v > 0:
logger.info("Rejected %d results %s" % (v, k))
logger.info("Returning %d results" % len(nzb_search_results))
return {"results": nzb_search_results, "indexer_infos": cache_entry["indexer_infos"], "dbsearchid": cache_entry["dbsearch"].id, "total": cache_entry["total"], "offset": external_offset, "rejected": cache_entry["rejected"].items()}
def pick_indexers(search_request):
# type: (nzbhydra.search.SearchRequest, bool) -> List[nzbhydra.search_modules.SearchModule]
query_supplied = True if search_request.query else False
queryCanBeGenerated = None # Store if we can generate a query from IDs. Initiall true but when we need this the first time and query generation fails we set it to false
picked_indexers = []
selected_indexers = search_request.indexers.split("|") if search_request.indexers is not None else None
notPickedReasons = {}
for p in indexers.enabled_indexers:
if not p.settings.enabled:
logger.debug("Did not pick %s because it is disabled" % p)
add_not_picked_indexer(notPickedReasons, "Disabled", p.name)
continue
if search_request.internal and p.settings.accessType == "external":
logger.debug("Did not pick %s because it is only enabled for external searches" % p)
add_not_picked_indexer(notPickedReasons, "Disabled for API searches", p.name)
continue
if not search_request.internal and p.settings.accessType == "internal":
logger.debug("Did not pick %s because it is only enabled for internal searches" % p)
add_not_picked_indexer(notPickedReasons, "Disabled for API searches", p.name)
continue
if selected_indexers and p.name not in selected_indexers:
logger.debug("Did not pick %s because it was not selected by the user" % p)
add_not_picked_indexer(notPickedReasons, "Not selected by user", p.name)
continue
try:
status = p.indexer.status.get()
if status.disabled_until and status.disabled_until > arrow.utcnow() and not config.settings.searching.ignoreTemporarilyDisabled:
logger.info("Did not pick %s because it is disabled temporarily due to an error: %s" % (p, status.reason))
add_not_picked_indexer(notPickedReasons, "Temporarily disabled", p.name)
continue
except IndexerStatus.DoesNotExist:
pass
if hasattr(p.settings, "categories") and len(p.settings.categories) > 0:
if search_request.category.category.name != "all" and search_request.category.category.name not in p.settings.categories:
logger.debug("Did not pick %s because it is not enabled for category %s" % (p, search_request.category.category.pretty))
add_not_picked_indexer(notPickedReasons, "Disabled for this category %s" % search_request.category.category.pretty, p.name)
continue
if p.settings.hitLimit > 0:
if p.settings.hitLimitResetTime:
comparisonTime = arrow.utcnow().replace(hour=p.settings.hitLimitResetTime, minute=0, second=0)
if comparisonTime > arrow.utcnow():
comparisonTime = arrow.get(comparisonTime.datetime - datetime.timedelta(days=1)) # Arrow is too dumb to properly subtract 1 day (throws an error on every first of the month)
else:
# Use rolling time window
comparisonTime = arrow.get(arrow.utcnow().datetime - datetime.timedelta(days=1))
apiHits = IndexerApiAccess().select().where((IndexerApiAccess.indexer == p.indexer) & (IndexerApiAccess.time > comparisonTime) & IndexerApiAccess.response_successful).count()
if apiHits >= p.settings.hitLimit:
if p.settings.hitLimitResetTime:
logger.info("Did not pick %s because its API hit limit of %d was reached. Will pick again after %02d:00" % (p, p.settings.hitLimit, p.settings.hitLimitResetTime))
else:
try:
firstHitTimeInWindow = IndexerApiAccess().select().where(IndexerApiAccess.indexer == p.indexer & IndexerApiAccess.response_successful).order_by(IndexerApiAccess.time.desc()).offset(p.settings.hitLimit).limit(1).get().time.datetime
nextHitAfter = arrow.get(firstHitTimeInWindow + datetime.timedelta(days=1))
logger.info("Did not pick %s because its API hit limit of %d was reached. Next possible hit at %s" % (p, p.settings.hitLimit, nextHitAfter.format('YYYY-MM-DD HH:mm')))
except IndexerApiAccess.DoesNotExist:
logger.info("Did not pick %s because its API hit limit of %d was reached" % (p, p.settings.hitLimit))
add_not_picked_indexer(notPickedReasons, "API limit reached", p.name)
continue
else:
logger.debug("%s has had %d of a maximum of %d API hits since %02d:%02d" % (p, apiHits, p.settings.hitLimit, comparisonTime.hour, comparisonTime.minute))
if (query_supplied or search_request.identifier_key is not None) and not p.supports_queries:
logger.debug("Did not pick %s because a query was supplied but the indexer does not support queries" % p)
add_not_picked_indexer(notPickedReasons, "Does not support queries", p.name)
continue
# Here on we check if we could supply the indexer with generated/retrieved data like the title of a series
if not query_supplied and p.needs_queries and search_request.identifier_key is None:
logger.debug("Did not pick %s because no query was supplied but the indexer needs queries" % p)
add_not_picked_indexer(notPickedReasons, "Query needed", p.name)
continue
# If we can theoretically do that we must try to actually get the title, otherwise the indexer won't be able to search
allow_query_generation = (config.InternalExternalSelection.internal in config.settings.searching.generate_queries and search_request.internal) or (config.InternalExternalSelection.external in config.settings.searching.generate_queries and not search_request.internal)
if search_request.identifier_key is not None and not canUseIdKey(p, search_request.identifier_key):
if not (allow_query_generation and p.generate_queries):
logger.debug("Did not pick %s because search will be done by an identifier and the indexer or system wide settings don't allow query generation" % p)
add_not_picked_indexer(notPickedReasons, "Does not support ID based searches", p.name)
continue
else:
if queryCanBeGenerated is None:
try:
title = infos.convertId(search_request.identifier_key, "title", search_request.identifier_value)
if title:
search_request.title = title
queryCanBeGenerated = True
else:
queryCanBeGenerated = False
except:
queryCanBeGenerated = False
logger.debug("Unable to get title for supplied ID. Indexers that don't support the ID will be skipped")
if not queryCanBeGenerated:
logger.debug("Did not pick %s because search will be done by an identifier and retrieval of the title for query generation failed" % p)
add_not_picked_indexer(notPickedReasons, "Does not support ID based searches", p.name)
continue
logger.debug("Picked %s" % p)
picked_indexers.append(p)
if len(picked_indexers) == 0:
warning = "No indexeres were selected for this search:"
for reason, notPickedIndexers in notPickedReasons.items():
warning += "\r\n%s: %s" % (reason, ", ".join(notPickedIndexers))
logger.warn(warning)
return picked_indexers
def testGetMovieTitleDoesNotExist(self):
MovieIdCache.delete().execute()
title = infos.convertId("imdb", "title", "016954739339")
self.assertIsNone(title)
def pick_indexers(search_request):
query_supplied = True if search_request.query else False
queryCanBeGenerated = None # Store if we can generate a query from IDs. Initiall true but when we need this the first time and query generation fails we set it to false
picked_indexers = []
selected_indexers = search_request.indexers.split("|") if search_request.indexers is not None else None
notPickedReasons = {}
for p in indexers.enabled_indexers:
if not p.settings.enabled:
logger.debug("Did not pick %s because it is disabled" % p)
add_not_picked_indexer(notPickedReasons, "Disabled", p.name)
continue
if search_request.internal and p.settings.accessType == "external":
logger.debug("Did not pick %s because it is only enabled for external searches" % p)
add_not_picked_indexer(notPickedReasons, "Disabled for API searches", p.name)
continue
if not search_request.internal and p.settings.accessType == "internal":
logger.debug("Did not pick %s because it is only enabled for internal searches" % p)
add_not_picked_indexer(notPickedReasons, "Disabled for API searches", p.name)
continue
if selected_indexers and p.name not in selected_indexers:
logger.debug("Did not pick %s because it was not selected by the user" % p)
add_not_picked_indexer(notPickedReasons, "Not selected by user", p.name)
continue
try:
status = p.indexer.status.get()
if status.disabled_until and status.disabled_until > arrow.utcnow() and not config.settings.searching.ignoreTemporarilyDisabled:
logger.info("Did not pick %s because it is disabled temporarily due to an error: %s" % (p, status.reason))
add_not_picked_indexer(notPickedReasons, "Temporarily disabled", p.name)
continue
except IndexerStatus.DoesNotExist:
pass
if hasattr(p.settings, "categories") and len(p.settings.categories) > 0:
if search_request.category.category.name != "all" and search_request.category.category.name not in p.settings.categories:
logger.debug("Did not pick %s because it is not enabled for category %s" % (p, search_request.category.category.pretty))
add_not_picked_indexer(notPickedReasons, "Disabled for this category %s" % search_request.category.category.pretty, p.name)
continue
picked, reason = checkHitOrDownloadLimit(p)
if not picked:
add_not_picked_indexer(notPickedReasons, reason, p.name)
continue
if (query_supplied or search_request.identifier_key is not None) and not p.supports_queries:
logger.debug("Did not pick %s because a query was supplied but the indexer does not support queries" % p)
add_not_picked_indexer(notPickedReasons, "Does not support queries", p.name)
continue
# Here on we check if we could supply the indexer with generated/retrieved data like the title of a series
if not query_supplied and p.needs_queries and search_request.identifier_key is None:
logger.debug("Did not pick %s because no query was supplied but the indexer needs queries" % p)
add_not_picked_indexer(notPickedReasons, "Query needed", p.name)
continue
if p.settings.loadLimitOnRandom and not search_request.internal and not random.randint(1, p.settings.loadLimitOnRandom) == 1:
logger.debug("Did not pick %s because load limiting prevented it. Chances of this indexer being picked: %d/%d" % (p, 1, p.settings.loadLimitOnRandom))
add_not_picked_indexer(notPickedReasons, "Load limiting", p.name)
continue
# If we can theoretically do that we must try to actually get the title, otherwise the indexer won't be able to search
allow_query_generation = (config.InternalExternalSelection.internal in config.settings.searching.generate_queries and search_request.internal) or (config.InternalExternalSelection.external in config.settings.searching.generate_queries and not search_request.internal)
if search_request.identifier_key is not None and not canUseIdKey(p, search_request.identifier_key):
if not (allow_query_generation and p.generate_queries):
logger.debug("Did not pick %s because search will be done by an identifier and the indexer or system wide settings don't allow query generation" % p)
add_not_picked_indexer(notPickedReasons, "Does not support searching by the supplied ID %s and query generation is not allowed" % search_request.identifier_key, p.name)
continue
else:
if queryCanBeGenerated is None:
try:
title = infos.convertId(search_request.identifier_key, "title", search_request.identifier_value)
if title:
search_request.title = title
queryCanBeGenerated = True
else:
queryCanBeGenerated = False
except:
queryCanBeGenerated = False
logger.debug("Unable to get title for supplied ID. Indexers that don't support the ID will be skipped")
if not queryCanBeGenerated:
logger.debug("Did not pick %s because search will be done by an identifier and retrieval of the title for query generation or conversion of the ID failed" % p)
add_not_picked_indexer(notPickedReasons, "Does not support searching by the supplied ID %s and query generation or ID conversion is not possible" % search_request.identifier_key, p.name)
continue
logger.debug("Picked %s" % p)
picked_indexers.append(p)
allNotPickedIndexers = Set([item for sublist in notPickedReasons.values() for item in sublist])
notPickedIndexersListString = ""
for reason, notPickedIndexers in notPickedReasons.items():
notPickedIndexersListString = "\r\n%s: %s" % (reason, ", ".join(notPickedIndexers))
if len(allNotPickedIndexers) == 0:
logger.warn("No indexers were selected for this search:" + notPickedIndexersListString)
elif len(allNotPickedIndexers) > 0:
logger.info("Some indexers were not selected for this search: " + notPickedIndexersListString)
return picked_indexers
def pick_indexers(search_request):
# type: (nzbhydra.search.SearchRequest, bool) -> List[nzbhydra.search_modules.SearchModule]
query_supplied = True if search_request.query else False
queryCanBeGenerated = None # Store if we can generate a query from IDs. Initiall true but when we need this the first time and query generation fails we set it to false
picked_indexers = []
selected_indexers = search_request.indexers.split("|") if search_request.indexers is not None else None
for p in indexers.enabled_indexers:
if not p.settings.enabled:
logger.debug("Did not pick %s because it is disabled" % p)
continue
if search_request.internal and p.settings.accessType == "external":
logger.debug("Did not pick %s because it is only enabled for external searches" % p)
continue
if not search_request.internal and p.settings.accessType == "internal":
logger.debug("Did not pick %s because it is only enabled for internal searches" % p)
continue
if selected_indexers and p.name not in selected_indexers:
logger.debug("Did not pick %s because it was not selected by the user" % p)
continue
try:
status = p.indexer.status.get()
if status.disabled_until > arrow.utcnow() and not config.settings.searching.ignoreTemporarilyDisabled:
logger.info("Did not pick %s because it is disabled temporarily due to an error: %s" % (p, status.reason))
continue
except IndexerStatus.DoesNotExist:
pass
if p.settings.hitLimit > 0:
hitLimitResetTime = arrow.get(p.settings.hitLimitResetTime)
if p.settings.hitLimitResetTime:
comparisonTime = arrow.utcnow().replace(hour=hitLimitResetTime.hour, minute=hitLimitResetTime.minute, second=0)
if comparisonTime > arrow.utcnow():
comparisonTime = arrow.get(comparisonTime.datetime - datetime.timedelta(days=1)) # Arrow is too dumb to properly subtract 1 day (throws an error on every first of the month)
else:
# Use 00:00 as reset time when no reset time is set
comparisonTime = arrow.now().replace(hour=0, minute=0, second=0)
apiHits = IndexerApiAccess().select().where((IndexerApiAccess.indexer == p.indexer) & (IndexerApiAccess.time > comparisonTime) & IndexerApiAccess.response_successful).count()
if apiHits > p.settings.hitLimit:
logger.info("Did not pick %s because its API hit limit of %d was reached. Will pick again after %02d:%02d tomorrow" % (p, p.settings.hitLimit, hitLimitResetTime.hour, hitLimitResetTime.minute))
continue
else:
logger.debug("%s has had %d of a maximum of %d API hits since %02d:%02d" % (p, apiHits, p.settings.hitLimit, comparisonTime.hour, comparisonTime.minute))
if (query_supplied or search_request.identifier_key is not None) and not p.supports_queries:
logger.debug("Did not pick %s because a query was supplied but the indexer does not support queries" % p)
continue
# Here on we check if we could supply the indexer with generated/retrieved data like the title of a series
if not query_supplied and p.needs_queries and search_request.identifier_key is None:
logger.debug("Did not pick %s because no query was supplied but the indexer needs queries" % p)
continue
# If we can theoretically do that we must try to actually get the title, otherwise the indexer won't be able to search
allow_query_generation = (config.InternalExternalSelection.internal in config.settings.searching.generate_queries and search_request.internal) or (config.InternalExternalSelection.external in config.settings.searching.generate_queries and not search_request.internal)
if search_request.identifier_key is not None and not canUseIdKey(p, search_request.identifier_key):
if not (allow_query_generation and p.generate_queries):
logger.debug("Did not pick %s because search will be done by an identifier and the indexer or system wide settings don't allow query generation" % p)
continue
else:
if queryCanBeGenerated is None:
try:
title = infos.convertId(search_request.identifier_key, "title", search_request.identifier_value)
if title:
search_request.title = title
queryCanBeGenerated = True
else:
queryCanBeGenerated = False
except:
queryCanBeGenerated = False
logger.debug("Unable to get title for supplied ID. Indexers that don't support the ID will be skipped")
if not queryCanBeGenerated:
logger.debug("Did not pick %s because search will be done by an identifier and retrieval of the title for query generation failed" % p)
continue
logger.debug("Picked %s" % p)
picked_indexers.append(p)
return picked_indexers
def internalapi_redirect_rid(args):
logger.debug("Redirect TVRage id request")
tvdbid = infos.convertId("tvrage", "tvdb", args["rid"])
if tvdbid is None:
return "Unable to find TVDB link for TVRage ID", 404
return redirect("https://thetvdb.com/?tab=series&id=%s" % tvdbid)
def pick_indexers(search_request):
query_supplied = True if search_request.query else False
queryCanBeGenerated = None # Store if we can generate a query from IDs. Initiall true but when we need this the first time and query generation fails we set it to false
picked_indexers = []
selected_indexers = search_request.indexers.split("|") if search_request.indexers is not None else None
notPickedReasons = {}
for p in indexers.enabled_indexers:
if not p.settings.enabled:
logger.debug("Did not pick %s because it is disabled" % p)
add_not_picked_indexer(notPickedReasons, "Disabled", p.name)
continue
if search_request.internal and p.settings.accessType == "external":
logger.debug("Did not pick %s because it is only enabled for external searches" % p)
add_not_picked_indexer(notPickedReasons, "Disabled for API searches", p.name)
continue
if not search_request.internal and p.settings.accessType == "internal":
logger.debug("Did not pick %s because it is only enabled for internal searches" % p)
add_not_picked_indexer(notPickedReasons, "Disabled for API searches", p.name)
continue
if selected_indexers and p.name not in selected_indexers:
logger.debug("Did not pick %s because it was not selected by the user" % p)
add_not_picked_indexer(notPickedReasons, "Not selected by user", p.name)
continue
try:
status = p.indexer.status.get()
if status.disabled_until and status.disabled_until > arrow.utcnow() and not config.settings.searching.ignoreTemporarilyDisabled:
logger.info("Did not pick %s because it is disabled temporarily due to an error: %s" % (p, status.reason))
add_not_picked_indexer(notPickedReasons, "Temporarily disabled", p.name)
continue
except IndexerStatus.DoesNotExist:
pass
if hasattr(p.settings, "categories") and len(p.settings.categories) > 0:
if search_request.category.category.name != "all" and search_request.category.category.name not in p.settings.categories:
logger.debug("Did not pick %s because it is not enabled for category %s" % (p, search_request.category.category.pretty))
add_not_picked_indexer(notPickedReasons, "Disabled for this category %s" % search_request.category.category.pretty, p.name)
continue
picked, reason = checkHitOrDownloadLimit(p)
if not picked:
add_not_picked_indexer(notPickedReasons, reason, p.name)
continue
if (query_supplied or search_request.identifier_key is not None) and not p.supports_queries:
logger.debug("Did not pick %s because a query was supplied but the indexer does not support queries" % p)
add_not_picked_indexer(notPickedReasons, "Does not support queries", p.name)
continue
# Here on we check if we could supply the indexer with generated/retrieved data like the title of a series
if not query_supplied and p.needs_queries and search_request.identifier_key is None:
logger.debug("Did not pick %s because no query was supplied but the indexer needs queries" % p)
add_not_picked_indexer(notPickedReasons, "Query needed", p.name)
continue
# If we can theoretically do that we must try to actually get the title, otherwise the indexer won't be able to search
allow_query_generation = (config.InternalExternalSelection.internal in config.settings.searching.generate_queries and search_request.internal) or (config.InternalExternalSelection.external in config.settings.searching.generate_queries and not search_request.internal)
if search_request.identifier_key is not None and not canUseIdKey(p, search_request.identifier_key):
if not (allow_query_generation and p.generate_queries):
logger.debug("Did not pick %s because search will be done by an identifier and the indexer or system wide settings don't allow query generation" % p)
add_not_picked_indexer(notPickedReasons, "Does not support searching by the supplied ID %s and query generation is not allowed" % search_request.identifier_key, p.name)
continue
else:
if queryCanBeGenerated is None:
try:
title = infos.convertId(search_request.identifier_key, "title", search_request.identifier_value)
if title:
search_request.title = title
queryCanBeGenerated = True
else:
queryCanBeGenerated = False
except:
queryCanBeGenerated = False
logger.debug("Unable to get title for supplied ID. Indexers that don't support the ID will be skipped")
if not queryCanBeGenerated:
logger.debug("Did not pick %s because search will be done by an identifier and retrieval of the title for query generation or conversion of the ID failed" % p)
add_not_picked_indexer(notPickedReasons, "Does not support searching by the supplied ID %s and query generation or ID conversion is not possible" % search_request.identifier_key, p.name)
continue
logger.debug("Picked %s" % p)
picked_indexers.append(p)
allNotPickedIndexers = Set([item for sublist in notPickedReasons.values() for item in sublist])
notPickedIndexersListString = ""
for reason, notPickedIndexers in notPickedReasons.items():
notPickedIndexersListString = "\r\n%s: %s" % (reason, ", ".join(notPickedIndexers))
if len(allNotPickedIndexers) == 0:
logger.warn("No indexers were selected for this search:" + notPickedIndexersListString)
elif len(allNotPickedIndexers) > 0:
logger.info("Some indexers were not selected for this search: " + notPickedIndexersListString)
return picked_indexers
def testGetTvTitleDoesntExist(self):
TvIdCache.delete().execute()
id = infos.convertId("tvdb", "title", "299350000")
self.assertIsNone(id)
def internalapi_redirect_rid(args):
logger.debug("Redirect TVRage id request")
tvdbid = infos.convertId("tvrage", "tvdb", args["rid"])
if tvdbid is None:
return "Unable to find TVDB link for TVRage ID", 404
return redirect("https://thetvdb.com/?tab=series&id=%s" % tvdbid)