def download_nzb_and_log(indexer_name, provider_guid, title, searchid):
"""
Gets the NZB link from the indexer using the guid, downloads it and logs the download
:param indexer_name: name of the indexer
:param provider_guid: guid to build link
:param title: the title to build the link
:param searchid: the id of the IndexerSearch entry so we can link the download to a search
:return: IndexerNzbDownloadResult
"""
for p in indexers.enabled_indexers:
if p.name == indexer_name:
link = p.get_nzb_link(provider_guid, title)
indexer = Indexer.get(fn.lower(Indexer.name) == indexer_name.lower())
psearch = IndexerSearch.get((IndexerSearch.indexer == indexer) & (IndexerSearch.search == searchid))
papiaccess = IndexerApiAccess(indexer=p.indexer, type="nzb", url=link, indexer_search=psearch)
papiaccess.save()
internallink, guid = get_nzb_link_and_guid(indexer_name, provider_guid, searchid, title)
pnzbdl = IndexerNzbDownload(indexer=indexer, indexer_search=searchid, api_access=papiaccess, mode="serve", title=title, guid=internallink)
pnzbdl.save()
try:
r = p.get(link, timeout=10)
r.raise_for_status()
papiaccess.response_successful = True
papiaccess.response_time = r.elapsed.microseconds / 1000
return IndexerNzbDownloadResult(content=r.content, headers=r.headers)
except RequestException as e:
logger.error("Error while connecting to URL %s: %s" % (link, str(e)))
papiaccess.error = str(e)
return None
finally:
papiaccess.save()
else:
return "Unable to find NZB link"
def execute_queries(self, queries, searchRequest):
if len(queries) == 0:
return QueriesExecutionResult(didsearch=False, results=[], indexerSearchEntry=None, indexerApiAccessEntry=None, indexerStatus=None, total=0, loaded_results=0, total_known=True, has_more=False)
results = []
executed_queries = set()
psearch = IndexerSearch(indexer=self.indexer)
papiaccess = IndexerApiAccess()
indexerStatus = None
#psearch.save()
total_results = 0
total_known = False
has_more = False
while len(queries) > 0:
query = queries.pop()
if query in executed_queries:
# To make sure that in case an offset is reported wrong or we have a bug we don't get stuck in an endless loop
continue
try:
request, papiaccess, indexerStatus = self.get_url_with_papi_access(query, "search", saveToDb=False)
papiaccess.indexer_search = psearch
executed_queries.add(query)
#papiaccess.save()
if request is not None:
self.check_auth(request.text)
self.debug("Successfully loaded URL %s" % request.url)
try:
parsed_results = self.process_query_result(request.content, searchRequest)
results.extend(parsed_results.entries) # Retrieve the processed results
queries.extend(parsed_results.queries) # Add queries that were added as a result of the parsing, e.g. when the next result page should also be loaded
total_results += parsed_results.total
total_known = parsed_results.total_known
has_more = parsed_results.has_more
papiaccess.response_successful = True
self.handle_indexer_success(False)
except IndexerResultParsingException as e:
self.error("Error while processing search results from indexer %s" % e)
except Exception:
self.exception("Error while processing search results from indexer %s" % self)
raise IndexerResultParsingException("Error while parsing the results from indexer", self)
except IndexerAuthException as e:
self.error("Unable to authorize with %s: %s" % (e.search_module, e.message))
papiaccess.error = "Authorization error :%s" % e.message
self.handle_indexer_failure(reason="Authentication failed", disable_permanently=True)
papiaccess.response_successful = False
except IndexerAccessException as e:
self.error("Unable to access %s: %s" % (e.search_module, e.message))
papiaccess.error = "Access error: %s" % e.message
self.handle_indexer_failure(reason="Access failed")
papiaccess.response_successful = False
except IndexerResultParsingException as e:
papiaccess.exception = "Access error: %s" % e.message
self.handle_indexer_failure(reason="Parsing results failed")
papiaccess.response_successful = False
except Exception as e:
self.exception("An error error occurred while searching: %s", e)
if papiaccess is not None:
papiaccess.error = "Unknown error :%s" % e
papiaccess.response_successful = False
finally:
if papiaccess is not None:
#papiaccess.save()
psearch.successful = papiaccess.response_successful
else:
self.error("Unable to save API response to database")
psearch.resultsCount = total_results
#psearch.save()
return QueriesExecutionResult(didsearch= True, results=results, indexerSearchEntry=psearch, indexerApiAccessEntry=papiaccess, indexerStatus=indexerStatus, total=total_results, loaded_results=len(results), total_known=total_known, has_more=has_more)
def execute_queries(self, queries, searchRequest):
if len(queries) == 0:
return QueriesExecutionResult(didsearch=False,
results=[],
indexerSearchEntry=None,
indexerApiAccessEntry=None,
indexerStatus=None,
total=0,
loaded_results=0,
total_known=True,
has_more=False,
rejected=self.getRejectedCountDict())
results = []
executed_queries = set()
psearch = IndexerSearch(indexer=self.indexer)
papiaccess = IndexerApiAccess()
indexerStatus = None
total_results = 0
total_known = False
has_more = False
rejected = self.getRejectedCountDict()
while len(queries) > 0:
query = queries.pop()
if query in executed_queries:
# To make sure that in case an offset is reported wrong or we have a bug we don't get stuck in an endless loop
continue
try:
request, papiaccess, indexerStatus = self.get_url_with_papi_access(
query, "search", saveToDb=False)
papiaccess.indexer_search = psearch
executed_queries.add(query)
if request is not None:
if request.text == "":
raise IndexerResultParsingException(
"Indexer returned an empty page", self)
self.check_auth(request.text)
self.debug("Successfully loaded URL %s" % request.url)
try:
parsed_results = self.process_query_result(
request.content, searchRequest)
results.extend(parsed_results.entries
) # Retrieve the processed results
queries.extend(
parsed_results.queries
) # Add queries that were added as a result of the parsing, e.g. when the next result page should also be loaded
total_results += parsed_results.total
total_known = parsed_results.total_known
has_more = parsed_results.has_more
rejected = parsed_results.rejected
papiaccess.response_successful = True
indexerStatus = self.handle_indexer_success(False)
except Exception:
self.exception(
"Error while processing search results from indexer %s"
% self)
raise IndexerResultParsingException(
"Error while parsing the results from indexer",
self)
except IndexerAuthException as e:
papiaccess.error = "Authorization error :%s" % e.message
self.error(papiaccess.error)
indexerStatus = self.handle_indexer_failure(
reason="Authentication failed", disable_permanently=True)
papiaccess.response_successful = False
except IndexerAccessException as e:
papiaccess.error = "Access error: %s" % e.message
self.error(papiaccess.error)
indexerStatus = self.handle_indexer_failure(
reason="Access failed")
papiaccess.response_successful = False
except IndexerResultParsingException as e:
papiaccess.error = "Access error: %s" % e.message
self.error(papiaccess.error)
indexerStatus = self.handle_indexer_failure(
reason="Parsing results failed")
papiaccess.response_successful = False
except Exception as e:
self.exception("An error error occurred while searching: %s",
e)
if papiaccess is not None:
papiaccess.error = "Unknown error :%s" % e
papiaccess.response_successful = False
finally:
if papiaccess is not None:
psearch.successful = papiaccess.response_successful
else:
self.error("Unable to save API response to database")
psearch.resultsCount = total_results
return QueriesExecutionResult(didsearch=True,
results=results,
indexerSearchEntry=psearch,
indexerApiAccessEntry=papiaccess,
indexerStatus=indexerStatus,
total=total_results,
loaded_results=len(results),
total_known=total_known,
has_more=has_more,
rejected=rejected)
def testThatDatabaseValuesAreStored(self):
with self.app.test_request_context('/'):
with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
newznabItems = [
[mockbuilder.buildNewznabItem(title="title1", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1")],
[mockbuilder.buildNewznabItem(title="title2", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2")]
]
self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems)
# Make the second access unsuccessful
rsps._urls.pop(1)
rsps.add(responses.GET, r".*",
body="an error message", status=500,
content_type='application/x-html')
searchRequest = SearchRequest(type="search", query="aquery", category="acategory", identifier_key="imdbid", identifier_value="animdbid", season=1, episode=2, indexers="newznab1|newznab2")
result = search.search(searchRequest)
results = result["results"]
self.assertEqual(1, len(results))
dbSearch = Search().get()
self.assertEqual(True, dbSearch.internal)
self.assertEqual("aquery", dbSearch.query)
self.assertEqual("All", dbSearch.category)
self.assertEqual("imdbid", dbSearch.identifier_key)
self.assertEqual("animdbid", dbSearch.identifier_value)
self.assertEqual("1", dbSearch.season)
self.assertEqual("2", dbSearch.episode)
self.assertEqual("search", dbSearch.type)
self.assertEqual(18, dbSearch.time.hour)
indexerSearch1 = IndexerSearch.get(IndexerSearch.indexer == Indexer.get(Indexer.name == "newznab1"))
self.assertEqual(indexerSearch1.search, dbSearch)
self.assertEqual(18, indexerSearch1.time.hour)
indexerSearch2 = IndexerSearch.get(IndexerSearch.indexer == Indexer.get(Indexer.name == "newznab2"))
self.assertEqual(indexerSearch2.search, dbSearch)
self.assertEqual(18, indexerSearch2.time.hour)
calledUrls = sorted([x.request.url for x in rsps.calls])
indexerApiAccess1 = IndexerApiAccess.get(IndexerApiAccess.indexer == Indexer.get(Indexer.name == "newznab1"))
self.assertEqual(indexerSearch1, indexerApiAccess1.indexer_search)
self.assertEqual(18, indexerApiAccess1.time.hour)
self.assertEqual("search", indexerApiAccess1.type)
self.assertEqual(calledUrls[0], indexerApiAccess1.url)
self.assertTrue(indexerApiAccess1.response_successful)
self.assertEqual(0, indexerApiAccess1.response_time)
self.assertIsNone(indexerApiAccess1.error)
indexerApiAccess2 = IndexerApiAccess.get(IndexerApiAccess.indexer == Indexer.get(Indexer.name == "newznab2"))
self.assertEqual(indexerSearch2, indexerApiAccess2.indexer_search)
self.assertEqual(18, indexerApiAccess2.time.hour)
self.assertEqual("search", indexerApiAccess2.type)
self.assertEqual(calledUrls[1], indexerApiAccess2.url)
self.assertFalse(indexerApiAccess2.response_successful)
self.assertIsNone(indexerApiAccess2.response_time)
self.assertTrue("Connection refused" in indexerApiAccess2.error)
indexerStatus2 = IndexerStatus.get(IndexerStatus.indexer == Indexer.get(Indexer.name == "newznab2"))
self.assertEqual(1, indexerStatus2.level)
self.assertTrue("Connection refused" in indexerStatus2.reason)
def testThatDatabaseValuesAreStored(self):
with self.app.test_request_context('/'):
with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps:
newznabItems = [
[mockbuilder.buildNewznabItem(title="title1", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1")],
[mockbuilder.buildNewznabItem(title="title2", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2")]
]
self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems)
# Make the second access unsuccessful
rsps._urls.pop(1)
rsps.add(responses.GET, r".*",
body="an error message", status=500,
content_type='application/x-html')
searchRequest = SearchRequest(type="search", query="aquery", category="acategory", identifier_key="imdbid", identifier_value="animdbid", season=1, episode=2, indexers="newznab1|newznab2")
result = search.search(searchRequest)
results = result["results"]
self.assertEqual(1, len(results))
dbSearch = Search().get()
self.assertEqual(True, dbSearch.internal)
self.assertEqual("aquery", dbSearch.query)
self.assertEqual("All", dbSearch.category)
self.assertEqual("imdbid", dbSearch.identifier_key)
self.assertEqual("animdbid", dbSearch.identifier_value)
self.assertEqual("1", dbSearch.season)
self.assertEqual("2", dbSearch.episode)
self.assertEqual("search", dbSearch.type)
self.assertEqual(18, dbSearch.time.hour)
indexerSearch1 = IndexerSearch.get(IndexerSearch.indexer == Indexer.get(Indexer.name == "newznab1"))
self.assertEqual(indexerSearch1.search, dbSearch)
self.assertEqual(18, indexerSearch1.time.hour)
indexerSearch2 = IndexerSearch.get(IndexerSearch.indexer == Indexer.get(Indexer.name == "newznab2"))
self.assertEqual(indexerSearch2.search, dbSearch)
self.assertEqual(18, indexerSearch2.time.hour)
calledUrls = sorted([x.request.url for x in rsps.calls])
indexerApiAccess1 = IndexerApiAccess.get(IndexerApiAccess.indexer == Indexer.get(Indexer.name == "newznab1"))
self.assertEqual(indexerSearch1, indexerApiAccess1.indexer_search)
self.assertEqual(18, indexerApiAccess1.time.hour)
self.assertEqual("search", indexerApiAccess1.type)
self.assertEqual(calledUrls[0], indexerApiAccess1.url)
self.assertTrue(indexerApiAccess1.response_successful)
self.assertEqual(0, indexerApiAccess1.response_time)
self.assertIsNone(indexerApiAccess1.error)
indexerApiAccess2 = IndexerApiAccess.get(IndexerApiAccess.indexer == Indexer.get(Indexer.name == "newznab2"))
self.assertEqual(indexerSearch2, indexerApiAccess2.indexer_search)
self.assertEqual(18, indexerApiAccess2.time.hour)
self.assertEqual("search", indexerApiAccess2.type)
self.assertEqual(calledUrls[1], indexerApiAccess2.url)
self.assertFalse(indexerApiAccess2.response_successful)
self.assertIsNone(indexerApiAccess2.response_time)
self.assertTrue("Connection refused" in indexerApiAccess2.error)
indexerStatus2 = IndexerStatus.get(IndexerStatus.indexer == Indexer.get(Indexer.name == "newznab2"))
self.assertEqual(1, indexerStatus2.level)
self.assertTrue("Connection refused" in indexerStatus2.reason)
