def testDuplicateRemovalForExternalApi(self): with self.app.test_request_context('/'): with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: newznabItems = [ [mockbuilder.buildNewznabItem(title="title", pubdate=arrow.get(0000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1")], [mockbuilder.buildNewznabItem(title="title", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2")], [mockbuilder.buildNewznabItem(title="title", pubdate=arrow.get(3000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab3")], [mockbuilder.buildNewznabItem(title="title", pubdate=arrow.get(2000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab4")] ] self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) # Test that the newest result is chosen if all scores are equal searchRequest = SearchRequest(type="search", internal=False) result = search.search(searchRequest) results = result["results"] self.assertEqual(1, len(results)) self.assertEqual("newznab3", results[0].indexer) # Test that results from an indexer with a higher score are preferred self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) getIndexerSettingByName("newznab2").score = 99 searchRequest = SearchRequest(type="search", internal=False) result = search.search(searchRequest) results = result["results"] self.assertEqual(1, len(results)) self.assertEqual("newznab2", results[0].indexer)
def testDuplicateRemovalForExternalApi(self): config.settings.searching.removeDuplicatesExternal = True with self.app.test_request_context('/'): with responses.RequestsMock( assert_all_requests_are_fired=False) as rsps: newznabItems = [[ mockbuilder.buildNewznabItem( title="title", pubdate=arrow.get(0000).format( "ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1") ], [ mockbuilder.buildNewznabItem( title="title", pubdate=arrow.get(1000).format( "ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2") ], [ mockbuilder.buildNewznabItem( title="title", pubdate=arrow.get(3000).format( "ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab3") ], [ mockbuilder.buildNewznabItem( title="title", pubdate=arrow.get(2000).format( "ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab4") ]] self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) # Test that the newest result is chosen if all scores are equal searchRequest = SearchRequest(type="search", internal=False) result = search.search(searchRequest) results = result["results"] self.assertEqual(1, len(results)) self.assertEqual("newznab3", results[0].indexer) # Test that results from an indexer with a higher score are preferred self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) getIndexerSettingByName("newznab2").score = 99 searchRequest = SearchRequest(type="search", internal=False) result = search.search(searchRequest) results = result["results"] self.assertEqual(1, len(results)) self.assertEqual("newznab2", results[0].indexer)
def testDuplicateTaggingForInternalApi(self): with self.app.test_request_context('/'): with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: newznabItems = [[mockbuilder.buildNewznabItem(title="title%d" % i, pubdate=arrow.get(4000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=i, indexer_name="newznab1", guid="newznab1result%d" % i) for i in range(1, 250)], [mockbuilder.buildNewznabItem(title="title%d" % i, pubdate=arrow.get(4000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=i, indexer_name="newznab2", guid="newznab2result%d" % i) for i in range(1, 250)], [mockbuilder.buildNewznabItem(title="title%d" % i, pubdate=arrow.get(4000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=i, indexer_name="newznab3", guid="newznab3result%d" % i) for i in range(1, 250)]] self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) searchRequest = SearchRequest(type="search") result = search.search(searchRequest)
def testDuplicateTaggingForInternalApi(self): with self.app.test_request_context('/'): with responses.RequestsMock() as rsps: newznabItems = [[ mockbuilder.buildNewznabItem( title="title1", pubdate=arrow.get(1000).format( "ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1", guid="newznab1result1"), mockbuilder.buildNewznabItem( title="title2", pubdate=arrow.get(3000).format( "ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1", guid="newznab1result2") ], [ mockbuilder.buildNewznabItem( title="title1", pubdate=arrow.get(2000).format( "ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2", guid="newznab1result1"), mockbuilder.buildNewznabItem( title="title2", pubdate=arrow.get(4000).format( "ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2", guid="newznab2result2") ]] self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) searchRequest = SearchRequest(type="search") result = search.search(searchRequest) results = result["results"] self.assertEqual(4, len(results)) results = sorted(results, key=lambda x: x.hash) self.assertEqual(results[0].hash, results[1].hash) self.assertEqual(results[2].hash, results[3].hash)
def testDuplicateTaggingForInternalApi(self): with self.app.test_request_context('/'): with responses.RequestsMock() as rsps: newznabItems = [ [mockbuilder.buildNewznabItem(title="title1", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1", guid="newznab1result1"), mockbuilder.buildNewznabItem(title="title2", pubdate=arrow.get(3000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1", guid="newznab1result2")], [mockbuilder.buildNewznabItem(title="title1", pubdate=arrow.get(2000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2", guid="newznab1result1"), mockbuilder.buildNewznabItem(title="title2", pubdate=arrow.get(4000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2", guid="newznab2result2")] ] self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) searchRequest = SearchRequest(type="search") result = search.search(searchRequest) results = result["results"] self.assertEqual(4, len(results)) results = sorted(results, key=lambda x: x.hash) self.assertEqual(results[0].hash, results[1].hash) self.assertEqual(results[2].hash, results[3].hash)
def testOffsetStuff(self): mockitem_nzbs = [] for i in range(100): mockitem_nzbs.append(mockbuilder.buildNewznabItem("myId", "myTitle", "myGuid", "http://nzbs.org/myId.nzb", None, None, 12345, "NZBs.org", [2000, 2040])) mockresponse_nzbs1 = mockbuilder.buildNewznabResponse("NZBs.org", mockitem_nzbs, offset=0, total=200) mockitem_nzbs.clear() for i in range(100): mockitem_nzbs.append(mockbuilder.buildNewznabItem("myId", "myTitle", "myGuid", "http://nzbs.org/myId.nzb", None, None, 12345, "NZBs.org", [2000, 2040])) mockresponse_nzbs2 = mockbuilder.buildNewznabResponse("NZBs.org", mockitem_nzbs, offset=100, total=200) r = self.n1.process_query_result(json.dumps(mockresponse_nzbs1), "http://127.0.0.1:5001/nzbsorg/q=whatever&offset=0&limit=0") further_queries = r.queries self.assertEqual(1, len(further_queries)) assert "offset=100" in further_queries[0] r = self.n1.process_query_result(json.dumps(mockresponse_nzbs2), "http://127.0.0.1:5001/nzbsorg/q=whatever&offset=0&limit=0") further_queries = r.queries self.assertEqual(0, len(further_queries))
def testThatResultsAreSortedByAgeDescending(self): with self.app.test_request_context('/'): with responses.RequestsMock() as rsps: newznabItems = [ [mockbuilder.buildNewznabItem(title="title1", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1")], [mockbuilder.buildNewznabItem(title="title2", pubdate=arrow.get(0000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2")], [mockbuilder.buildNewznabItem(title="title3", pubdate=arrow.get(3000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab3")], [mockbuilder.buildNewznabItem(title="title4", pubdate=arrow.get(4000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab4")], [mockbuilder.buildNewznabItem(title="title5", pubdate=arrow.get(2000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab5")] ] self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) searchRequest = SearchRequest(type="search") result = search.search(searchRequest) results = result["results"] self.assertEqual("title4", results[0].title) self.assertEqual("title3", results[1].title) self.assertEqual("title5", results[2].title) self.assertEqual("title1", results[3].title) self.assertEqual("title2", results[4].title)
def prepareSearchMocks(self, requestsMock, indexerCount=2, resultsPerIndexers=1, newznabItems=None, title="newznab%dresult%d.title", categories=None, skip=None): """ :param requestsMock: :param indexerCount: :param resultsPerIndexers: :param newznabItems: :param title: :param categories: :param skip: List of tuples with indexer and result index which will not be returned :return: """ if skip is None: skip = [] allNewznabItems = [] self.response_callbacks = [] self.prepareIndexers(indexerCount) for i in range(1, indexerCount + 1): # Prepare search results if newznabItems is not None: indexerNewznabItems = newznabItems[i - 1] else: indexerNewznabItems = [ mockbuilder.buildNewznabItem( title % (i, j), "newznab%dresult%d.guid" % (i, j), " http://newznab%dresult%d.link" % (i, j), arrow.get(0).format("ddd, DD MMM YYYY HH:mm:ss Z"), "newznab%dresult%d.description" % (i, j), 1000, "newznab%d" % i, categories) for j in range(1, resultsPerIndexers + 1) if not (i, j) in skip ] allNewznabItems.extend(indexerNewznabItems) xml = mockbuilder.buildNewznabResponse("newznab%dResponse" % i, indexerNewznabItems, 0, len(indexerNewznabItems)) requestsMock.register_uri('GET', re.compile(r'.*newznab%d.*' % i), text=xml) read_indexers_from_config() allNewznabItems = sorted(allNewznabItems, key=lambda x: x.title) return allNewznabItems
def testIgnoreByCategories(self, m): web.app.template_folder = "../templates" config.settings.categories.categories[ "moviessd"].ignoreResults = "always" config.settings.categories.categories["xxx"].ignoreResults = "always" config.settings.categories.categories["pc"].ignoreResults = "internal" movieSdResult = mockbuilder.buildNewznabItem( title="result1", indexer_name="newznab1", categories=[2030] ) #MoviesSD: Is dismissed because its specific category is ignored xxxResult = mockbuilder.buildNewznabItem( title="result2", indexer_name="newznab1", categories=[6000] ) #XXX: Is dismissed because its category is ignored (no more or less specific category exists) movieResult = mockbuilder.buildNewznabItem( title="result3", indexer_name="newznab1", categories=[2000] ) #Is kept because more specific category Movies SD is ignored but not movies in general movieHdResult = mockbuilder.buildNewznabItem( title="result4", indexer_name="newznab1", categories=[2040] ) #MoviesHD: Is kept because the other specific category is ignored but not this one tvResult = mockbuilder.buildNewznabItem( title="result5", indexer_name="newznab1", categories=[ 5000 ]) #TV: Is kept because its category (tv) is never ignored pcResult = mockbuilder.buildNewznabItem( title="result6", indexer_name="newznab1", categories=[4000] ) # PC: Is kept because its category (tv) is only ignored for internal searches (but this is API) expectedItems = self.prepareSearchMocks(m, 1, newznabItems=[[ movieSdResult, xxxResult, movieResult, movieHdResult, tvResult, pcResult ]]) expectedItems = expectedItems[2:] #First two results will be dismissed with web.app.test_request_context('/api?t=search&q=query&apikey=%s' % config.settings.main.apikey): response = web.api() entries, _, _ = newznab.NewzNab( Bunch.fromDict({ "name": "forTest", "score": 0, "host": "host" })).parseXml(response.data) self.assertSearchResults(entries, expectedItems) calledUrls = sorted([x.url for x in m.request_history]) self.assertTrue( compare( 'http://www.newznab1.com/api?apikey=apikeyindexer.com&t=search&extended=1&offset=0&limit=100&q=query', calledUrls[0]), calledUrls[0])
def prepareSearchMocks(self, rsps, indexerCount=2, resultsPerIndexers=1, newznabItems=None, title="newznab%dresult%d.title", sleep=0): testData = [] self.response_callbacks = [] self.prepareIndexers(indexerCount) for i in range(1, indexerCount + 1): # Prepare search results if newznabItems is not None: indexerNewznabItems = newznabItems[i - 1] else: indexerNewznabItems = [ mockbuilder.buildNewznabItem( title % (i, j), "newznab%dresult%d.guid" % (i, j), "newznab%dresult%d.link" % (i, j), arrow.get(0).format("ddd, DD MMM YYYY HH:mm:ss Z"), "newznab%dresult%d.description" % (i, j), 1000, "newznab%d" % i, None) for j in range(1, resultsPerIndexers + 1) ] xml = mockbuilder.buildNewznabResponse("newznab%dResponse" % i, indexerNewznabItems, 0, len(indexerNewznabItems)) self.response_callbacks.append( ('newznab%d' % i, randint(0, sleep), xml)) # Prepare response mock url_re = re.compile(r'.*newznab%d.*' % i) rsps.add_callback(responses.GET, url_re, callback=self.rsps_callback, content_type='application/x-html') read_indexers_from_config() return testData
def prepareSearchMocks(self, requestsMock, indexerCount=2, resultsPerIndexers=1, newznabItems=None, title="newznab%dresult%d.title", categories=None, skip=None): """ :param requestsMock: :param indexerCount: :param resultsPerIndexers: :param newznabItems: :param title: :param categories: :param skip: List of tuples with indexer and result index which will not be returned :return: """ if skip is None: skip = [] allNewznabItems = [] self.response_callbacks = [] self.prepareIndexers(indexerCount) for i in range(1, indexerCount + 1): # Prepare search results if newznabItems is not None: indexerNewznabItems = newznabItems[i - 1] else: indexerNewznabItems = [mockbuilder.buildNewznabItem(title % (i, j), "newznab%dresult%d.guid" % (i, j), " http://newznab%dresult%d.link" % (i, j), arrow.get(0).format("ddd, DD MMM YYYY HH:mm:ss Z"), "newznab%dresult%d.description" % (i, j), 1000, "newznab%d" % i, categories) for j in range(1, resultsPerIndexers + 1) if not (i, j) in skip ] allNewznabItems.extend(indexerNewznabItems) xml = mockbuilder.buildNewznabResponse("newznab%dResponse" % i, indexerNewznabItems, 0, len(indexerNewznabItems)) requestsMock.register_uri('GET', re.compile(r'.*newznab%d.*' % i), text=xml) read_indexers_from_config() allNewznabItems = sorted(allNewznabItems, key=lambda x: x.title) return allNewznabItems
def prepareSearchMocks(self, rsps, indexerCount=2, resultsPerIndexers=1, newznabItems=None, title="newznab%dresult%d.title", sleep=0): testData = [] self.response_callbacks = [] self.prepareIndexers(indexerCount) for i in range(1, indexerCount + 1): # Prepare search results if newznabItems is not None: indexerNewznabItems = newznabItems[i - 1] else: indexerNewznabItems = [mockbuilder.buildNewznabItem(title % (i, j), "newznab%dresult%d.guid" % (i, j), "newznab%dresult%d.link" % (i, j), arrow.get(0).format("ddd, DD MMM YYYY HH:mm:ss Z"), "newznab%dresult%d.description" % (i, j), 1000, "newznab%d" % i, None) for j in range(1, resultsPerIndexers + 1)] xml = mockbuilder.buildNewznabResponse("newznab%dResponse" % i, indexerNewznabItems, 0, len(indexerNewznabItems)) self.response_callbacks.append(('newznab%d' % i, randint(0, sleep), xml)) # Prepare response mock url_re = re.compile(r'.*newznab%d.*' % i) rsps.add_callback(responses.GET, url_re, callback=self.rsps_callback, content_type='application/x-html') read_indexers_from_config() return testData
def testIgnoreByCategories(self, m): web.app.template_folder = "../templates" config.settings.categories.categories["moviessd"].ignoreResults = "always" config.settings.categories.categories["xxx"].ignoreResults = "always" config.settings.categories.categories["pc"].ignoreResults = "internal" movieSdResult = mockbuilder.buildNewznabItem(title="result1", indexer_name="newznab1", categories=[2030]) #MoviesSD: Is dismissed because its specific category is ignored xxxResult = mockbuilder.buildNewznabItem(title="result2", indexer_name="newznab1", categories=[6000]) #XXX: Is dismissed because its category is ignored (no more or less specific category exists) movieResult = mockbuilder.buildNewznabItem(title="result3", indexer_name="newznab1", categories=[2000]) #Is kept because more specific category Movies SD is ignored but not movies in general movieHdResult = mockbuilder.buildNewznabItem(title="result4", indexer_name="newznab1", categories=[2040]) #MoviesHD: Is kept because the other specific category is ignored but not this one tvResult = mockbuilder.buildNewznabItem(title="result5", indexer_name="newznab1", categories=[5000]) #TV: Is kept because its category (tv) is never ignored pcResult = mockbuilder.buildNewznabItem(title="result6", indexer_name="newznab1", categories=[4000]) # PC: Is kept because its category (tv) is only ignored for internal searches (but this is API) expectedItems = self.prepareSearchMocks(m, 1, newznabItems=[[movieSdResult, xxxResult, movieResult, movieHdResult, tvResult, pcResult]]) expectedItems = expectedItems[2:] #First two results will be dismissed with web.app.test_request_context('/api?t=search&q=query&apikey=%s' % config.settings.main.apikey): response = web.api() entries, _, _ = newznab.NewzNab(Bunch.fromDict({"name": "forTest", "score": 0, "host": "host"})).parseXml(response.data) self.assertSearchResults(entries, expectedItems) calledUrls = sorted([x.url for x in m.request_history]) self.assertTrue(compare('http://www.newznab1.com/api?apikey=apikeyindexer.com&t=search&extended=1&offset=0&limit=100&q=query', calledUrls[0]), calledUrls[0])
def testThatDatabaseValuesAreStored(self): with self.app.test_request_context('/'): with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: newznabItems = [ [mockbuilder.buildNewznabItem(title="title1", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab1")], [mockbuilder.buildNewznabItem(title="title2", pubdate=arrow.get(1000).format("ddd, DD MMM YYYY HH:mm:ss Z"), size=1000, indexer_name="newznab2")] ] self.prepareSearchMocks(rsps, indexerCount=len(newznabItems), newznabItems=newznabItems) # Make the second access unsuccessful rsps._urls.pop(1) rsps.add(responses.GET, r".*", body="an error message", status=500, content_type='application/x-html') searchRequest = SearchRequest(type="search", query="aquery", category="acategory", identifier_key="imdbid", identifier_value="animdbid", season=1, episode=2, indexers="newznab1|newznab2") result = search.search(searchRequest) results = result["results"] self.assertEqual(1, len(results)) dbSearch = Search().get() self.assertEqual(True, dbSearch.internal) self.assertEqual("aquery", dbSearch.query) self.assertEqual("All", dbSearch.category) self.assertEqual("imdbid", dbSearch.identifier_key) self.assertEqual("animdbid", dbSearch.identifier_value) self.assertEqual("1", dbSearch.season) self.assertEqual("2", dbSearch.episode) self.assertEqual("search", dbSearch.type) self.assertEqual(18, dbSearch.time.hour) indexerSearch1 = IndexerSearch.get(IndexerSearch.indexer == Indexer.get(Indexer.name == "newznab1")) self.assertEqual(indexerSearch1.search, dbSearch) self.assertEqual(18, indexerSearch1.time.hour) indexerSearch2 = IndexerSearch.get(IndexerSearch.indexer == Indexer.get(Indexer.name == "newznab2")) self.assertEqual(indexerSearch2.search, dbSearch) self.assertEqual(18, indexerSearch2.time.hour) calledUrls = sorted([x.request.url for x in rsps.calls]) indexerApiAccess1 = IndexerApiAccess.get(IndexerApiAccess.indexer == Indexer.get(Indexer.name == "newznab1")) self.assertEqual(indexerSearch1, indexerApiAccess1.indexer_search) self.assertEqual(18, indexerApiAccess1.time.hour) self.assertEqual("search", indexerApiAccess1.type) self.assertEqual(calledUrls[0], indexerApiAccess1.url) self.assertTrue(indexerApiAccess1.response_successful) self.assertEqual(0, indexerApiAccess1.response_time) self.assertIsNone(indexerApiAccess1.error) indexerApiAccess2 = IndexerApiAccess.get(IndexerApiAccess.indexer == Indexer.get(Indexer.name == "newznab2")) self.assertEqual(indexerSearch2, indexerApiAccess2.indexer_search) self.assertEqual(18, indexerApiAccess2.time.hour) self.assertEqual("search", indexerApiAccess2.type) self.assertEqual(calledUrls[1], indexerApiAccess2.url) self.assertFalse(indexerApiAccess2.response_successful) self.assertIsNone(indexerApiAccess2.response_time) self.assertTrue("Connection refused" in indexerApiAccess2.error) indexerStatus2 = IndexerStatus.get(IndexerStatus.indexer == Indexer.get(Indexer.name == "newznab2")) self.assertEqual(1, indexerStatus2.level) self.assertTrue("Connection refused" in indexerStatus2.reason)