def process_query_result(self, xml, searchRequest, maxResults=None): entries = [] countRejected = self.getRejectedCountDict() try: tree = ET.fromstring(xml) except Exception: self.exception("Error parsing XML: %s..." % xml[:500]) logger.debug(xml) raise IndexerResultParsingException("Error parsing XML", self) for elem in tree.iter('item'): title = elem.find("title") url = elem.find("enclosure") pubdate = elem.find("pubDate") if title is None or url is None or pubdate is None: continue entry = self.create_nzb_search_result() entry.title = title.text entry.link = url.attrib["url"] entry.has_nfo = NzbSearchResult.HAS_NFO_NO p = re.compile("(.*)\(Size:(\d*)") m = p.search(elem.find("description").text) if m: entry.description = m.group(1) entry.size = int(m.group(2)) * 1024 * 1024 #megabyte to byte if elem.find("category").text.lower() == "tv-dvdrip" or elem.find( "category").text.lower() == "tv-sd": entry.category = getCategoryByName("tvsd") elif elem.find("category").text.lower() == "tv-x264" or elem.find( "category").text.lower == "tv-hd": entry.category = getCategoryByName("tvhd") else: entry.category = getUnknownCategory() entry.indexerguid = elem.find("guid").text[ 30:] #39a/The.Almighty.Johnsons.S03E06.720p.BluRay.x264-YELLOWBiRD.nzb is the GUID, only the 39a doesn't work pubdate = arrow.get(pubdate.text, 'M/D/YYYY h:mm:ss A') entry.epoch = pubdate.timestamp entry.pubdate_utc = str(pubdate) entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z") entry.age_days = (arrow.utcnow() - pubdate).days entry.age = (arrow.utcnow() - pubdate).days accepted, reason, ri = self.accept_result(entry, searchRequest, self.supportedFilters) if accepted: entries.append(entry) else: countRejected[ri] += 1 self.debug("Rejected search result. Reason: %s" % reason) return IndexerProcessingResult(entries=entries, queries=[], total_known=True, has_more=False, total=len(entries), rejected=countRejected)
def process_query_result(self, xml, searchRequest, maxResults=None): entries = [] countRejected = self.getRejectedCountDict() try: tree = ET.fromstring(xml) except Exception: self.exception("Error parsing XML: %s..." % xml[:500]) logger.debug(xml) raise IndexerResultParsingException("Error parsing XML", self) for elem in tree.iter('item'): title = elem.find("title") url = elem.find("enclosure") pubdate = elem.find("pubDate") if title is None or url is None or pubdate is None: continue entry = self.create_nzb_search_result() entry.title = title.text entry.link = url.attrib["url"] entry.has_nfo = NzbSearchResult.HAS_NFO_NO p = re.compile("(.*)\(Size:(\d*)") m = p.search(elem.find("description").text) if m: entry.description = m.group(1) entry.size = int(m.group(2)) * 1024 * 1024 #megabyte to byte if elem.find("category").text.lower() == "tv-dvdrip" or elem.find("category").text.lower() == "tv-sd": entry.category = getCategoryByName("tvsd") elif elem.find("category").text.lower() == "tv-x264" or elem.find("category").text.lower == "tv-hd": entry.category = getCategoryByName("tvhd") else: entry.category = getUnknownCategory() entry.indexerguid = elem.find("guid").text[30:] #39a/The.Almighty.Johnsons.S03E06.720p.BluRay.x264-YELLOWBiRD.nzb is the GUID, only the 39a doesn't work pubdate = arrow.get(pubdate.text, 'M/D/YYYY h:mm:ss A') entry.epoch = pubdate.timestamp entry.pubdate_utc = str(pubdate) entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z") entry.age_days = (arrow.utcnow() - pubdate).days entry.age = (arrow.utcnow() - pubdate).days accepted, reason, ri = self.accept_result(entry, searchRequest, self.supportedFilters) if accepted: entries.append(entry) else: countRejected[ri] += 1 self.debug("Rejected search result. Reason: %s" % reason) return IndexerProcessingResult(entries=entries, queries=[], total_known=True, has_more=False, total=len(entries), rejected=countRejected)
def testIgnoreCategories(self): sm = SearchModule(None) cat = categories.getCategoryByName("movies") sr = SearchRequest(category=Bunch({"category": cat})) nsr = NzbSearchResult(pubdate_utc="", category=cat) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) cat.ignoreResults = "always" accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("always" in reason) cat.ignoreResults = "internal" sr = SearchRequest(internal=True, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("internal" in reason) cat.ignoreResults = "external" sr = SearchRequest(internal=True, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) cat.ignoreResults = "internal" sr = SearchRequest(internal=False, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) cat.ignoreResults = "external" sr = SearchRequest(internal=False, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("API" in reason)
def process_query_result(self, xml, searchRequest, maxResults=None): entries = [] countRejected = self.getRejectedCountDict() try: tree = ET.fromstring(xml) except Exception: self.exception("Error parsing XML: %s..." % xml[:500]) logger.debug(xml) raise IndexerResultParsingException("Error parsing XML", self) for elem in tree.iter('item'): title = elem.find("title") url = elem.find("enclosure") pubdate = elem.find("pubDate") if title is None or url is None or pubdate is None: continue entry = self.create_nzb_search_result() entry.title = title.text entry.link = url.attrib["url"] entry.size = int(url.attrib["length"]) entry.has_nfo = NzbSearchResult.HAS_NFO_NO entry.category = getCategoryByName("anime") entry.indexerguid = elem.find("guid").text entry.details_link = entry.link.replace("dl", "info") pubdate = arrow.get(pubdate.text, 'ddd, DD MMM YYYY HH:mm:ss Z') self.getDates(entry, pubdate) accepted, reason, ri = self.accept_result(entry, searchRequest, self.supportedFilters) if accepted: entries.append(entry) else: countRejected[ri] += 1 self.debug("Rejected search result. Reason: %s" % reason) return IndexerProcessingResult(entries=entries, queries=[], total_known=True, has_more=False, total=len(entries), rejected=countRejected)
def testIgnoreCategories(self): sm = SearchModule(None) cat = categories.getCategoryByName("movies") sr = SearchRequest(category=Bunch({"category": cat})) nsr = NzbSearchResult(pubdate_utc="", category=cat) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) cat.ignoreResults = "always" accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("always" in reason) cat.ignoreResults = "internal" sr = SearchRequest(internal=True, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("internal" in reason) cat.ignoreResults = "external" sr = SearchRequest(internal=True, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) cat.ignoreResults = "internal" sr = SearchRequest(internal=False, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) cat.ignoreResults = "external" sr = SearchRequest(internal=False, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("API" in reason)
def testLimits(self): sm = SearchModule(None) cat = categories.getCategoryByName("movies") nsr = NzbSearchResult(pubdate_utc="", size=90 * 1024 * 1024, category=cat) sr = SearchRequest(minsize=100, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("Smaller than" in reason) nsr = NzbSearchResult(pubdate_utc="", size=110 * 1024 * 1024, category=cat) sr = SearchRequest(minsize=100, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", size=110 * 1024 * 1024, category=cat) sr = SearchRequest(maxsize=100, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("Bigger than" in reason) nsr = NzbSearchResult(pubdate_utc="", size=90 * 1024 * 1024, category=cat) sr = SearchRequest(maxsize=100, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", age_days=90, category=cat) sr = SearchRequest(minage=100, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("Younger than" in reason) nsr = NzbSearchResult(pubdate_utc="", age_days=110, category=cat) sr = SearchRequest(minage=100, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", age_days=110, category=cat) sr = SearchRequest(maxage=100, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("Older than" in reason) nsr = NzbSearchResult(pubdate_utc="", age_days=90, category=cat) sr = SearchRequest(maxage=100, category=Bunch({"category": cat})) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted)
def testProcess_results(self): with open("mock/womble--sec-tv-dvd.xml") as f: entries = self.womble.process_query_result(f.read(), SearchRequest()).entries self.assertEqual("testtitle1", entries[0].title) self.assertEqual("http://www.newshost.co.za/nzb/79d/testtitle1.nzb", entries[0].link) self.assertEqual(336592896, entries[0].size) self.assertEqual(getCategoryByName("tvsd").name, entries[0].category.name) self.assertEqual("79d/testtitle1.nzb", entries[0].indexerguid) self.assertEqual(1442790103, entries[0].epoch) self.assertEqual("2015-09-20T23:01:43+00:00", entries[0].pubdate_utc) self.assertEqual(0, entries[0].age_days) self.assertEqual("79d/testtitle1.nzb", entries[0].indexerguid)
def testLimits(self): sm = SearchModule(None) cat = categories.getCategoryByName("movies") nsr = NzbSearchResult(pubdate_utc="", size=90 * 1024 * 1024, category=cat) sr = SearchRequest(minsize=100, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("Smaller than" in reason) nsr = NzbSearchResult(pubdate_utc="", size=110 * 1024 * 1024, category=cat) sr = SearchRequest(minsize=100, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", size=110 * 1024 * 1024, category=cat) sr = SearchRequest(maxsize=100, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("Bigger than" in reason) nsr = NzbSearchResult(pubdate_utc="", size=90 * 1024 * 1024, category=cat) sr = SearchRequest(maxsize=100, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", age_days=90, category=cat) sr = SearchRequest(minage=100, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("Younger than" in reason) nsr = NzbSearchResult(pubdate_utc="", age_days=110, category=cat) sr = SearchRequest(minage=100, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", age_days=110, category=cat) sr = SearchRequest(maxage=100, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("Older than" in reason) nsr = NzbSearchResult(pubdate_utc="", age_days=90, category=cat) sr = SearchRequest(maxage=100, category=Bunch({"category": cat})) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted)
def testProcess_results(self): with open("mock/womble--sec-tv-dvd.xml") as f: entries = self.womble.process_query_result(f.read(), SearchRequest()).entries self.assertEqual("testtitle1", entries[0].title) self.assertEqual( "http://www.newshost.co.za/nzb/79d/testtitle1.nzb", entries[0].link) self.assertEqual(336592896, entries[0].size) self.assertEqual( getCategoryByName("tvsd").name, entries[0].category.name) self.assertEqual("79d/testtitle1.nzb", entries[0].indexerguid) self.assertEqual(1442790103, entries[0].epoch) self.assertEqual("2015-09-20T23:01:43+00:00", entries[0].pubdate_utc) self.assertEqual(0, entries[0].age_days) self.assertEqual("79d/testtitle1.nzb", entries[0].indexerguid)
def process_query_result(self, xml, searchRequest, maxResults=None): entries = [] countRejected = 0 try: tree = ET.fromstring(xml) except Exception: self.exception("Error parsing XML: %s..." % xml[:500]) logger.debug(xml) raise IndexerResultParsingException("Error parsing XML", self) for elem in tree.iter('item'): title = elem.find("title") url = elem.find("enclosure") pubdate = elem.find("pubDate") if title is None or url is None or pubdate is None: continue entry = self.create_nzb_search_result() entry.title = title.text entry.link = url.attrib["url"] entry.size = int(url.attrib["length"]) entry.has_nfo = NzbSearchResult.HAS_NFO_NO entry.category = getCategoryByName("anime") entry.indexerguid = elem.find("guid").text entry.details_link = entry.link.replace("dl", "info") pubdate = arrow.get(pubdate.text, 'ddd, DD MMM YYYY HH:mm:ss Z') entry.epoch = pubdate.timestamp entry.pubdate_utc = str(pubdate) entry.pubDate = pubdate entry.age_days = (arrow.utcnow() - pubdate).days accepted, reason = self.accept_result(entry, searchRequest, self.supportedFilters) if accepted: entries.append(entry) else: countRejected += 1 self.debug("Rejected search result. Reason: %s" % reason) return IndexerProcessingResult(entries=entries, queries=[], total_known=True, has_more=False, total=len(entries), rejected=countRejected)
def parseItem(self, item): entry = self.create_nzb_search_result() entry.indexerguid = item.find("nzbid").text entry.title = item.find("release").text entry.group = item.find("group").text entry.link = item.find("getnzb").text entry.size = long(item.find("sizebytes").text) entry.epoch = long(item.find("usenetage").text) pubdate = arrow.get(entry.epoch) entry.pubdate_utc = str(pubdate) entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z") entry.age_days = (arrow.utcnow() - pubdate).days entry.age_precise = True entry.details_link = item.find("details").text entry.has_nfo = NzbSearchResult.HAS_NFO_YES if item.find("getnfo") is not None else NzbSearchResult.HAS_NFO_NO categoryid = item.find("categoryid").text if categoryid in omgwtf_to_categories.keys(): entry.category = getCategoryByName(omgwtf_to_categories[categoryid]) else: entry.category = getUnknownCategory() return entry
def testWords(self): sm = SearchModule(None) #Required words sr = SearchRequest(forbiddenWords=[], requiredWords=["rqa", "rqb", "rq-c", "rq.d"], category=Bunch({"category": categories.getCategoryByName("all")})) nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="rqa", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="a.title.rqa.xyz", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="a title rqa xyz", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="a-title-rq-c", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="a title rq.d", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="rqatsch", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) nsr = NzbSearchResult(pubdate_utc="", title="xyz.rqa", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa rqb", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="xyz", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("None of the required" in reason) #Forbidden words sr = SearchRequest(forbiddenWords=["fba", "fbb", "fb-c", "fb.d"], requiredWords=[], category=Bunch({"category": categories.getCategoryByName("all")})) nsr = NzbSearchResult(pubdate_utc="", title="xyz fba") accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fba\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyzfba") accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="xyzfb-ca") accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fb-c\" is in the list" in reason) #Both sr = SearchRequest(forbiddenWords=["fba", "fbb", "fb-c", "fb.d"], requiredWords=["rqa", "rqb", "rq-c", "rq.d"], category=Bunch({"category": categories.getCategoryByName("all")})) nsr = NzbSearchResult(pubdate_utc="", title="xyz fba rqa") accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fba\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyz FBA rqb") accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fba\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa.rqb.fbb") accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fbb\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa.rqb.fba.fbc") accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fba\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyz acd") sr = SearchRequest(forbiddenWords=["ACD"]) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"ACD\" is in the list" in reason) sr = SearchRequest(forbiddenWords=[], requiredWords=[], category=Bunch({"category": categories.getCategoryByName("all")})) config.settings.searching.applyRestrictions = "both" config.settings.searching.requiredWords = "" config.settings.searching.forbiddenWords = "" config.settings.searching.requiredRegex = "" config.settings.searching.forbiddenRegex = "" sr.category.category.applyRestrictions = "both" sr.category.category.forbiddenWords = "" sr.category.category.requiredWords = "" sr.category.category.forbiddenRegex = "" sr.category.category.requiredRegex = "" sr.internal = True config.settings.searching.applyRestrictions = "both" config.settings.searching.forbiddenRegex = "abc" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) config.settings.searching.applyRestrictions = "internal" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) config.settings.searching.applyRestrictions = "external" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) sr.internal = False config.settings.searching.applyRestrictions = "both" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) config.settings.searching.applyRestrictions = "internal" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) config.settings.searching.applyRestrictions = "external" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) sr.internal = True config.settings.searching.forbiddenRegex = "" sr.category.category.forbiddenRegex = "abc" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) sr.category.category.applyRestrictions = "internal" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) sr.category.category.applyRestrictions = "external" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) sr.internal = False sr.category.category.applyRestrictions = "both" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) sr.category.category.applyRestrictions = "internal" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) sr.category.category.applyRestrictions = "external" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason, ri = sm.accept_result(nsr, sr, None) self.assertFalse(accepted)
def testWords(self): sm = SearchModule(None) #Required words sr = SearchRequest( forbiddenWords=[], requiredWords=["rqa", "rqb", "rq-c", "rq.d"], category=Bunch({"category": categories.getCategoryByName("all")})) nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="rqa", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="a.title.rqa.xyz", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="a title rqa xyz", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="a-title-rq-c", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="a title rq.d", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="rqatsch", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) nsr = NzbSearchResult(pubdate_utc="", title="xyz.rqa", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa rqb", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="xyz", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("None of the required" in reason) #Forbidden words sr = SearchRequest( forbiddenWords=["fba", "fbb", "fb-c", "fb.d"], requiredWords=[], category=Bunch({"category": categories.getCategoryByName("all")})) nsr = NzbSearchResult(pubdate_utc="", title="xyz fba") accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fba\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyzfba") accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) nsr = NzbSearchResult(pubdate_utc="", title="xyzfb-ca") accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fb-c\" is in the list" in reason) #Both sr = SearchRequest( forbiddenWords=["fba", "fbb", "fb-c", "fb.d"], requiredWords=["rqa", "rqb", "rq-c", "rq.d"], category=Bunch({"category": categories.getCategoryByName("all")})) nsr = NzbSearchResult(pubdate_utc="", title="xyz fba rqa") accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fba\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyz FBA rqb") accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fba\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa.rqb.fbb") accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fbb\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa.rqb.fba.fbc") accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"fba\" is in the list" in reason) nsr = NzbSearchResult(pubdate_utc="", title="xyz acd") sr = SearchRequest(forbiddenWords=["ACD"]) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) self.assertTrue("\"ACD\" is in the list" in reason) sr = SearchRequest( forbiddenWords=[], requiredWords=[], category=Bunch({"category": categories.getCategoryByName("all")})) config.settings.searching.applyRestrictions = "both" config.settings.searching.requiredWords = "" config.settings.searching.forbiddenWords = "" config.settings.searching.requiredRegex = "" config.settings.searching.forbiddenRegex = "" sr.category.category.applyRestrictions = "both" sr.category.category.forbiddenWords = "" sr.category.category.requiredWords = "" sr.category.category.forbiddenRegex = "" sr.category.category.requiredRegex = "" sr.internal = True config.settings.searching.applyRestrictions = "both" config.settings.searching.forbiddenRegex = "abc" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) config.settings.searching.applyRestrictions = "internal" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) config.settings.searching.applyRestrictions = "external" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) sr.internal = False config.settings.searching.applyRestrictions = "both" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) config.settings.searching.applyRestrictions = "internal" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) config.settings.searching.applyRestrictions = "external" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) sr.internal = True config.settings.searching.forbiddenRegex = "" sr.category.category.forbiddenRegex = "abc" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) sr.category.category.applyRestrictions = "internal" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) sr.category.category.applyRestrictions = "external" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) sr.internal = False sr.category.category.applyRestrictions = "both" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted) sr.category.category.applyRestrictions = "internal" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertTrue(accepted) sr.category.category.applyRestrictions = "external" nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all")) accepted, reason = sm.accept_result(nsr, sr, None) self.assertFalse(accepted)
def process_query_result(self, xml_response, searchRequest, maxResults=None): self.debug("Started processing results") if "0 results found" in xml_response: return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=0) if "search to short" in xml_response: self.info("omgwtf says the query was too short") return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=0) entries = [] countRejected = 0 try: tree = ET.fromstring(xml_response) except Exception: self.exception("Error parsing XML: %s..." % xml_response[:500]) raise IndexerResultParsingException("Error parsing XML", self) if tree.tag == "xml": total = int(tree.find("info").find("results").text) current_page = int(tree.find("info").find("current_page").text) total_pages = int(tree.find("info").find("pages").text) has_more = current_page < total_pages for item in tree.find("search_req").findall("post"): entry = self.parseItem(item) accepted, reason = self.accept_result(entry, searchRequest, self.supportedFilters) if accepted: entries.append(entry) else: countRejected += 1 self.debug("Rejected search result. Reason: %s" % reason) return IndexerProcessingResult(entries=entries, queries=[], total=total, total_known=True, has_more=has_more, rejected=countRejected) elif tree.tag == "rss": for item in tree.find("channel").findall("item"): entry = self.create_nzb_search_result() indexerguid = item.find("guid").text m = self.regexGuid.match(indexerguid) if m: entry.indexerguid = m.group(1) else: self.warn("Unable to find GUID in " + indexerguid) raise IndexerResultParsingRowException("Unable to find GUID") entry.title = item.find("title").text description = item.find("description").text m = self.regexGroup.match(description) if m: entry.group = m.group(1) else: self.warn("Unable to find group in " + description) raise IndexerResultParsingRowException("Unable to find usenet group") entry.size = long(item.find("enclosure").attrib["length"]) entry.pubDate = item.find("pubDate").text pubdate = arrow.get(entry.pubDate, 'ddd, DD MMM YYYY HH:mm:ss Z') entry.epoch = pubdate.timestamp entry.pubdate_utc = str(pubdate) entry.age_days = (arrow.utcnow() - pubdate).days entry.precise_date = True entry.link = item.find("link").text entry.has_nfo = NzbSearchResult.HAS_NFO_MAYBE categoryid = item.find("categoryid").text entry.details_link = self.get_details_link(entry.indexerguid) if categoryid in omgwtf_to_categories.keys(): entry.category = getCategoryByName(omgwtf_to_categories[categoryid]) else: entry.category = getUnknownCategory() accepted, reason = self.accept_result(entry, searchRequest, self.supportedFilters) if accepted: entries.append(entry) else: countRejected += 1 self.debug("Rejected search result. Reason: %s" % reason) return IndexerProcessingResult(entries=entries, queries=[], total=len(entries), total_known=True, has_more=False, rejected=countRejected) else: self.warn("Unknown response type: %s" % xml_response[:100]) return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=countRejected)
def test_pick_indexers(self): config.settings.searching.generate_queries = [] config.settings.indexers.extend([self.newznab1, self.newznab2]) getIndexerSettingByName("womble").enabled = True getIndexerSettingByName("womble").accessType = "both" getIndexerSettingByName("nzbclub").enabled = True getIndexerSettingByName("nzbclub").accessType = "both" read_indexers_from_config() search_request = SearchRequest() indexers = search.pick_indexers(search_request) self.assertEqual(3, len(indexers)) # Indexers with tv search and which support queries (actually searching for particular releases) search_request.query = "bla" indexers = search.pick_indexers(search_request) self.assertEqual(3, len(indexers)) # Indexers with tv search, including those that only provide a list of latest releases (womble) but excluding the one that needs a query (nzbclub) search_request.query = None indexers = search.pick_indexers(search_request) self.assertEqual(3, len(indexers)) search_request.identifier_key = "tvdbid" indexers = search.pick_indexers(search_request) self.assertEqual(2, len(indexers)) self.assertEqual("newznab1", indexers[0].name) self.assertEqual("newznab2", indexers[1].name) search_request.identifier_key = "imdbid" search_request.category = getCategoryByName("movies") indexers = search.pick_indexers(search_request) self.assertEqual(1, len(indexers)) self.assertEqual("newznab1", indexers[0].name) # WIth query generation NZBClub should also be returned infos.title_from_id = mock config.settings.searching.generate_queries = ["internal"] search_request.identifier_key = "tvdbid" search_request.query = None search_request.category = None indexers = search.pick_indexers(search_request) self.assertEqual(3, len(indexers)) self.assertEqual("nzbclub", indexers[0].name) self.assertEqual("newznab1", indexers[1].name) self.assertEqual("newznab2", indexers[2].name) # Test picking depending on internal, external, both getIndexerSettingByName("womble").enabled = False getIndexerSettingByName("nzbclub").enabled = False getIndexerSettingByName("newznab1").accessType = "both" search_request.internal = True indexers = search.pick_indexers(search_request) self.assertEqual(2, len(indexers)) search_request.internal = False indexers = search.pick_indexers(search_request) self.assertEqual(2, len(indexers)) config.settings.indexers = [self.newznab1, self.newznab2] getIndexerSettingByName("newznab1").accessType = "external" read_indexers_from_config() search_request.internal = True indexers = search.pick_indexers(search_request) self.assertEqual(1, len(indexers)) search_request.internal = False indexers = search.pick_indexers(search_request) self.assertEqual(2, len(indexers)) getIndexerSettingByName("newznab1").accessType = "internal" read_indexers_from_config() search_request.internal = True indexers = search.pick_indexers(search_request) self.assertEqual(2, len(indexers)) search_request.internal = False indexers = search.pick_indexers(search_request) self.assertEqual(1, len(indexers))