def process_query_result(self, xml, searchRequest, maxResults=None):
entries = []
countRejected = self.getRejectedCountDict()
try:
tree = ET.fromstring(xml)
except Exception:
self.exception("Error parsing XML: %s..." % xml[:500])
logger.debug(xml)
raise IndexerResultParsingException("Error parsing XML", self)
for elem in tree.iter('item'):
title = elem.find("title")
url = elem.find("enclosure")
pubdate = elem.find("pubDate")
if title is None or url is None or pubdate is None:
continue
entry = self.create_nzb_search_result()
entry.title = title.text
entry.link = url.attrib["url"]
entry.has_nfo = NzbSearchResult.HAS_NFO_NO
p = re.compile("(.*)\(Size:(\d*)")
m = p.search(elem.find("description").text)
if m:
entry.description = m.group(1)
entry.size = int(m.group(2)) * 1024 * 1024 #megabyte to byte
if elem.find("category").text.lower() == "tv-dvdrip" or elem.find(
"category").text.lower() == "tv-sd":
entry.category = getCategoryByName("tvsd")
elif elem.find("category").text.lower() == "tv-x264" or elem.find(
"category").text.lower == "tv-hd":
entry.category = getCategoryByName("tvhd")
else:
entry.category = getUnknownCategory()
entry.indexerguid = elem.find("guid").text[
30:] #39a/The.Almighty.Johnsons.S03E06.720p.BluRay.x264-YELLOWBiRD.nzb is the GUID, only the 39a doesn't work
pubdate = arrow.get(pubdate.text, 'M/D/YYYY h:mm:ss A')
entry.epoch = pubdate.timestamp
entry.pubdate_utc = str(pubdate)
entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z")
entry.age_days = (arrow.utcnow() - pubdate).days
entry.age = (arrow.utcnow() - pubdate).days
accepted, reason, ri = self.accept_result(entry, searchRequest,
self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected[ri] += 1
self.debug("Rejected search result. Reason: %s" % reason)
return IndexerProcessingResult(entries=entries,
queries=[],
total_known=True,
has_more=False,
total=len(entries),
rejected=countRejected)
def process_query_result(self, xml, searchRequest, maxResults=None):
entries = []
countRejected = self.getRejectedCountDict()
try:
tree = ET.fromstring(xml)
except Exception:
self.exception("Error parsing XML: %s..." % xml[:500])
logger.debug(xml)
raise IndexerResultParsingException("Error parsing XML", self)
for elem in tree.iter('item'):
title = elem.find("title")
url = elem.find("enclosure")
pubdate = elem.find("pubDate")
if title is None or url is None or pubdate is None:
continue
entry = self.create_nzb_search_result()
entry.title = title.text
entry.link = url.attrib["url"]
entry.has_nfo = NzbSearchResult.HAS_NFO_NO
p = re.compile("(.*)\(Size:(\d*)")
m = p.search(elem.find("description").text)
if m:
entry.description = m.group(1)
entry.size = int(m.group(2)) * 1024 * 1024 #megabyte to byte
if elem.find("category").text.lower() == "tv-dvdrip" or elem.find("category").text.lower() == "tv-sd":
entry.category = getCategoryByName("tvsd")
elif elem.find("category").text.lower() == "tv-x264" or elem.find("category").text.lower == "tv-hd":
entry.category = getCategoryByName("tvhd")
else:
entry.category = getUnknownCategory()
entry.indexerguid = elem.find("guid").text[30:] #39a/The.Almighty.Johnsons.S03E06.720p.BluRay.x264-YELLOWBiRD.nzb is the GUID, only the 39a doesn't work
pubdate = arrow.get(pubdate.text, 'M/D/YYYY h:mm:ss A')
entry.epoch = pubdate.timestamp
entry.pubdate_utc = str(pubdate)
entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z")
entry.age_days = (arrow.utcnow() - pubdate).days
entry.age = (arrow.utcnow() - pubdate).days
accepted, reason, ri = self.accept_result(entry, searchRequest, self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected[ri] += 1
self.debug("Rejected search result. Reason: %s" % reason)
return IndexerProcessingResult(entries=entries, queries=[], total_known=True, has_more=False, total=len(entries), rejected=countRejected)
def testIgnoreCategories(self):
sm = SearchModule(None)
cat = categories.getCategoryByName("movies")
sr = SearchRequest(category=Bunch({"category": cat}))
nsr = NzbSearchResult(pubdate_utc="", category=cat)
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
cat.ignoreResults = "always"
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("always" in reason)
cat.ignoreResults = "internal"
sr = SearchRequest(internal=True, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("internal" in reason)
cat.ignoreResults = "external"
sr = SearchRequest(internal=True, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
cat.ignoreResults = "internal"
sr = SearchRequest(internal=False, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
cat.ignoreResults = "external"
sr = SearchRequest(internal=False, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("API" in reason)
def process_query_result(self, xml, searchRequest, maxResults=None):
entries = []
countRejected = self.getRejectedCountDict()
try:
tree = ET.fromstring(xml)
except Exception:
self.exception("Error parsing XML: %s..." % xml[:500])
logger.debug(xml)
raise IndexerResultParsingException("Error parsing XML", self)
for elem in tree.iter('item'):
title = elem.find("title")
url = elem.find("enclosure")
pubdate = elem.find("pubDate")
if title is None or url is None or pubdate is None:
continue
entry = self.create_nzb_search_result()
entry.title = title.text
entry.link = url.attrib["url"]
entry.size = int(url.attrib["length"])
entry.has_nfo = NzbSearchResult.HAS_NFO_NO
entry.category = getCategoryByName("anime")
entry.indexerguid = elem.find("guid").text
entry.details_link = entry.link.replace("dl", "info")
pubdate = arrow.get(pubdate.text, 'ddd, DD MMM YYYY HH:mm:ss Z')
self.getDates(entry, pubdate)
accepted, reason, ri = self.accept_result(entry, searchRequest, self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected[ri] += 1
self.debug("Rejected search result. Reason: %s" % reason)
return IndexerProcessingResult(entries=entries, queries=[], total_known=True, has_more=False, total=len(entries), rejected=countRejected)
def testIgnoreCategories(self):
sm = SearchModule(None)
cat = categories.getCategoryByName("movies")
sr = SearchRequest(category=Bunch({"category": cat}))
nsr = NzbSearchResult(pubdate_utc="", category=cat)
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
cat.ignoreResults = "always"
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("always" in reason)
cat.ignoreResults = "internal"
sr = SearchRequest(internal=True, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("internal" in reason)
cat.ignoreResults = "external"
sr = SearchRequest(internal=True, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
cat.ignoreResults = "internal"
sr = SearchRequest(internal=False, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
cat.ignoreResults = "external"
sr = SearchRequest(internal=False, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("API" in reason)
def testLimits(self):
sm = SearchModule(None)
cat = categories.getCategoryByName("movies")
nsr = NzbSearchResult(pubdate_utc="",
size=90 * 1024 * 1024,
category=cat)
sr = SearchRequest(minsize=100, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("Smaller than" in reason)
nsr = NzbSearchResult(pubdate_utc="",
size=110 * 1024 * 1024,
category=cat)
sr = SearchRequest(minsize=100, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
size=110 * 1024 * 1024,
category=cat)
sr = SearchRequest(maxsize=100, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("Bigger than" in reason)
nsr = NzbSearchResult(pubdate_utc="",
size=90 * 1024 * 1024,
category=cat)
sr = SearchRequest(maxsize=100, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", age_days=90, category=cat)
sr = SearchRequest(minage=100, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("Younger than" in reason)
nsr = NzbSearchResult(pubdate_utc="", age_days=110, category=cat)
sr = SearchRequest(minage=100, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", age_days=110, category=cat)
sr = SearchRequest(maxage=100, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("Older than" in reason)
nsr = NzbSearchResult(pubdate_utc="", age_days=90, category=cat)
sr = SearchRequest(maxage=100, category=Bunch({"category": cat}))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
def testProcess_results(self):
with open("mock/womble--sec-tv-dvd.xml") as f:
entries = self.womble.process_query_result(f.read(), SearchRequest()).entries
self.assertEqual("testtitle1", entries[0].title)
self.assertEqual("http://www.newshost.co.za/nzb/79d/testtitle1.nzb", entries[0].link)
self.assertEqual(336592896, entries[0].size)
self.assertEqual(getCategoryByName("tvsd").name, entries[0].category.name)
self.assertEqual("79d/testtitle1.nzb", entries[0].indexerguid)
self.assertEqual(1442790103, entries[0].epoch)
self.assertEqual("2015-09-20T23:01:43+00:00", entries[0].pubdate_utc)
self.assertEqual(0, entries[0].age_days)
self.assertEqual("79d/testtitle1.nzb", entries[0].indexerguid)
def testLimits(self):
sm = SearchModule(None)
cat = categories.getCategoryByName("movies")
nsr = NzbSearchResult(pubdate_utc="", size=90 * 1024 * 1024, category=cat)
sr = SearchRequest(minsize=100, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("Smaller than" in reason)
nsr = NzbSearchResult(pubdate_utc="", size=110 * 1024 * 1024, category=cat)
sr = SearchRequest(minsize=100, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", size=110 * 1024 * 1024, category=cat)
sr = SearchRequest(maxsize=100, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("Bigger than" in reason)
nsr = NzbSearchResult(pubdate_utc="", size=90 * 1024 * 1024, category=cat)
sr = SearchRequest(maxsize=100, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", age_days=90, category=cat)
sr = SearchRequest(minage=100, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("Younger than" in reason)
nsr = NzbSearchResult(pubdate_utc="", age_days=110, category=cat)
sr = SearchRequest(minage=100, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", age_days=110, category=cat)
sr = SearchRequest(maxage=100, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("Older than" in reason)
nsr = NzbSearchResult(pubdate_utc="", age_days=90, category=cat)
sr = SearchRequest(maxage=100, category=Bunch({"category": cat}))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
def testProcess_results(self):
with open("mock/womble--sec-tv-dvd.xml") as f:
entries = self.womble.process_query_result(f.read(),
SearchRequest()).entries
self.assertEqual("testtitle1", entries[0].title)
self.assertEqual(
"http://www.newshost.co.za/nzb/79d/testtitle1.nzb",
entries[0].link)
self.assertEqual(336592896, entries[0].size)
self.assertEqual(
getCategoryByName("tvsd").name, entries[0].category.name)
self.assertEqual("79d/testtitle1.nzb", entries[0].indexerguid)
self.assertEqual(1442790103, entries[0].epoch)
self.assertEqual("2015-09-20T23:01:43+00:00",
entries[0].pubdate_utc)
self.assertEqual(0, entries[0].age_days)
self.assertEqual("79d/testtitle1.nzb", entries[0].indexerguid)
def process_query_result(self, xml, searchRequest, maxResults=None):
entries = []
countRejected = 0
try:
tree = ET.fromstring(xml)
except Exception:
self.exception("Error parsing XML: %s..." % xml[:500])
logger.debug(xml)
raise IndexerResultParsingException("Error parsing XML", self)
for elem in tree.iter('item'):
title = elem.find("title")
url = elem.find("enclosure")
pubdate = elem.find("pubDate")
if title is None or url is None or pubdate is None:
continue
entry = self.create_nzb_search_result()
entry.title = title.text
entry.link = url.attrib["url"]
entry.size = int(url.attrib["length"])
entry.has_nfo = NzbSearchResult.HAS_NFO_NO
entry.category = getCategoryByName("anime")
entry.indexerguid = elem.find("guid").text
entry.details_link = entry.link.replace("dl", "info")
pubdate = arrow.get(pubdate.text, 'ddd, DD MMM YYYY HH:mm:ss Z')
entry.epoch = pubdate.timestamp
entry.pubdate_utc = str(pubdate)
entry.pubDate = pubdate
entry.age_days = (arrow.utcnow() - pubdate).days
accepted, reason = self.accept_result(entry, searchRequest,
self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected += 1
self.debug("Rejected search result. Reason: %s" % reason)
return IndexerProcessingResult(entries=entries,
queries=[],
total_known=True,
has_more=False,
total=len(entries),
rejected=countRejected)
def parseItem(self, item):
entry = self.create_nzb_search_result()
entry.indexerguid = item.find("nzbid").text
entry.title = item.find("release").text
entry.group = item.find("group").text
entry.link = item.find("getnzb").text
entry.size = long(item.find("sizebytes").text)
entry.epoch = long(item.find("usenetage").text)
pubdate = arrow.get(entry.epoch)
entry.pubdate_utc = str(pubdate)
entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z")
entry.age_days = (arrow.utcnow() - pubdate).days
entry.age_precise = True
entry.details_link = item.find("details").text
entry.has_nfo = NzbSearchResult.HAS_NFO_YES if item.find("getnfo") is not None else NzbSearchResult.HAS_NFO_NO
categoryid = item.find("categoryid").text
if categoryid in omgwtf_to_categories.keys():
entry.category = getCategoryByName(omgwtf_to_categories[categoryid])
else:
entry.category = getUnknownCategory()
return entry
def parseItem(self, item):
entry = self.create_nzb_search_result()
entry.indexerguid = item.find("nzbid").text
entry.title = item.find("release").text
entry.group = item.find("group").text
entry.link = item.find("getnzb").text
entry.size = long(item.find("sizebytes").text)
entry.epoch = long(item.find("usenetage").text)
pubdate = arrow.get(entry.epoch)
entry.pubdate_utc = str(pubdate)
entry.pubDate = pubdate.format("ddd, DD MMM YYYY HH:mm:ss Z")
entry.age_days = (arrow.utcnow() - pubdate).days
entry.age_precise = True
entry.details_link = item.find("details").text
entry.has_nfo = NzbSearchResult.HAS_NFO_YES if item.find("getnfo") is not None else NzbSearchResult.HAS_NFO_NO
categoryid = item.find("categoryid").text
if categoryid in omgwtf_to_categories.keys():
entry.category = getCategoryByName(omgwtf_to_categories[categoryid])
else:
entry.category = getUnknownCategory()
return entry
def testWords(self):
sm = SearchModule(None)
#Required words
sr = SearchRequest(forbiddenWords=[], requiredWords=["rqa", "rqb", "rq-c", "rq.d"], category=Bunch({"category": categories.getCategoryByName("all")}))
nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="rqa", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="a.title.rqa.xyz", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="a title rqa xyz", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="a-title-rq-c", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="a title rq.d", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="rqatsch", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="xyz.rqa", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa rqb", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="xyz", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("None of the required" in reason)
#Forbidden words
sr = SearchRequest(forbiddenWords=["fba", "fbb", "fb-c", "fb.d"], requiredWords=[], category=Bunch({"category": categories.getCategoryByName("all")}))
nsr = NzbSearchResult(pubdate_utc="", title="xyz fba")
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fba\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyzfba")
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="xyzfb-ca")
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fb-c\" is in the list" in reason)
#Both
sr = SearchRequest(forbiddenWords=["fba", "fbb", "fb-c", "fb.d"], requiredWords=["rqa", "rqb", "rq-c", "rq.d"], category=Bunch({"category": categories.getCategoryByName("all")}))
nsr = NzbSearchResult(pubdate_utc="", title="xyz fba rqa")
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fba\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyz FBA rqb")
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fba\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa.rqb.fbb")
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fbb\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa.rqb.fba.fbc")
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fba\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyz acd")
sr = SearchRequest(forbiddenWords=["ACD"])
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"ACD\" is in the list" in reason)
sr = SearchRequest(forbiddenWords=[], requiredWords=[], category=Bunch({"category": categories.getCategoryByName("all")}))
config.settings.searching.applyRestrictions = "both"
config.settings.searching.requiredWords = ""
config.settings.searching.forbiddenWords = ""
config.settings.searching.requiredRegex = ""
config.settings.searching.forbiddenRegex = ""
sr.category.category.applyRestrictions = "both"
sr.category.category.forbiddenWords = ""
sr.category.category.requiredWords = ""
sr.category.category.forbiddenRegex = ""
sr.category.category.requiredRegex = ""
sr.internal = True
config.settings.searching.applyRestrictions = "both"
config.settings.searching.forbiddenRegex = "abc"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
config.settings.searching.applyRestrictions = "internal"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
config.settings.searching.applyRestrictions = "external"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
sr.internal = False
config.settings.searching.applyRestrictions = "both"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
config.settings.searching.applyRestrictions = "internal"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
config.settings.searching.applyRestrictions = "external"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
sr.internal = True
config.settings.searching.forbiddenRegex = ""
sr.category.category.forbiddenRegex = "abc"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
sr.category.category.applyRestrictions = "internal"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
sr.category.category.applyRestrictions = "external"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
sr.internal = False
sr.category.category.applyRestrictions = "both"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
sr.category.category.applyRestrictions = "internal"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
sr.category.category.applyRestrictions = "external"
nsr = NzbSearchResult(pubdate_utc="", title="abc", category=categories.getCategoryByName("all"))
accepted, reason, ri = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
def testWords(self):
sm = SearchModule(None)
#Required words
sr = SearchRequest(
forbiddenWords=[],
requiredWords=["rqa", "rqb", "rq-c", "rq.d"],
category=Bunch({"category": categories.getCategoryByName("all")}))
nsr = NzbSearchResult(pubdate_utc="",
title="xyz rqa",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="rqa",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="a.title.rqa.xyz",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="a title rqa xyz",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="a-title-rq-c",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="a title rq.d",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="rqatsch",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="xyz.rqa",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="xyz rqa rqb",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="",
title="xyz",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("None of the required" in reason)
#Forbidden words
sr = SearchRequest(
forbiddenWords=["fba", "fbb", "fb-c", "fb.d"],
requiredWords=[],
category=Bunch({"category": categories.getCategoryByName("all")}))
nsr = NzbSearchResult(pubdate_utc="", title="xyz fba")
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fba\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyzfba")
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
nsr = NzbSearchResult(pubdate_utc="", title="xyzfb-ca")
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fb-c\" is in the list" in reason)
#Both
sr = SearchRequest(
forbiddenWords=["fba", "fbb", "fb-c", "fb.d"],
requiredWords=["rqa", "rqb", "rq-c", "rq.d"],
category=Bunch({"category": categories.getCategoryByName("all")}))
nsr = NzbSearchResult(pubdate_utc="", title="xyz fba rqa")
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fba\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyz FBA rqb")
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fba\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa.rqb.fbb")
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fbb\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyz rqa.rqb.fba.fbc")
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"fba\" is in the list" in reason)
nsr = NzbSearchResult(pubdate_utc="", title="xyz acd")
sr = SearchRequest(forbiddenWords=["ACD"])
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
self.assertTrue("\"ACD\" is in the list" in reason)
sr = SearchRequest(
forbiddenWords=[],
requiredWords=[],
category=Bunch({"category": categories.getCategoryByName("all")}))
config.settings.searching.applyRestrictions = "both"
config.settings.searching.requiredWords = ""
config.settings.searching.forbiddenWords = ""
config.settings.searching.requiredRegex = ""
config.settings.searching.forbiddenRegex = ""
sr.category.category.applyRestrictions = "both"
sr.category.category.forbiddenWords = ""
sr.category.category.requiredWords = ""
sr.category.category.forbiddenRegex = ""
sr.category.category.requiredRegex = ""
sr.internal = True
config.settings.searching.applyRestrictions = "both"
config.settings.searching.forbiddenRegex = "abc"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
config.settings.searching.applyRestrictions = "internal"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
config.settings.searching.applyRestrictions = "external"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
sr.internal = False
config.settings.searching.applyRestrictions = "both"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
config.settings.searching.applyRestrictions = "internal"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
config.settings.searching.applyRestrictions = "external"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
sr.internal = True
config.settings.searching.forbiddenRegex = ""
sr.category.category.forbiddenRegex = "abc"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
sr.category.category.applyRestrictions = "internal"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
sr.category.category.applyRestrictions = "external"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
sr.internal = False
sr.category.category.applyRestrictions = "both"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
sr.category.category.applyRestrictions = "internal"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertTrue(accepted)
sr.category.category.applyRestrictions = "external"
nsr = NzbSearchResult(pubdate_utc="",
title="abc",
category=categories.getCategoryByName("all"))
accepted, reason = sm.accept_result(nsr, sr, None)
self.assertFalse(accepted)
def process_query_result(self, xml_response, searchRequest, maxResults=None):
self.debug("Started processing results")
if "0 results found" in xml_response:
return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=0)
if "search to short" in xml_response:
self.info("omgwtf says the query was too short")
return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=0)
entries = []
countRejected = 0
try:
tree = ET.fromstring(xml_response)
except Exception:
self.exception("Error parsing XML: %s..." % xml_response[:500])
raise IndexerResultParsingException("Error parsing XML", self)
if tree.tag == "xml":
total = int(tree.find("info").find("results").text)
current_page = int(tree.find("info").find("current_page").text)
total_pages = int(tree.find("info").find("pages").text)
has_more = current_page < total_pages
for item in tree.find("search_req").findall("post"):
entry = self.parseItem(item)
accepted, reason = self.accept_result(entry, searchRequest, self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected += 1
self.debug("Rejected search result. Reason: %s" % reason)
return IndexerProcessingResult(entries=entries, queries=[], total=total, total_known=True, has_more=has_more, rejected=countRejected)
elif tree.tag == "rss":
for item in tree.find("channel").findall("item"):
entry = self.create_nzb_search_result()
indexerguid = item.find("guid").text
m = self.regexGuid.match(indexerguid)
if m:
entry.indexerguid = m.group(1)
else:
self.warn("Unable to find GUID in " + indexerguid)
raise IndexerResultParsingRowException("Unable to find GUID")
entry.title = item.find("title").text
description = item.find("description").text
m = self.regexGroup.match(description)
if m:
entry.group = m.group(1)
else:
self.warn("Unable to find group in " + description)
raise IndexerResultParsingRowException("Unable to find usenet group")
entry.size = long(item.find("enclosure").attrib["length"])
entry.pubDate = item.find("pubDate").text
pubdate = arrow.get(entry.pubDate, 'ddd, DD MMM YYYY HH:mm:ss Z')
entry.epoch = pubdate.timestamp
entry.pubdate_utc = str(pubdate)
entry.age_days = (arrow.utcnow() - pubdate).days
entry.precise_date = True
entry.link = item.find("link").text
entry.has_nfo = NzbSearchResult.HAS_NFO_MAYBE
categoryid = item.find("categoryid").text
entry.details_link = self.get_details_link(entry.indexerguid)
if categoryid in omgwtf_to_categories.keys():
entry.category = getCategoryByName(omgwtf_to_categories[categoryid])
else:
entry.category = getUnknownCategory()
accepted, reason = self.accept_result(entry, searchRequest, self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected += 1
self.debug("Rejected search result. Reason: %s" % reason)
return IndexerProcessingResult(entries=entries, queries=[], total=len(entries), total_known=True, has_more=False, rejected=countRejected)
else:
self.warn("Unknown response type: %s" % xml_response[:100])
return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=countRejected)
def test_pick_indexers(self):
config.settings.searching.generate_queries = []
config.settings.indexers.extend([self.newznab1, self.newznab2])
getIndexerSettingByName("womble").enabled = True
getIndexerSettingByName("womble").accessType = "both"
getIndexerSettingByName("nzbclub").enabled = True
getIndexerSettingByName("nzbclub").accessType = "both"
read_indexers_from_config()
search_request = SearchRequest()
indexers = search.pick_indexers(search_request)
self.assertEqual(3, len(indexers))
# Indexers with tv search and which support queries (actually searching for particular releases)
search_request.query = "bla"
indexers = search.pick_indexers(search_request)
self.assertEqual(3, len(indexers))
# Indexers with tv search, including those that only provide a list of latest releases (womble) but excluding the one that needs a query (nzbclub)
search_request.query = None
indexers = search.pick_indexers(search_request)
self.assertEqual(3, len(indexers))
search_request.identifier_key = "tvdbid"
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
self.assertEqual("newznab1", indexers[0].name)
self.assertEqual("newznab2", indexers[1].name)
search_request.identifier_key = "imdbid"
search_request.category = getCategoryByName("movies")
indexers = search.pick_indexers(search_request)
self.assertEqual(1, len(indexers))
self.assertEqual("newznab1", indexers[0].name)
# WIth query generation NZBClub should also be returned
infos.title_from_id = mock
config.settings.searching.generate_queries = ["internal"]
search_request.identifier_key = "tvdbid"
search_request.query = None
search_request.category = None
indexers = search.pick_indexers(search_request)
self.assertEqual(3, len(indexers))
self.assertEqual("nzbclub", indexers[0].name)
self.assertEqual("newznab1", indexers[1].name)
self.assertEqual("newznab2", indexers[2].name)
# Test picking depending on internal, external, both
getIndexerSettingByName("womble").enabled = False
getIndexerSettingByName("nzbclub").enabled = False
getIndexerSettingByName("newznab1").accessType = "both"
search_request.internal = True
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
search_request.internal = False
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
config.settings.indexers = [self.newznab1, self.newznab2]
getIndexerSettingByName("newznab1").accessType = "external"
read_indexers_from_config()
search_request.internal = True
indexers = search.pick_indexers(search_request)
self.assertEqual(1, len(indexers))
search_request.internal = False
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
getIndexerSettingByName("newznab1").accessType = "internal"
read_indexers_from_config()
search_request.internal = True
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
search_request.internal = False
indexers = search.pick_indexers(search_request)
self.assertEqual(1, len(indexers))
def test_pick_indexers(self):
config.settings.searching.generate_queries = []
config.settings.indexers.extend([self.newznab1, self.newznab2])
getIndexerSettingByName("womble").enabled = True
getIndexerSettingByName("womble").accessType = "both"
getIndexerSettingByName("nzbclub").enabled = True
getIndexerSettingByName("nzbclub").accessType = "both"
read_indexers_from_config()
search_request = SearchRequest()
indexers = search.pick_indexers(search_request)
self.assertEqual(3, len(indexers))
# Indexers with tv search and which support queries (actually searching for particular releases)
search_request.query = "bla"
indexers = search.pick_indexers(search_request)
self.assertEqual(3, len(indexers))
# Indexers with tv search, including those that only provide a list of latest releases (womble) but excluding the one that needs a query (nzbclub)
search_request.query = None
indexers = search.pick_indexers(search_request)
self.assertEqual(3, len(indexers))
search_request.identifier_key = "tvdbid"
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
self.assertEqual("newznab1", indexers[0].name)
self.assertEqual("newznab2", indexers[1].name)
search_request.identifier_key = "imdbid"
search_request.category = getCategoryByName("movies")
indexers = search.pick_indexers(search_request)
self.assertEqual(1, len(indexers))
self.assertEqual("newznab1", indexers[0].name)
# WIth query generation NZBClub should also be returned
infos.title_from_id = mock
config.settings.searching.generate_queries = ["internal"]
search_request.identifier_key = "tvdbid"
search_request.query = None
search_request.category = None
indexers = search.pick_indexers(search_request)
self.assertEqual(3, len(indexers))
self.assertEqual("nzbclub", indexers[0].name)
self.assertEqual("newznab1", indexers[1].name)
self.assertEqual("newznab2", indexers[2].name)
# Test picking depending on internal, external, both
getIndexerSettingByName("womble").enabled = False
getIndexerSettingByName("nzbclub").enabled = False
getIndexerSettingByName("newznab1").accessType = "both"
search_request.internal = True
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
search_request.internal = False
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
config.settings.indexers = [self.newznab1, self.newznab2]
getIndexerSettingByName("newznab1").accessType = "external"
read_indexers_from_config()
search_request.internal = True
indexers = search.pick_indexers(search_request)
self.assertEqual(1, len(indexers))
search_request.internal = False
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
getIndexerSettingByName("newznab1").accessType = "internal"
read_indexers_from_config()
search_request.internal = True
indexers = search.pick_indexers(search_request)
self.assertEqual(2, len(indexers))
search_request.internal = False
indexers = search.pick_indexers(search_request)
self.assertEqual(1, len(indexers))
def process_query_result(self, xml_response, searchRequest, maxResults=None):
self.debug("Started processing results")
if "0 results found" in xml_response:
return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=0)
if "search to short" in xml_response:
self.info("omgwtf says the query was too short")
return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=0)
entries = []
countRejected = 0
try:
tree = ET.fromstring(xml_response)
except Exception:
self.exception("Error parsing XML: %s..." % xml_response[:500])
raise IndexerResultParsingException("Error parsing XML", self)
if tree.tag == "xml":
total = int(tree.find("info").find("results").text)
current_page = int(tree.find("info").find("current_page").text)
total_pages = int(tree.find("info").find("pages").text)
has_more = current_page < total_pages
for item in tree.find("search_req").findall("post"):
entry = self.parseItem(item)
accepted, reason = self.accept_result(entry, searchRequest, self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected += 1
self.debug("Rejected search result. Reason: %s" % reason)
return IndexerProcessingResult(entries=entries, queries=[], total=total, total_known=True, has_more=has_more, rejected=countRejected)
elif tree.tag == "rss":
for item in tree.find("channel").findall("item"):
entry = self.create_nzb_search_result()
indexerguid = item.find("guid").text
m = self.regexGuid.match(indexerguid)
if m:
entry.indexerguid = m.group(1)
else:
self.warn("Unable to find GUID in " + indexerguid)
raise IndexerResultParsingRowException("Unable to find GUID")
entry.title = item.find("title").text
description = item.find("description").text
m = self.regexGroup.match(description)
if m:
entry.group = m.group(1)
else:
self.warn("Unable to find group in " + description)
raise IndexerResultParsingRowException("Unable to find usenet group")
entry.size = long(item.find("enclosure").attrib["length"])
entry.pubDate = item.find("pubDate").text
pubdate = arrow.get(entry.pubDate, 'ddd, DD MMM YYYY HH:mm:ss Z')
entry.epoch = pubdate.timestamp
entry.pubdate_utc = str(pubdate)
entry.age_days = (arrow.utcnow() - pubdate).days
entry.precise_date = True
entry.link = item.find("link").text
entry.has_nfo = NzbSearchResult.HAS_NFO_MAYBE
categoryid = item.find("categoryid").text
entry.details_link = self.get_details_link(entry.indexerguid)
if categoryid in omgwtf_to_categories.keys():
entry.category = getCategoryByName(omgwtf_to_categories[categoryid])
else:
entry.category = getUnknownCategory()
accepted, reason = self.accept_result(entry, searchRequest, self.supportedFilters)
if accepted:
entries.append(entry)
else:
countRejected += 1
self.debug("Rejected search result. Reason: %s" % reason)
return IndexerProcessingResult(entries=entries, queries=[], total=len(entries), total_known=True, has_more=False, rejected=countRejected)
else:
self.warn("Unknown response type: %s" % xml_response[:100])
return IndexerProcessingResult(entries=[], queries=[], total=0, total_known=True, has_more=False, rejected=countRejected)
