def create_nzb_search_result(self):
result = NzbSearchResult(indexer=self.name,
indexerscore=self.score,
attributes=[{
"name": "hydraIndexerName",
"value": self.settings.name
}, {
"name": "hydraIndexerHost",
"value": self.settings.host
}, {
"name": "hydraIndexerScore",
"value": self.settings.score
}])
return result
def testFindDuplicatesNew(self):
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1",
poster="postera",
group="groupa")
result2 = NzbSearchResult(title="Title1",
epoch=0,
size=2,
indexer="2",
indexerguid="2",
poster="postera",
group="groupb")
result3 = NzbSearchResult(title="Title1",
epoch=0,
size=3,
indexer="3",
indexerguid="3",
poster="postera",
group="groupb")
results = search.find_duplicates([result1, result2, result3])
self.assertEqual(3, len(results))
def create_nzb_search_result(self):
return NzbSearchResult(indexer=self.name, indexerscore=self.score)
def testFindDuplicates(self):
config.settings.searching.duplicateAgeThreshold = 3600
config.settings.searching.duplicateSizeThresholdInPercent = 0.1
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1")
result2 = NzbSearchResult(title="Title2",
epoch=0,
size=1,
indexer="2",
indexerguid="2")
result3 = NzbSearchResult(title="Title2",
epoch=0,
size=1,
indexer="3",
indexerguid="3")
result4 = NzbSearchResult(title="Title3",
epoch=0,
size=1,
indexer="4",
indexerguid="4")
result5 = NzbSearchResult(title="TITLE1",
epoch=0,
size=1,
indexer="5",
indexerguid="5")
result6 = NzbSearchResult(title="Title4",
epoch=0,
size=1,
indexer="6",
indexerguid="6")
results = search.find_duplicates(
[result1, result2, result3, result4, result5, result6])
self.assertEqual(4, len(results))
self.assertEqual(2, len(results[0]))
self.assertEqual(2, len(results[1]))
self.assertEqual(1, len(results[2]))
self.assertEqual(1, len(results[3]))
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1")
result2 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="2",
indexerguid="2")
result3 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="3",
indexerguid="3")
result4 = NzbSearchResult(title="Title1",
epoch=100000000,
size=1,
indexer="4",
indexerguid="4")
results = search.find_duplicates([result1, result2, result3, result4])
self.assertEqual(2, len(results))
self.assertEqual(3, len(results[0]))
self.assertEqual(1, len(results[1]))
result1 = NzbSearchResult(
title="Title1",
epoch=0,
size=1,
indexer="1a",
indexerguid="1",
pubdate_utc=arrow.get(0).format('YYYY-MM-DD HH:mm:ss ZZ'))
result2 = NzbSearchResult(
title="Title1",
epoch=10000000,
size=1,
indexer="2a",
indexerguid="2",
pubdate_utc=arrow.get(10000000).format('YYYY-MM-DD HH:mm:ss ZZ'))
result3 = NzbSearchResult(
title="Title1",
epoch=0,
size=1,
indexer="1b",
indexerguid="3",
pubdate_utc=arrow.get(10000000).format('YYYY-MM-DD HH:mm:ss ZZ'))
result4 = NzbSearchResult(
title="Title1",
epoch=10000000,
size=1,
indexer="2b",
indexerguid="4",
pubdate_utc=arrow.get(10000000).format('YYYY-MM-DD HH:mm:ss ZZ'))
result5 = NzbSearchResult(
title="Title1",
epoch=1000000000,
size=1,
indexer="3",
indexerguid="5",
pubdate_utc=arrow.get(1000000000).format('YYYY-MM-DD HH:mm:ss ZZ'))
results = search.find_duplicates(
[result1, result2, result3, result4, result5])
results = sorted(results, key=lambda x: len(x), reverse=True)
self.assertEqual(3, len(results))
self.assertEqual(2, len(results[0]))
self.assertEqual(2, len(results[1]))
self.assertEqual(1, len(results[2]))
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1")
result2 = NzbSearchResult(title="Title1",
epoch=100000000,
size=1,
indexer="2",
indexerguid="2")
results = search.find_duplicates([result1, result2])
results = sorted(results, key=lambda x: len(x), reverse=True)
self.assertEqual(2, len(results))
self.assertEqual(1, len(results[0]))
self.assertEqual(1, len(results[1]))
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1")
result2 = NzbSearchResult(title="Title1",
epoch=1,
size=100000000,
indexer="2",
indexerguid="2")
results = search.find_duplicates([result1, result2])
self.assertEqual(2, len(results))
self.assertEqual(1, len(results[0]))
self.assertEqual(1, len(results[1]))
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1")
result2 = NzbSearchResult(title="Title1",
epoch=1,
size=1,
indexer="2",
indexerguid="2")
results = search.find_duplicates([result1, result2])
self.assertEqual(1, len(results))
self.assertEqual(2, len(results[0]))
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1")
result2 = NzbSearchResult(title="Title1",
epoch=30 * 1000 * 60,
size=1,
indexer="2",
indexerguid="2")
result3 = NzbSearchResult(title="Title1",
epoch=60 * 1000 * 60,
size=1,
indexer="2",
indexerguid="3")
results = search.find_duplicates([result1, result2, result3])
self.assertEqual(3, len(results))
self.assertEqual(1, len(results[0]))
self.assertEqual(1, len(results[1]))
self.assertEqual(1, len(results[2]))
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1")
result2 = NzbSearchResult(title="Title2",
epoch=1000000,
size=1,
indexer="2",
indexerguid="2")
result3 = NzbSearchResult(title="Title3",
epoch=5000000,
size=1,
indexer="2",
indexerguid="3")
results = search.find_duplicates([result1, result2, result3])
self.assertEqual(3, len(results))
self.assertEqual(1, len(results[0]))
self.assertEqual(1, len(results[1]))
self.assertEqual(1, len(results[2]))
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1")
result2 = NzbSearchResult(title="Title2",
epoch=0,
size=1,
indexer="2",
indexerguid="2")
result3 = NzbSearchResult(title="Title3",
epoch=0,
size=1,
indexer="2",
indexerguid="3")
results = search.find_duplicates([result1, result2, result3])
self.assertEqual(3, len(results))
self.assertEqual(1, len(results[0]))
self.assertEqual(1, len(results[1]))
self.assertEqual(1, len(results[2]))
# Same size and age and group but different posters (very unlikely)
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1",
poster="postera",
group="groupa")
result2 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="2",
indexerguid="2",
poster="posterb",
group="groupa")
results = search.find_duplicates([result1, result2])
self.assertEqual(2, len(results))
# Same size and age and poster but different groups (very unlikely)
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1",
poster="postera",
group="groupa")
result2 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="2",
indexerguid="2",
poster="postera",
group="groupb")
results = search.find_duplicates([result1, result2])
self.assertEqual(2, len(results))
# Same size and age and poster but unknown group inside of 3 hours
result1 = NzbSearchResult(title="Title1",
epoch=0,
size=1,
indexer="1",
indexerguid="1",
poster="postera")
result2 = NzbSearchResult(title="Title1",
epoch=60 * 60 * 2,
size=1,
indexer="2",
indexerguid="2",
poster="postera",
group="groupb")
results = search.find_duplicates([result1, result2])
self.assertEqual(1, len(results))
def testTestForDuplicate(self):
config.settings.searching.duplicateAgeThreshold = 120
age_threshold = config.settings.searching.duplicateAgeThreshold
size_threshold = config.settings.searching.duplicateSizeThresholdInPercent
config.settings.searching.duplicateSizeThresholdInPercent = 1
# same title, age and size
result1 = NzbSearchResult(title="A title",
epoch=0,
size=1,
indexer="a")
result2 = NzbSearchResult(title="A title",
epoch=0,
size=1,
indexer="b")
assert search.test_for_duplicate_age(result1, result2, age_threshold)
assert search.test_for_duplicate_size(result1, result2, size_threshold)
# size in threshold
result1 = NzbSearchResult(title="A title",
epoch=0,
size=100,
indexer="a")
result2 = NzbSearchResult(title="A title",
epoch=0,
size=101,
indexer="b")
assert search.test_for_duplicate_size(result1, result2, size_threshold)
# age in threshold
result1 = NzbSearchResult(title="A title",
epoch=0,
size=1,
indexer="a")
result2 = NzbSearchResult(title="A title",
epoch=age_threshold * 120 - 1,
size=1,
indexer="b")
assert search.test_for_duplicate_age(result1, result2, age_threshold)
# size outside of threshold -> duplicate
result1 = NzbSearchResult(title="A title",
epoch=0,
size=1,
indexer="a")
result2 = NzbSearchResult(title="A title",
epoch=0,
size=2,
indexer="b")
assert not search.test_for_duplicate_size(result1, result2,
size_threshold)
# age outside of threshold -> duplicate
result1 = NzbSearchResult(title="A title",
epoch=0,
size=1,
indexer="a")
result2 = NzbSearchResult(title="A title",
epoch=age_threshold * 120 * 1000 + 1,
size=0,
indexer="b")
assert not search.test_for_duplicate_age(result1, result2,
age_threshold)
# age and size inside of threshold
result1 = NzbSearchResult(title="A title",
epoch=0,
size=101,
indexer="a")
result2 = NzbSearchResult(title="A title",
epoch=age_threshold * 120 - 1,
size=101,
indexer="b")
assert search.test_for_duplicate_size(result1, result2, size_threshold)
assert search.test_for_duplicate_age(result1, result2, age_threshold)
# age and size outside of threshold -> duplicate
result1 = NzbSearchResult(title="A title",
epoch=0,
size=1,
indexer="a")
result2 = NzbSearchResult(title="A title",
epoch=age_threshold * 120 * 1000 + 1,
size=200,
indexer="b")
assert not search.test_for_duplicate_size(result1, result2,
size_threshold)
assert not search.test_for_duplicate_age(result1, result2,
age_threshold)