def test_determine_aliases_with_only_generic_vulns_in_response(self):
"""Only generic vulnerabilities should be scraped.
Tests:
patchfinder.spiders.default_spider.DefaultSpider.determine_aliases
"""
vuln_id = "GLSA-200607-06"
vuln = context.create_vuln(vuln_id)
response = fake_response(
file_name="./mocks/gentoo_glsa_200607_06.xml",
url=vuln.base_url,
# The response content-type Scrapy gets for this URL is text/plain.
content_type=b"text/plain",
)
expected_urls = {
"https://gitweb.gentoo.org/data/glsa.git/plain/glsa-200601-06.xml",
"https://gitweb.gentoo.org/data/glsa.git/plain/glsa-200603-02.xml",
"https://gitweb.gentoo.org/data/glsa.git/plain/glsa-200605-04.xml",
}
self.spider.set_context(vuln)
req_urls = {
request.url
for request in self.spider.determine_aliases(response)
}
self.assertFalse(self.spider.cves)
self.assertEqual(req_urls, expected_urls)
def test_determine_aliases_with_generic_vulns_and_cves(self):
"""Aliases should be scraped and requests for generic vulns generated.
Tests:
patchfinder.spiders.default_spider.DefaultSpider.determine_aliases
"""
vuln_id = "GLSA-200602-01"
vuln = context.create_vuln(vuln_id)
expected_aliases = {"CVE-2005-4048"}
response = fake_response(
file_name="./mocks/gentoo_glsa_200602_01.xml",
url=vuln.base_url,
# The response content-type Scrapy gets for this URL is text/plain.
content_type=b"text/plain",
)
self.spider.set_context(vuln)
req_urls = [
request.url for request in self.spider.determine_aliases(response)
]
got_aliases = set(alias.vuln_id for alias in self.spider.cves)
self.assertEqual(expected_aliases, got_aliases)
# Requests for generic vulnerabilities found should be yielded.
self.assertIn(
"https://gitweb.gentoo.org/data/glsa.git/plain/glsa-200601-06.xml",
req_urls,
)
for cve in self.spider.cves:
self.assertTrue(url in req_urls for url in cve.entrypoint_urls)
def test_determine_aliases_with_no_generic_vulns(self):
"""The aliases of a vulnerability should be scraped from the response.
Tests:
patchfinder.spiders.default_spider.DefaultSpider.determine_aliases
"""
vuln_id = "DSA-4444-1"
vuln = context.create_vuln(vuln_id)
expected_aliases = {
"CVE-2018-12126",
"CVE-2018-12127",
"CVE-2018-12130",
"CVE-2019-11091",
}
response = fake_response(
file_name="./mocks/debsec_dsa_4444_1.html",
url=vuln.base_url,
content_type=b"text/html",
)
self.spider.set_context(vuln)
req_urls = [
request.url for request in self.spider.determine_aliases(response)
]
got_aliases = set(alias.vuln_id for alias in self.spider.cves)
self.assertEqual(expected_aliases, got_aliases)
for cve in self.spider.cves:
self.assertTrue(url in req_urls for url in cve.entrypoint_urls)
def spawn_crawler(args):
vuln = context.create_vuln(args["vuln_id"])
scrapy_settings = dict(ScrapySettings(values=args))
patchfinder_settings = dict(PatchfinderSettings(values=args))
if not vuln:
return False
process = CrawlerProcess(scrapy_settings)
process.crawl(default_spider.DefaultSpider,
vuln=vuln,
settings=patchfinder_settings)
process.start()
return True
def test_determine_aliases_with_same_alias_in_resulting_response(self):
"""Vulnerability should not scrape same alias in generic vuln response.
In this case, the input vuln response gives a generic vuln and an alias.
A request should be yielded for the generic vuln's base URL. On parsing
the response for the generic vuln's base URL, the same alias previously
found is found again. Thus, the same alias should not be scraped again,
and requests for the alias' entrypoint URLs should not be yielded again.
Tests:
patchfinder.spiders.default_spider.DefaultSpider.determine_aliases
"""
vuln_id = "GLSA-200602-01"
vuln = context.create_vuln(vuln_id)
expected_req_url = (
"https://gitweb.gentoo.org/data/glsa.git/plain/glsa-200601-06.xml")
expected_aliases = {"CVE-2005-4048"}
response = fake_response(
file_name="./mocks/gentoo_glsa_200602_01.xml",
url=vuln.base_url,
# The response content-type Scrapy gets for this URL is text/plain.
content_type=b"text/plain",
)
self.spider.set_context(vuln)
req_urls = [
request.url for request in self.spider.determine_aliases(response)
]
# Check if the expected request URL is present in req_urls (for the
# generic vuln).
# If so, execute determine_aliases with a response for that URL.
self.assertIn(expected_req_url, req_urls)
response = fake_response(
file_name="./mocks/gentoo_glsa_200602_01.xml",
url=expected_req_url,
# The response content-type Scrapy gets for this URL is text/plain.
content_type=b"text/plain",
)
req_urls = [
request.url for request in self.spider.determine_aliases(response)
]
# After executing determine_aliases with both the above responses,
# got aliases should be equal to expected aliases.
got_aliases = [alias.vuln_id for alias in self.spider.cves]
self.assertEqual(len(got_aliases), 1)
# No requests should be yielded from the second time determine_aliases
# is executed.
self.assertFalse(req_urls)
self.assertEqual(expected_aliases, set(got_aliases))
def test_start_requests_with_vuln(self):
"""Start requests for a normal vulnerability.
For a normal vulnerability (i.e. a CVE), the spider should generate as
many requests as the entrypoint URLs for the vulnerability.
Tests:
patchfinder.spiders.default_spider.DefaultSpider.start_requests
"""
vuln = context.create_vuln("CVE-2016-4796")
self.spider.set_context(vuln)
requests = list(self.spider.start_requests())
self.assertEqual(len(requests), len(vuln.entrypoint_urls))
def test_start_requests_with_generic_vuln(self):
"""Start requests for a generic vulnerability.
For a generic vulnerability, the spider should generate only
one request, i.e. for the base URL of the vulnerability.
Tests:
patchfinder.spiders.default_spider.DefaultSpider.start_requests
"""
vuln = context.create_vuln("DSA-4431-1")
self.spider.set_context(vuln)
requests = list(self.spider.start_requests())
self.assertEqual(len(requests), 1)
self.assertEqual(requests[0].url, vuln.base_url)
def test_determine_aliases_with_no_aliases_in_response(self):
"""No aliases and no vulerabilities should be scraped.
Tests:
patchfinder.spiders.default_spider.DefaultSpider.determine_aliases
"""
vuln_id = "GLSA-200311-04"
vuln = context.create_vuln(vuln_id)
response = fake_response(
file_name="./mocks/gentoo_glsa_200311_04.xml",
url=vuln.base_url,
# The response content-type Scrapy gets for this URL is text/plain.
content_type=b"text/plain",
)
self.spider.set_context(vuln)
req_urls = [
request.url for request in self.spider.determine_aliases(response)
]
self.assertFalse(self.spider.cves)
self.assertFalse(req_urls)
def test_create_vuln_for_cve(self):
"""Vulnerability instantiation for a CVE"""
vuln = context.create_vuln("CVE-2018-20406", {"upstream": ["python"]})
self.assertTrue(vuln)
def test_create_vuln_for_rhsa(self):
"""Vulnerability instantiation for a RHSA"""
vuln = context.create_vuln("RHSA-2019:0094")
self.assertTrue(vuln)
def test_create_vuln_for_glsa(self):
"""Vulnerability instantiation for a GLSA"""
vuln = context.create_vuln("GLSA-200602-01")
self.assertTrue(vuln)
def test_create_vuln_for_dsa(self):
"""Vulnerability instantiation for a DSA"""
vuln = context.create_vuln("DSA-4444-1")
self.assertTrue(vuln)
def test_create_vuln_for_inconsistent_cve(self):
"""Vulnerability instantiation for an inconsistent CVE notation"""
vuln = context.create_vuln("cve 2019-4040")
self.assertTrue(vuln)
def test_create_vuln_for_unknown_vuln(self):
"""Vulnerability instantiation for an unknown vulnerability"""
vuln = context.create_vuln("foo bar")
self.assertFalse(vuln)
