def test_extract_public_keys(self):
meta_payload = {
"public_keys": [
{
"key_identifier": "90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1c2S+CINXEihVeXz95He1bmWfhPc\n"
"ri7XBJXSEtW2IuZZyrlQP7wDXVupMZ3OsGsZaNX0SL4/nOx2S4OTrF1miA==\n"
"-----END PUBLIC KEY-----\n",
"is_current": True,
}
]
}
cache = integrations.PublicKeysCache(cache_time=12)
vuln_report_verifier = osv.VulnerabilityReportVerifier(
public_keys_api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=cache,
)
keys = vuln_report_verifier.extract_public_keys(pubkey_api_data=meta_payload)
assert keys == [
{
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1c2S+CINXEihVeXz95He1bmWfhPc\n"
"ri7XBJXSEtW2IuZZyrlQP7wDXVupMZ3OsGsZaNX0SL4/nOx2S4OTrF1miA==\n"
"-----END PUBLIC KEY-----\n",
"key_id": "90a421169f0a406205f1563a953312f0be"
"898d3c7b6c06b681aa86a874555f4a",
}
]
assert cache.cache == keys
def test_extract_public_keys(self):
meta_payload = {
"public_keys": [
{
"key_identifier": "90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----",
"is_current": True,
}
]
}
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=cache,
)
keys = github_verifier.extract_public_keys(pubkey_api_data=meta_payload)
assert keys == [
{
"key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcD"
"QgAE9MJJHnMfn2+H4xL4YaPDA4RpJqUq\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ"
"8qpVIW4clayyef9gWhFbNHWAA==\n-----END PUBLIC KEY-----",
"key_id": "90a421169f0a406205f1563a953312f0be"
"898d3c7b6c06b681aa86a874555f4a",
}
]
assert cache.cache == keys
def test_verify_cache_miss(self):
# Example taken from
# https://gist.github.com/ewjoachim/7dde11c31d9686ed6b4431c3ca166da2
meta_payload = {
"public_keys": [
{
"key_identifier": "90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1c2S+CINXEihVeXz95He1bmWfhPc\n"
"ri7XBJXSEtW2IuZZyrlQP7wDXVupMZ3OsGsZaNX0SL4/nOx2S4OTrF1miA==\n"
"-----END PUBLIC KEY-----\n",
"is_current": True,
}
]
}
response = pretend.stub(
json=lambda: meta_payload, raise_for_status=lambda: None
)
session = pretend.stub(get=lambda *a, **k: response)
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
cache = integrations.PublicKeysCache(cache_time=12)
vuln_report_verifier = osv.VulnerabilityReportVerifier(
public_keys_api_url="http://foo",
session=session,
metrics=metrics,
public_keys_cache=cache,
)
key_id = "90a421169f0a406205f1563a953312f0be898d3c7b6c06b681aa86a874555f4a"
signature = (
"MEUCIQDz4wvDZjrX2YHsWhmu5Cvvp0gny6xYMD0AGrwEhTHGRAIgXCSvx"
"Tl2SdnaY7fImXFRSKhbw3IRf68g1LMaQRetM80="
)
payload = (
b'[{"project":"vuln_project",'
b'"versions":["v1","v2"],'
b'"id":"vuln_id",'
b'"link":"vulns.com/vuln_id",'
b'"aliases":["vuln_alias"]}]'
)
assert (
vuln_report_verifier.verify(
payload=payload, key_id=key_id, signature=signature
)
is True
)
assert metrics.increment.calls == [
pretend.call("warehouse.vulnerabilities.osv.auth.cache.miss"),
pretend.call("warehouse.vulnerabilities.osv.auth.success"),
]
def test_get_cached_public_key_cache_miss_no_cache(self):
metrics = pretend.stub()
session = pretend.stub()
cache = integrations.PublicKeysCache(cache_time=12)
vuln_report_verifier = osv.VulnerabilityReportVerifier(
public_keys_api_url="http://foo",
session=session,
metrics=metrics,
public_keys_cache=cache,
)
with pytest.raises(integrations.CacheMissError):
vuln_report_verifier._get_cached_public_keys()
def test_init(self):
metrics = pretend.stub()
session = pretend.stub()
cache = integrations.PublicKeysCache(cache_time=12)
vuln_report_verifier = osv.VulnerabilityReportVerifier(
session=session,
metrics=metrics,
public_keys_cache=cache,
)
# assert vuln_report_verifier._session is session
assert vuln_report_verifier._metrics is metrics
assert vuln_report_verifier._public_keys_cache is cache
def test_get_cached_public_key_cache_miss_no_cache(self):
metrics = pretend.stub()
session = pretend.stub()
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
public_keys_cache=cache,
)
with pytest.raises(integrations.CacheMissError):
github_verifier._get_cached_public_keys()
def test_verify_cache_miss(self):
# Example taken from
# https://gist.github.com/ewjoachim/7dde11c31d9686ed6b4431c3ca166da2
meta_payload = {
"public_keys": [
{
"key_identifier": "90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----",
"is_current": True,
}
]
}
response = pretend.stub(
json=lambda: meta_payload, raise_for_status=lambda: None
)
session = pretend.stub(get=lambda *a, **k: response)
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
api_token="api-token",
public_keys_cache=cache,
)
key_id = "90a421169f0a406205f1563a953312f0be898d3c7b6c06b681aa86a874555f4a"
signature = (
"MEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/"
"cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A=="
)
payload = (
b'[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299'
b'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/'
b'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]'
)
assert (
github_verifier.verify(payload=payload, key_id=key_id, signature=signature)
is True
)
assert metrics.increment.calls == [
pretend.call("warehouse.token_leak.github.auth.cache.miss"),
pretend.call("warehouse.token_leak.github.auth.success"),
]
def test_extract_public_keys_error(self, payload, expected):
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=cache,
)
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
list(github_verifier.extract_public_keys(pubkey_api_data=payload))
assert exc.value.reason == "public_key_api.format_error"
assert str(exc.value) == expected
assert cache.cache is None
def test_get_cached_public_key_cache_hit(self):
metrics = pretend.stub()
session = pretend.stub()
cache = integrations.PublicKeysCache(cache_time=12)
cache_value = pretend.stub()
cache.set(now=time.time(), value=cache_value)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
public_keys_cache=cache,
)
assert github_verifier._get_cached_public_keys() is cache_value
def test_unimplemented(self):
metrics = pretend.stub(
increment=pretend.call_recorder(lambda str: None))
cache = integrations.PublicKeysCache(cache_time=10)
payload_verifier = integrations.PayloadVerifier(
metrics=metrics, public_keys_cache=cache)
with pytest.raises(NotImplementedError):
payload_verifier.metric_name
with pytest.raises(NotImplementedError):
payload_verifier.retrieve_public_key_payload()
with pytest.raises(NotImplementedError):
payload_verifier.extract_public_keys({})
def test_extract_public_keys_error(self, payload, expected):
cache = integrations.PublicKeysCache(cache_time=12)
vuln_report_verifier = osv.VulnerabilityReportVerifier(
public_keys_api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=cache,
)
with pytest.raises(osv.OSVPublicKeyAPIError) as exc:
list(vuln_report_verifier.extract_public_keys(pubkey_api_data=payload))
assert exc.value.reason == "public_key_api.format_error"
assert str(exc.value) == expected
assert cache.cache is None
def test_get_cached_public_key_cache_hit(self):
metrics = pretend.stub()
session = pretend.stub()
cache = integrations.PublicKeysCache(cache_time=12)
cache_value = pretend.stub()
cache.set(now=time.time(), value=cache_value)
vuln_report_verifier = osv.VulnerabilityReportVerifier(
public_keys_api_url="http://foo",
session=session,
metrics=metrics,
public_keys_cache=cache,
)
assert vuln_report_verifier._get_cached_public_keys() is cache_value
def test_verify_cache_hit(self):
session = pretend.stub()
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
cache = integrations.PublicKeysCache(cache_time=12)
cache.cached_at = time.time()
cache.cache = [
{
"key_id": "90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1c2S+CINXEihVeXz95He1bmWfhPc\n"
"ri7XBJXSEtW2IuZZyrlQP7wDXVupMZ3OsGsZaNX0SL4/nOx2S4OTrF1miA==\n"
"-----END PUBLIC KEY-----\n",
}
]
vuln_report_verifier = osv.VulnerabilityReportVerifier(
public_keys_api_url="http://foo",
session=session,
metrics=metrics,
public_keys_cache=cache,
)
key_id = "90a421169f0a406205f1563a953312f0be898d3c7b6c06b681aa86a874555f4a"
signature = (
"MEUCIQDz4wvDZjrX2YHsWhmu5Cvvp0gny6xYMD0AGrwEhTHGRAIgXCSvx"
"Tl2SdnaY7fImXFRSKhbw3IRf68g1LMaQRetM80="
)
payload = (
b'[{"project":"vuln_project",'
b'"versions":["v1","v2"],'
b'"id":"vuln_id",'
b'"link":"vulns.com/vuln_id",'
b'"aliases":["vuln_alias"]}]'
)
assert (
vuln_report_verifier.verify(
payload=payload, key_id=key_id, signature=signature
)
is True
)
assert metrics.increment.calls == [
pretend.call("warehouse.vulnerabilities.osv.auth.cache.hit"),
pretend.call("warehouse.vulnerabilities.osv.auth.success"),
]
def test_verify_error(self):
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=metrics,
api_token="api-token",
public_keys_cache=cache,
)
github_verifier.retrieve_public_key_payload = pretend.raiser(
integrations.InvalidPayloadSignatureError("Bla", "bla")
)
assert github_verifier.verify(payload={}, key_id="a", signature="a") is False
assert metrics.increment.calls == [
pretend.call("warehouse.token_leak.github.auth.cache.miss"),
pretend.call("warehouse.token_leak.github.auth.error.bla"),
]
def test_init(self):
metrics = pretend.stub()
session = pretend.stub()
token = "api_token"
url = "http://foo"
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
api_token=token,
public_keys_cache=cache,
)
assert github_verifier._session is session
assert github_verifier._metrics is metrics
assert github_verifier._api_token == token
assert github_verifier._api_url == url
assert github_verifier._public_keys_cache is cache
def test_verify_error(self):
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
cache = integrations.PublicKeysCache(cache_time=12)
vuln_report_verifier = osv.VulnerabilityReportVerifier(
public_keys_api_url="http://foo",
session=pretend.stub(),
metrics=metrics,
public_keys_cache=cache,
)
vuln_report_verifier.retrieve_public_key_payload = pretend.raiser(
integrations.InvalidPayloadSignatureError("Bla", "bla")
)
assert (
vuln_report_verifier.verify(payload={}, key_id="a", signature="a") is False
)
assert metrics.increment.calls == [
pretend.call("warehouse.vulnerabilities.osv.auth.cache.miss"),
pretend.call("warehouse.vulnerabilities.osv.auth.error.bla"),
]
def test_set(self):
cache = integrations.PublicKeysCache(cache_time=10)
cache.set(now=1, value="foo")
assert cache.cached_at == 1
assert cache.cache == "foo"
def test_get_old_cache(self):
cache = integrations.PublicKeysCache(cache_time=10)
cache.set(now=5, value="foo")
with pytest.raises(integrations.CacheMissError):
cache.get(now=20)
def test_get_valid(self):
cache = integrations.PublicKeysCache(cache_time=10)
cache.set(now=5, value="foo")
assert cache.get(now=10) == "foo"
def test_get_no_cache(self):
cache = integrations.PublicKeysCache(cache_time=10)
with pytest.raises(integrations.CacheMissError):
cache.get(now=1)
except ExtractionFailedError:
raise InvalidTokenLeakRequestError(
"Cannot extract token from recieved match", reason="extraction"
)
return cls(
token=extracted_token, public_url=record["url"], source=record.get("source")
)
class GitHubPublicKeyMetaAPIError(InvalidTokenLeakRequestError):
pass
PUBLIC_KEYS_CACHE_TIME = 60 * 30 # 30 minutes
PUBLIC_KEYS_CACHE = integrations.PublicKeysCache(cache_time=PUBLIC_KEYS_CACHE_TIME)
class GitHubTokenScanningPayloadVerifier(integrations.PayloadVerifier):
"""
Checks payload signature using:
- `requests` for HTTP calls
- `cryptography` for signature verification
"""
def __init__(
self,
session,
metrics,
api_url: str,
api_token: Optional[str] = None,
import time
import requests
from warehouse import integrations
from warehouse.integrations import vulnerabilities
class OSVPublicKeyAPIError(vulnerabilities.InvalidVulnerabilityReportError):
pass
OSV_PUBLIC_KEYS_URL = "https://osv.dev/public_keys/pypa"
DEFAULT_PUBLIC_KEYS_CACHE_SECONDS = 60 * 30 # 30 minutes
DEFAULT_PUBLIC_KEYS_CACHE = integrations.PublicKeysCache(
cache_time=DEFAULT_PUBLIC_KEYS_CACHE_SECONDS
)
class VulnerabilityReportVerifier(vulnerabilities.VulnerabilityVerifier):
def __init__(
self,
session,
metrics,
public_keys_api_url: str = OSV_PUBLIC_KEYS_URL,
public_keys_cache=DEFAULT_PUBLIC_KEYS_CACHE,
):
super().__init__(
metrics=metrics, source="osv", public_keys_cache=public_keys_cache
)
self._session = session
