def test_extract_public_keys(self):
meta_payload = {
"public_keys": [{
"key_identifier":
"90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key":
"-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----",
"is_current":
True,
}]
}
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=pretend.stub(), metrics=pretend.stub())
keys = list(
verifier._extract_public_keys(pubkey_api_data=meta_payload))
assert keys == [{
"key":
"-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcD"
"QgAE9MJJHnMfn2+H4xL4YaPDA4RpJqUq\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ"
"8qpVIW4clayyef9gWhFbNHWAA==\n-----END PUBLIC KEY-----",
"key_id":
"90a421169f0a406205f1563a953312f0be"
"898d3c7b6c06b681aa86a874555f4a",
}]
def test_check_signature_invalid_signature(self):
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=pretend.stub(),
)
public_key = (
"-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----"
)
# Changed the initial N for an M
signature = (
"NEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/"
"cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A=="
)
payload = (
b'[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299'
b'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/'
b'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]'
)
with pytest.raises(integrations.InvalidPayloadSignatureError) as exc:
github_verifier._check_signature(
payload=payload, public_key=public_key, signature=signature
)
assert str(exc.value) == "Invalid signature"
assert exc.value.reason == "invalid_signature"
def test_retrieve_public_key_payload(self):
meta_payload = {
"public_keys": [
{
"key_identifier": "90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----",
"is_current": True,
}
]
}
response = pretend.stub(
json=lambda: meta_payload, raise_for_status=lambda: None
)
session = pretend.stub(get=pretend.call_recorder(lambda *a, **k: response))
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
api_token="api-token",
public_keys_cache=pretend.stub(),
)
assert github_verifier.retrieve_public_key_payload() == meta_payload
assert session.get.calls == [
pretend.call(
"http://foo",
headers={"Authorization": "token api-token"},
)
]
def test_verify_cache_hit(self):
session = pretend.stub()
metrics = pretend.stub(
increment=pretend.call_recorder(lambda str: None))
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=session, metrics=metrics, api_token="api-token")
verifier.public_keys_cached_at = time.time()
verifier.public_keys_cache = [{
"key_id":
"90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key":
"-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----",
}]
key_id = "90a421169f0a406205f1563a953312f0be898d3c7b6c06b681aa86a874555f4a"
signature = ("MEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/"
"cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A==")
payload = (
'[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299'
'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/'
'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]')
assert (verifier.verify(
payload=payload, key_id=key_id, signature=signature) is True)
assert metrics.increment.calls == [
pretend.call("warehouse.token_leak.github.auth.cache.hit"),
pretend.call("warehouse.token_leak.github.auth.success"),
]
def test_extract_public_keys(self):
meta_payload = {
"public_keys": [
{
"key_identifier": "90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----",
"is_current": True,
}
]
}
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=cache,
)
keys = github_verifier.extract_public_keys(pubkey_api_data=meta_payload)
assert keys == [
{
"key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcD"
"QgAE9MJJHnMfn2+H4xL4YaPDA4RpJqUq\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ"
"8qpVIW4clayyef9gWhFbNHWAA==\n-----END PUBLIC KEY-----",
"key_id": "90a421169f0a406205f1563a953312f0be"
"898d3c7b6c06b681aa86a874555f4a",
}
]
assert cache.cache == keys
def test_get_cached_public_key_cache_miss_no_cache(self):
metrics = pretend.stub()
session = pretend.stub()
verifier = utils.GitHubTokenScanningPayloadVerifier(session=session,
metrics=metrics)
with pytest.raises(utils.CacheMiss):
verifier._get_cached_public_keys()
def test_extract_public_keys_error(self, payload, expected):
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=pretend.stub(), metrics=pretend.stub())
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
list(verifier._extract_public_keys(pubkey_api_data=payload))
assert exc.value.reason == "public_key_api.format_error"
assert str(exc.value) == expected
def test_headers_auth_no_token(self):
headers = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
api_token=None,
public_keys_cache=pretend.stub(),
)._headers_auth()
assert headers == {}
def test_headers_auth_token(self):
headers = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
api_token="api-token",
public_keys_cache=pretend.stub(),
)._headers_auth()
assert headers == {"Authorization": "token api-token"}
def test_check_public_key_error(self):
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=pretend.stub(), metrics=pretend.stub())
with pytest.raises(utils.InvalidTokenLeakRequest) as exc:
verifier._check_public_key(github_public_keys=[], key_id="c")
assert str(exc.value) == "Key c not found in github public keys"
assert exc.value.reason == "wrong_key_id"
def test_get_cached_public_key_cache_hit(self):
metrics = pretend.stub()
session = pretend.stub()
verifier = utils.GitHubTokenScanningPayloadVerifier(session=session,
metrics=metrics)
verifier.public_keys_cached_at = time.time()
cache = verifier.public_keys_cache = pretend.stub()
assert verifier._get_cached_public_keys() is cache
def test_retrieve_public_key_payload_connection_error(self):
session = pretend.stub(get=pretend.raiser(requests.ConnectionError))
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=session, metrics=pretend.stub())
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
verifier._retrieve_public_key_payload()
assert str(exc.value) == "Could not connect to GitHub"
assert exc.value.reason == "public_key_api.network_error"
def test_init(self):
metrics = pretend.stub()
session = pretend.stub()
token = "api_token"
verifier = utils.GitHubTokenScanningPayloadVerifier(session=session,
metrics=metrics,
api_token=token)
assert verifier._session is session
assert verifier._metrics is metrics
assert verifier._api_token == token
def test_check_public_key(self):
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=pretend.stub(),
)
keys = [
{"key_id": "a", "key": "b"},
{"key_id": "c", "key": "d"},
]
assert github_verifier._check_public_key(public_keys=keys, key_id="c") == "d"
def test_check_public_key_error(self):
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=pretend.stub(),
)
with pytest.raises(integrations.InvalidPayloadSignatureError) as exc:
github_verifier._check_public_key(public_keys=[], key_id="c")
assert str(exc.value) == "Key c not found in public keys"
assert exc.value.reason == "wrong_key_id"
def test_verify_error(self):
metrics = pretend.stub(
increment=pretend.call_recorder(lambda str: None))
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=pretend.stub(), metrics=metrics, api_token="api-token")
verifier._retrieve_public_key_payload = pretend.raiser(
utils.InvalidTokenLeakRequest("Bla", "bla"))
assert verifier.verify(payload={}, key_id="a", signature="a") is False
assert metrics.increment.calls == [
pretend.call("warehouse.token_leak.github.auth.cache.miss"),
pretend.call("warehouse.token_leak.github.auth.error.bla"),
]
def test_get_cached_public_key_cache_miss_no_cache(self):
metrics = pretend.stub()
session = pretend.stub()
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
public_keys_cache=cache,
)
with pytest.raises(integrations.CacheMissError):
github_verifier._get_cached_public_keys()
def test_retrieve_public_key_payload_http_error(self):
response = pretend.stub(
status_code=418,
text="I'm a teapot",
raise_for_status=pretend.raiser(requests.HTTPError),
)
session = pretend.stub(get=lambda *a, **k: response, )
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=session, metrics=pretend.stub())
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
verifier._retrieve_public_key_payload()
assert str(exc.value) == "Invalid response code 418: I'm a teapot"
assert exc.value.reason == "public_key_api.status.418"
def test_verify_cache_miss(self):
# Example taken from
# https://gist.github.com/ewjoachim/7dde11c31d9686ed6b4431c3ca166da2
meta_payload = {
"public_keys": [
{
"key_identifier": "90a421169f0a406205f1563a953312f0be898d3c"
"7b6c06b681aa86a874555f4a",
"key": "-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----",
"is_current": True,
}
]
}
response = pretend.stub(
json=lambda: meta_payload, raise_for_status=lambda: None
)
session = pretend.stub(get=lambda *a, **k: response)
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
api_token="api-token",
public_keys_cache=cache,
)
key_id = "90a421169f0a406205f1563a953312f0be898d3c7b6c06b681aa86a874555f4a"
signature = (
"MEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/"
"cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A=="
)
payload = (
b'[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299'
b'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/'
b'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]'
)
assert (
github_verifier.verify(payload=payload, key_id=key_id, signature=signature)
is True
)
assert metrics.increment.calls == [
pretend.call("warehouse.token_leak.github.auth.cache.miss"),
pretend.call("warehouse.token_leak.github.auth.success"),
]
def test_get_cached_public_key_cache_hit(self):
metrics = pretend.stub()
session = pretend.stub()
cache = integrations.PublicKeysCache(cache_time=12)
cache_value = pretend.stub()
cache.set(now=time.time(), value=cache_value)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
public_keys_cache=cache,
)
assert github_verifier._get_cached_public_keys() is cache_value
def test_extract_public_keys_error(self, payload, expected):
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=cache,
)
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
list(github_verifier.extract_public_keys(pubkey_api_data=payload))
assert exc.value.reason == "public_key_api.format_error"
assert str(exc.value) == expected
assert cache.cache is None
def test_retrieve_public_key_payload_json_error(self):
response = pretend.stub(
text="Still a non-json teapot",
json=pretend.raiser(json.JSONDecodeError("", "", 3)),
raise_for_status=lambda: None,
)
session = pretend.stub(get=lambda *a, **k: response)
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=session, metrics=pretend.stub())
with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc:
verifier._retrieve_public_key_payload()
assert str(
exc.value) == "Non-JSON response received: Still a non-json teapot"
assert exc.value.reason == "public_key_api.invalid_json"
def test_check_signature_invalid_crypto(self):
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=pretend.stub(), metrics=pretend.stub())
public_key = ""
signature = ""
payload = "yeah, nope, that won't pass"
with pytest.raises(utils.InvalidTokenLeakRequest) as exc:
verifier._check_signature(payload=payload,
public_key=public_key,
signature=signature)
assert str(exc.value) == "Invalid cryptographic values"
assert exc.value.reason == "invalid_crypto"
def test_check_public_key(self):
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=pretend.stub(), metrics=pretend.stub())
keys = [
{
"key_id": "a",
"key": "b"
},
{
"key_id": "c",
"key": "d"
},
]
assert verifier._check_public_key(github_public_keys=keys,
key_id="c") == "d"
def github_disclose_token(request):
# GitHub calls this API view when they have identified a string matching
# the regular expressions we provided them.
# Our job is to validate we're talking to github, check if the string contains
# valid credentials and, if they do, invalidate them and warn the owner
# The documentation for this process is at
# https://developer.github.com/partnerships/token-scanning/
body = request.body
# Thanks to the predicates, we know the headers we need are defined.
key_id = request.headers.get("GITHUB-PUBLIC-KEY-IDENTIFIER")
signature = request.headers.get("GITHUB-PUBLIC-KEY-SIGNATURE")
metrics = request.find_service(IMetricsService, context=None)
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=request.http,
metrics=metrics,
api_url=request.registry.
settings["github.token_scanning_meta_api.url"],
api_token=request.registry.settings.get("github.token"),
)
if not verifier.verify(payload=body, key_id=key_id, signature=signature):
return Response(status=400)
try:
disclosures = request.json_body
except json.decoder.JSONDecodeError:
metrics.increment(
"warehouse.token_leak.github.error.payload.json_error")
return Response(status=400)
try:
utils.analyze_disclosures(
request=request,
disclosure_records=disclosures,
origin="github",
metrics=metrics,
)
except utils.InvalidTokenLeakRequestError:
return Response(status=400)
# 204 No Content: we acknowledge but we won't comment on the outcome.
return Response(status=204)
def test_check_signature(self):
verifier = utils.GitHubTokenScanningPayloadVerifier(
session=pretend.stub(), metrics=pretend.stub())
public_key = (
"-----BEGIN PUBLIC KEY-----\n"
"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU"
"q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n"
"-----END PUBLIC KEY-----")
signature = ("MEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/"
"cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A==")
payload = (
'[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299'
'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/'
'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]')
assert (verifier._check_signature(payload=payload,
public_key=public_key,
signature=signature) is None)
def test_check_signature_invalid_crypto(self):
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=pretend.stub(),
public_keys_cache=pretend.stub(),
)
public_key = ""
signature = ""
payload = "yeah, nope, that won't pass"
with pytest.raises(integrations.InvalidPayloadSignatureError) as exc:
github_verifier._check_signature(
payload=payload, public_key=public_key, signature=signature
)
assert str(exc.value) == "Invalid cryptographic values"
assert exc.value.reason == "invalid_crypto"
def test_init(self):
metrics = pretend.stub()
session = pretend.stub()
token = "api_token"
url = "http://foo"
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=session,
metrics=metrics,
api_token=token,
public_keys_cache=cache,
)
assert github_verifier._session is session
assert github_verifier._metrics is metrics
assert github_verifier._api_token == token
assert github_verifier._api_url == url
assert github_verifier._public_keys_cache is cache
def test_verify_error(self):
metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None))
cache = integrations.PublicKeysCache(cache_time=12)
github_verifier = utils.GitHubTokenScanningPayloadVerifier(
api_url="http://foo",
session=pretend.stub(),
metrics=metrics,
api_token="api-token",
public_keys_cache=cache,
)
github_verifier.retrieve_public_key_payload = pretend.raiser(
integrations.InvalidPayloadSignatureError("Bla", "bla")
)
assert github_verifier.verify(payload={}, key_id="a", signature="a") is False
assert metrics.increment.calls == [
pretend.call("warehouse.token_leak.github.auth.cache.miss"),
pretend.call("warehouse.token_leak.github.auth.error.bla"),
]
def test_headers_auth_token(self):
headers = utils.GitHubTokenScanningPayloadVerifier(
session=pretend.stub(),
metrics=pretend.stub(),
api_token="api-token")._headers_auth()
assert headers == {"Authorization": "token api-token"}