def test_extract_public_keys(self): meta_payload = { "public_keys": [{ "key_identifier": "90a421169f0a406205f1563a953312f0be898d3c" "7b6c06b681aa86a874555f4a", "key": "-----BEGIN PUBLIC KEY-----\n" "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU" "q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n" "-----END PUBLIC KEY-----", "is_current": True, }] } verifier = utils.GitHubTokenScanningPayloadVerifier( session=pretend.stub(), metrics=pretend.stub()) keys = list( verifier._extract_public_keys(pubkey_api_data=meta_payload)) assert keys == [{ "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcD" "QgAE9MJJHnMfn2+H4xL4YaPDA4RpJqUq\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ" "8qpVIW4clayyef9gWhFbNHWAA==\n-----END PUBLIC KEY-----", "key_id": "90a421169f0a406205f1563a953312f0be" "898d3c7b6c06b681aa86a874555f4a", }]
def test_check_signature_invalid_signature(self): github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=pretend.stub(), ) public_key = ( "-----BEGIN PUBLIC KEY-----\n" "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU" "q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n" "-----END PUBLIC KEY-----" ) # Changed the initial N for an M signature = ( "NEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/" "cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A==" ) payload = ( b'[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299' b'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/' b'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]' ) with pytest.raises(integrations.InvalidPayloadSignatureError) as exc: github_verifier._check_signature( payload=payload, public_key=public_key, signature=signature ) assert str(exc.value) == "Invalid signature" assert exc.value.reason == "invalid_signature"
def test_retrieve_public_key_payload(self): meta_payload = { "public_keys": [ { "key_identifier": "90a421169f0a406205f1563a953312f0be898d3c" "7b6c06b681aa86a874555f4a", "key": "-----BEGIN PUBLIC KEY-----\n" "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU" "q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n" "-----END PUBLIC KEY-----", "is_current": True, } ] } response = pretend.stub( json=lambda: meta_payload, raise_for_status=lambda: None ) session = pretend.stub(get=pretend.call_recorder(lambda *a, **k: response)) metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None)) github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=session, metrics=metrics, api_token="api-token", public_keys_cache=pretend.stub(), ) assert github_verifier.retrieve_public_key_payload() == meta_payload assert session.get.calls == [ pretend.call( "http://foo", headers={"Authorization": "token api-token"}, ) ]
def test_verify_cache_hit(self): session = pretend.stub() metrics = pretend.stub( increment=pretend.call_recorder(lambda str: None)) verifier = utils.GitHubTokenScanningPayloadVerifier( session=session, metrics=metrics, api_token="api-token") verifier.public_keys_cached_at = time.time() verifier.public_keys_cache = [{ "key_id": "90a421169f0a406205f1563a953312f0be898d3c" "7b6c06b681aa86a874555f4a", "key": "-----BEGIN PUBLIC KEY-----\n" "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU" "q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n" "-----END PUBLIC KEY-----", }] key_id = "90a421169f0a406205f1563a953312f0be898d3c7b6c06b681aa86a874555f4a" signature = ("MEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/" "cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A==") payload = ( '[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299' 'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/' 'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]') assert (verifier.verify( payload=payload, key_id=key_id, signature=signature) is True) assert metrics.increment.calls == [ pretend.call("warehouse.token_leak.github.auth.cache.hit"), pretend.call("warehouse.token_leak.github.auth.success"), ]
def test_extract_public_keys(self): meta_payload = { "public_keys": [ { "key_identifier": "90a421169f0a406205f1563a953312f0be898d3c" "7b6c06b681aa86a874555f4a", "key": "-----BEGIN PUBLIC KEY-----\n" "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU" "q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n" "-----END PUBLIC KEY-----", "is_current": True, } ] } cache = integrations.PublicKeysCache(cache_time=12) github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=cache, ) keys = github_verifier.extract_public_keys(pubkey_api_data=meta_payload) assert keys == [ { "key": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcD" "QgAE9MJJHnMfn2+H4xL4YaPDA4RpJqUq\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ" "8qpVIW4clayyef9gWhFbNHWAA==\n-----END PUBLIC KEY-----", "key_id": "90a421169f0a406205f1563a953312f0be" "898d3c7b6c06b681aa86a874555f4a", } ] assert cache.cache == keys
def test_get_cached_public_key_cache_miss_no_cache(self): metrics = pretend.stub() session = pretend.stub() verifier = utils.GitHubTokenScanningPayloadVerifier(session=session, metrics=metrics) with pytest.raises(utils.CacheMiss): verifier._get_cached_public_keys()
def test_extract_public_keys_error(self, payload, expected): verifier = utils.GitHubTokenScanningPayloadVerifier( session=pretend.stub(), metrics=pretend.stub()) with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc: list(verifier._extract_public_keys(pubkey_api_data=payload)) assert exc.value.reason == "public_key_api.format_error" assert str(exc.value) == expected
def test_headers_auth_no_token(self): headers = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=pretend.stub(), api_token=None, public_keys_cache=pretend.stub(), )._headers_auth() assert headers == {}
def test_headers_auth_token(self): headers = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=pretend.stub(), api_token="api-token", public_keys_cache=pretend.stub(), )._headers_auth() assert headers == {"Authorization": "token api-token"}
def test_check_public_key_error(self): verifier = utils.GitHubTokenScanningPayloadVerifier( session=pretend.stub(), metrics=pretend.stub()) with pytest.raises(utils.InvalidTokenLeakRequest) as exc: verifier._check_public_key(github_public_keys=[], key_id="c") assert str(exc.value) == "Key c not found in github public keys" assert exc.value.reason == "wrong_key_id"
def test_get_cached_public_key_cache_hit(self): metrics = pretend.stub() session = pretend.stub() verifier = utils.GitHubTokenScanningPayloadVerifier(session=session, metrics=metrics) verifier.public_keys_cached_at = time.time() cache = verifier.public_keys_cache = pretend.stub() assert verifier._get_cached_public_keys() is cache
def test_retrieve_public_key_payload_connection_error(self): session = pretend.stub(get=pretend.raiser(requests.ConnectionError)) verifier = utils.GitHubTokenScanningPayloadVerifier( session=session, metrics=pretend.stub()) with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc: verifier._retrieve_public_key_payload() assert str(exc.value) == "Could not connect to GitHub" assert exc.value.reason == "public_key_api.network_error"
def test_init(self): metrics = pretend.stub() session = pretend.stub() token = "api_token" verifier = utils.GitHubTokenScanningPayloadVerifier(session=session, metrics=metrics, api_token=token) assert verifier._session is session assert verifier._metrics is metrics assert verifier._api_token == token
def test_check_public_key(self): github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=pretend.stub(), ) keys = [ {"key_id": "a", "key": "b"}, {"key_id": "c", "key": "d"}, ] assert github_verifier._check_public_key(public_keys=keys, key_id="c") == "d"
def test_check_public_key_error(self): github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=pretend.stub(), ) with pytest.raises(integrations.InvalidPayloadSignatureError) as exc: github_verifier._check_public_key(public_keys=[], key_id="c") assert str(exc.value) == "Key c not found in public keys" assert exc.value.reason == "wrong_key_id"
def test_verify_error(self): metrics = pretend.stub( increment=pretend.call_recorder(lambda str: None)) verifier = utils.GitHubTokenScanningPayloadVerifier( session=pretend.stub(), metrics=metrics, api_token="api-token") verifier._retrieve_public_key_payload = pretend.raiser( utils.InvalidTokenLeakRequest("Bla", "bla")) assert verifier.verify(payload={}, key_id="a", signature="a") is False assert metrics.increment.calls == [ pretend.call("warehouse.token_leak.github.auth.cache.miss"), pretend.call("warehouse.token_leak.github.auth.error.bla"), ]
def test_get_cached_public_key_cache_miss_no_cache(self): metrics = pretend.stub() session = pretend.stub() cache = integrations.PublicKeysCache(cache_time=12) github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=session, metrics=metrics, public_keys_cache=cache, ) with pytest.raises(integrations.CacheMissError): github_verifier._get_cached_public_keys()
def test_retrieve_public_key_payload_http_error(self): response = pretend.stub( status_code=418, text="I'm a teapot", raise_for_status=pretend.raiser(requests.HTTPError), ) session = pretend.stub(get=lambda *a, **k: response, ) verifier = utils.GitHubTokenScanningPayloadVerifier( session=session, metrics=pretend.stub()) with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc: verifier._retrieve_public_key_payload() assert str(exc.value) == "Invalid response code 418: I'm a teapot" assert exc.value.reason == "public_key_api.status.418"
def test_verify_cache_miss(self): # Example taken from # https://gist.github.com/ewjoachim/7dde11c31d9686ed6b4431c3ca166da2 meta_payload = { "public_keys": [ { "key_identifier": "90a421169f0a406205f1563a953312f0be898d3c" "7b6c06b681aa86a874555f4a", "key": "-----BEGIN PUBLIC KEY-----\n" "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU" "q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n" "-----END PUBLIC KEY-----", "is_current": True, } ] } response = pretend.stub( json=lambda: meta_payload, raise_for_status=lambda: None ) session = pretend.stub(get=lambda *a, **k: response) metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None)) cache = integrations.PublicKeysCache(cache_time=12) github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=session, metrics=metrics, api_token="api-token", public_keys_cache=cache, ) key_id = "90a421169f0a406205f1563a953312f0be898d3c7b6c06b681aa86a874555f4a" signature = ( "MEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/" "cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A==" ) payload = ( b'[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299' b'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/' b'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]' ) assert ( github_verifier.verify(payload=payload, key_id=key_id, signature=signature) is True ) assert metrics.increment.calls == [ pretend.call("warehouse.token_leak.github.auth.cache.miss"), pretend.call("warehouse.token_leak.github.auth.success"), ]
def test_get_cached_public_key_cache_hit(self): metrics = pretend.stub() session = pretend.stub() cache = integrations.PublicKeysCache(cache_time=12) cache_value = pretend.stub() cache.set(now=time.time(), value=cache_value) github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=session, metrics=metrics, public_keys_cache=cache, ) assert github_verifier._get_cached_public_keys() is cache_value
def test_extract_public_keys_error(self, payload, expected): cache = integrations.PublicKeysCache(cache_time=12) github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=cache, ) with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc: list(github_verifier.extract_public_keys(pubkey_api_data=payload)) assert exc.value.reason == "public_key_api.format_error" assert str(exc.value) == expected assert cache.cache is None
def test_retrieve_public_key_payload_json_error(self): response = pretend.stub( text="Still a non-json teapot", json=pretend.raiser(json.JSONDecodeError("", "", 3)), raise_for_status=lambda: None, ) session = pretend.stub(get=lambda *a, **k: response) verifier = utils.GitHubTokenScanningPayloadVerifier( session=session, metrics=pretend.stub()) with pytest.raises(utils.GitHubPublicKeyMetaAPIError) as exc: verifier._retrieve_public_key_payload() assert str( exc.value) == "Non-JSON response received: Still a non-json teapot" assert exc.value.reason == "public_key_api.invalid_json"
def test_check_signature_invalid_crypto(self): verifier = utils.GitHubTokenScanningPayloadVerifier( session=pretend.stub(), metrics=pretend.stub()) public_key = "" signature = "" payload = "yeah, nope, that won't pass" with pytest.raises(utils.InvalidTokenLeakRequest) as exc: verifier._check_signature(payload=payload, public_key=public_key, signature=signature) assert str(exc.value) == "Invalid cryptographic values" assert exc.value.reason == "invalid_crypto"
def test_check_public_key(self): verifier = utils.GitHubTokenScanningPayloadVerifier( session=pretend.stub(), metrics=pretend.stub()) keys = [ { "key_id": "a", "key": "b" }, { "key_id": "c", "key": "d" }, ] assert verifier._check_public_key(github_public_keys=keys, key_id="c") == "d"
def github_disclose_token(request): # GitHub calls this API view when they have identified a string matching # the regular expressions we provided them. # Our job is to validate we're talking to github, check if the string contains # valid credentials and, if they do, invalidate them and warn the owner # The documentation for this process is at # https://developer.github.com/partnerships/token-scanning/ body = request.body # Thanks to the predicates, we know the headers we need are defined. key_id = request.headers.get("GITHUB-PUBLIC-KEY-IDENTIFIER") signature = request.headers.get("GITHUB-PUBLIC-KEY-SIGNATURE") metrics = request.find_service(IMetricsService, context=None) verifier = utils.GitHubTokenScanningPayloadVerifier( session=request.http, metrics=metrics, api_url=request.registry. settings["github.token_scanning_meta_api.url"], api_token=request.registry.settings.get("github.token"), ) if not verifier.verify(payload=body, key_id=key_id, signature=signature): return Response(status=400) try: disclosures = request.json_body except json.decoder.JSONDecodeError: metrics.increment( "warehouse.token_leak.github.error.payload.json_error") return Response(status=400) try: utils.analyze_disclosures( request=request, disclosure_records=disclosures, origin="github", metrics=metrics, ) except utils.InvalidTokenLeakRequestError: return Response(status=400) # 204 No Content: we acknowledge but we won't comment on the outcome. return Response(status=204)
def test_check_signature(self): verifier = utils.GitHubTokenScanningPayloadVerifier( session=pretend.stub(), metrics=pretend.stub()) public_key = ( "-----BEGIN PUBLIC KEY-----\n" "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9MJJHnMfn2+H4xL4YaPDA4RpJqU" "q\nkCmRCBnYERxZanmcpzQSXs1X/AljlKkbJ8qpVIW4clayyef9gWhFbNHWAA==\n" "-----END PUBLIC KEY-----") signature = ("MEQCIAfgjgz6Ou/3DXMYZBervz1TKCHFsvwMcbuJhNZse622AiAG86/" "cku2XdcmFWNHl2WSJi2fkE8t+auvB24eURaOd2A==") payload = ( '[{"type":"github_oauth_token","token":"cb4985f91f740272c0234202299' 'f43808034d7f5","url":" https://github.com/github/faketestrepo/blob/' 'b0dd59c0b500650cacd4551ca5989a6194001b10/production.env"}]') assert (verifier._check_signature(payload=payload, public_key=public_key, signature=signature) is None)
def test_check_signature_invalid_crypto(self): github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=pretend.stub(), public_keys_cache=pretend.stub(), ) public_key = "" signature = "" payload = "yeah, nope, that won't pass" with pytest.raises(integrations.InvalidPayloadSignatureError) as exc: github_verifier._check_signature( payload=payload, public_key=public_key, signature=signature ) assert str(exc.value) == "Invalid cryptographic values" assert exc.value.reason == "invalid_crypto"
def test_init(self): metrics = pretend.stub() session = pretend.stub() token = "api_token" url = "http://foo" cache = integrations.PublicKeysCache(cache_time=12) github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=session, metrics=metrics, api_token=token, public_keys_cache=cache, ) assert github_verifier._session is session assert github_verifier._metrics is metrics assert github_verifier._api_token == token assert github_verifier._api_url == url assert github_verifier._public_keys_cache is cache
def test_verify_error(self): metrics = pretend.stub(increment=pretend.call_recorder(lambda str: None)) cache = integrations.PublicKeysCache(cache_time=12) github_verifier = utils.GitHubTokenScanningPayloadVerifier( api_url="http://foo", session=pretend.stub(), metrics=metrics, api_token="api-token", public_keys_cache=cache, ) github_verifier.retrieve_public_key_payload = pretend.raiser( integrations.InvalidPayloadSignatureError("Bla", "bla") ) assert github_verifier.verify(payload={}, key_id="a", signature="a") is False assert metrics.increment.calls == [ pretend.call("warehouse.token_leak.github.auth.cache.miss"), pretend.call("warehouse.token_leak.github.auth.error.bla"), ]
def test_headers_auth_token(self): headers = utils.GitHubTokenScanningPayloadVerifier( session=pretend.stub(), metrics=pretend.stub(), api_token="api-token")._headers_auth() assert headers == {"Authorization": "token api-token"}