def test_e2e_gives_experimental_warning( self, moodbot_domain_path: Path, e2e_bot_config_file: Path, e2e_stories_path: Text, nlu_data_path: Text, caplog: LogCaptureFixture, tmp_path: Path, ): with caplog.at_level(logging.WARNING): rasa.train( str(moodbot_domain_path), str(e2e_bot_config_file), [e2e_stories_path, nlu_data_path], output=str(tmp_path), dry_run=True, ) assert any([ "The end-to-end training is currently experimental" in record.message for record in caplog.records ])
def test_not_store_backward_in_database( data_loader: DataLoader, caplog: LogCaptureFixture, monkeypatch: MonkeyPatch, tmp_path: Path, ) -> None: internal_database = Database(DatabaseType.internal) dsx_write_database = Database(DatabaseType.dsx_write) data_output = DataOutput(RUNTIME_CONFIG, internal_database, dsx_write_database) model_config_account_1 = ModelConfigAccount1(runtime_config=RUNTIME_CONFIG, data_loader=data_loader) account_data = data_loader.load_account_data(model_config_account_1, -1) model_run = ForecastModelRun() model_config_account_1.forecast_path = tmp_path with caplog.at_level(logging.DEBUG): returned_model_run = data_output.store_forecast( model_config=model_config_account_1, model_run=model_run, account_data=account_data, forecast_raw=TEST_ACCOUNT_1_RAW_DATA, forecast_post=TEST_ACCOUNT_1_POST_DATA, actuals_newest_month=datetime(2019, 10, 1, 0, 0), ) assert RUNTIME_CONFIG.engine_run_type == EngineRunType.backward assert returned_model_run is model_run assert "Skip storing forecast in internal database for backward run." in caplog.messages
def test_signed( caplog: LogCaptureFixture, clirunner, docker_registry_secure: DockerRegistrySecure, gpgsigner: GPGSigner, known_good_image: TypingKnownGoodImage, runner, ): """Test docker-verify can handle signed images.""" caplog.clear() caplog.set_level(logging.DEBUG) source = known_good_image["image_name"] destination = source.clone() destination.digest = None destination.tag += "_signed" with ca_trust_store(docker_registry_secure.cacerts), registry_credentials( docker_registry_secure): result = clirunner.invoke( cli_signer, args=[ "registry", "--keyid", gpgsigner.keyid, str(source), str(destination), ], env={"DSV_GPG_DATASTORE": str(gpgsigner.homedir)}, input=f"{gpgsigner.passphrase}\n", ) assert not result.exception assert "Integrity check passed." in caplog.text assert "Created new image" in caplog.text assert str(destination) in caplog.text caplog.clear() with gpg_datastore(gpgsigner.homedir): result = runner.invoke(cli, args=["registry", str(destination)]) assert not result.exception assert "Integrity check passed." in caplog.text assert "Signature check passed." in caplog.text assert " is consistent; 1 signature(s) verified." in caplog.text assert str(destination) in caplog.text
def test_retry_and_fail( self, database_type: DatabaseType, monkeypatch: MonkeyPatch, caplog: LogCaptureFixture ) -> None: monkeypatch.setattr(master_config, "db_connection_attempts", 3) monkeypatch.setattr(master_config, "db_connection_retry_sleep_seconds", 0.01) monkeypatch.setattr(pyodbc, "connect", Mock(side_effect=pyodbc.OperationalError("Always fails!"))) with pytest.raises(DatabaseConnectionFailure): with caplog.at_level(logging.DEBUG): database = Database(database_type) database.get_existing_table_names() # Run something that explicitly connects to the database assert caplog.messages == [ f"Trying to connect to {database_type.name} database, attempt #1", f"Unsuccessful attempt #1 to connect to {database_type.name} database: Always fails!", "Waiting 0.01 seconds before next connection attempt", f"Trying to connect to {database_type.name} database, attempt #2", f"Unsuccessful attempt #2 to connect to {database_type.name} database: Always fails!", "Waiting 0.01 seconds before next connection attempt", f"Trying to connect to {database_type.name} database, attempt #3", f"Unsuccessful attempt #3 to connect to {database_type.name} database: Always fails!", f"Unable to connect to {database_type.name} database after 3 attempt(s).", ]
def test_for_signature(caplog: LogCaptureFixture): """Tests subclass instantiation.""" caplog.set_level(logging.FATAL, logger="gnupg") result = Signer.for_signature("PGP SIGNATURE") assert result assert isinstance(result, GPGSigner)
def caplog_info_level( caplog: LogCaptureFixture) -> Iterable[LogCaptureFixture]: with caplog.at_level(logging.INFO, ): yield caplog
async def test_callback_error( setup: SetupTest, caplog: LogCaptureFixture ) -> None: """Test an error return from the OIDC token endpoint.""" setup.configure("oidc") assert setup.config.oidc return_url = "https://example.com/foo" r = await setup.client.get( "/login", params={"rd": return_url}, allow_redirects=False ) assert r.status_code == 307 url = urlparse(r.headers["Location"]) query = parse_qs(url.query) # Build an error response to return from the OIDC token URL and register # it as a result. response = { "error": "error_code", "error_description": "description", } setup.httpx_mock.add_response( url=setup.config.oidc.token_url, method="POST", json=response, status_code=400, ) # Simulate the return from the OpenID Connect provider. caplog.clear() r = await setup.client.get( "/oauth2/callback", params={"code": "some-code", "state": query["state"][0]}, allow_redirects=False, ) assert r.status_code == 500 assert "error_code: description" in r.text data = json.loads(caplog.record_tuples[-1][2]) assert data == { "error": "error_code: description", "event": "Provider authentication failed", "level": "warning", "logger": "gafaelfawr", "method": "GET", "path": "/oauth2/callback", "return_url": return_url, "remote": "127.0.0.1", "request_id": ANY, "user_agent": ANY, } # Change the mock error response to not contain an error. We should then # internally raise the exception for the return status, which should # translate into an internal server error. setup.httpx_mock.add_response( url=setup.config.oidc.token_url, method="POST", json={"foo": "bar"}, status_code=400, ) r = await setup.client.get( "/login", params={"rd": return_url}, allow_redirects=False ) query = parse_qs(urlparse(r.headers["Location"]).query) r = await setup.client.get( "/oauth2/callback", params={"code": "some-code", "state": query["state"][0]}, allow_redirects=False, ) assert r.status_code == 500 assert "Cannot contact authentication provider" in r.text # Now try a reply that returns 200 but doesn't have the field we # need. setup.httpx_mock.add_response( url=setup.config.oidc.token_url, method="POST", json={"foo": "bar"}, status_code=200, ) r = await setup.client.get( "/login", params={"rd": return_url}, allow_redirects=False ) query = parse_qs(urlparse(r.headers["Location"]).query) r = await setup.client.get( "/oauth2/callback", params={"code": "some-code", "state": query["state"][0]}, allow_redirects=False, ) assert r.status_code == 500 assert "No id_token in token reply" in r.text # Return invalid JSON, which should raise an error during JSON decoding. setup.httpx_mock.add_response( url=setup.config.oidc.token_url, method="POST", data="foo", status_code=200, ) r = await setup.client.get( "/login", params={"rd": return_url}, allow_redirects=False ) query = parse_qs(urlparse(r.headers["Location"]).query) r = await setup.client.get( "/oauth2/callback", params={"code": "some-code", "state": query["state"][0]}, allow_redirects=False, ) assert r.status_code == 500 assert "not valid JSON" in r.text # Finally, return invalid JSON and an error reply. setup.httpx_mock.add_response( url=setup.config.oidc.token_url, method="POST", data="foo", status_code=400, ) r = await setup.client.get( "/login", params={"rd": return_url}, allow_redirects=False ) query = parse_qs(urlparse(r.headers["Location"]).query) r = await setup.client.get( "/oauth2/callback", params={"code": "some-code", "state": query["state"][0]}, allow_redirects=False, ) assert r.status_code == 500 assert "Cannot contact authentication provider" in r.text
def test_subprocess_trace(datadog_tracer: ddtrace.Tracer, caplog: LogCaptureFixture): """Verify that spans created in subprocesses are written to the queue and then flushed to the server, when wrapped in the SubprocessTracer""" # Enable log output for this logger for duration of this test caplog.set_level(logging.DEBUG, DatadogLoggingTraceFilter._log.name) test = f"{inspect.stack()[0][3]}" # And also send its output through a multiprocessing queue to surface logs from the subprocess log_queue = mp.Queue() DatadogLoggingTraceFilter._log.addHandler(QueueHandler(log_queue)) DatadogLoggingTraceFilter.activate() subproc_test_msg = f"a test message was logged in a subprocess of {test}" state = mp.Queue() stop_sentinel = "-->STOP<--" with ddtrace.tracer.trace( name=f"{test}_operation", service=f"{test}_service", resource=f"{test}_resource", span_type=SpanTypes.TEST, ) as span: trace_id = span.trace_id logger = logging.getLogger(f"{test}_logger") test_msg = f"a test message was logged during {test}" logger.warning(test_msg) ctx = mp.get_context("fork") worker = ctx.Process( name=f"{test}_subproc", target=_do_things_in_subproc, args=( subproc_test_msg, state, ), ) worker.start() worker.join(timeout=10) if worker.is_alive(): worker.terminate() try: _drain_captured_log_queue(log_queue, stop_sentinel, caplog, force_immediate_stop=True) except Exception: print( "Error draining captured log queue when handling subproc TimeoutError" ) pass raise mp.TimeoutError( f"subprocess {worker.name} did not complete in timeout") DatadogLoggingTraceFilter._log.warning(stop_sentinel) subproc_trace_id, subproc_span_id = state.get(block=True, timeout=10) assert test_msg in caplog.text, "caplog.text did not seem to capture logging output during test" assert f"SPAN#{trace_id}" in caplog.text, "span marker not found in logging output" assert f"TRACE#{trace_id}" in caplog.text, "trace marker not found in logging output" assert f"resource {test}_resource" in caplog.text, "traced resource not found in logging output" assert subproc_trace_id == trace_id # subprocess tracing should be a continuation of the trace in parent process _drain_captured_log_queue(log_queue, stop_sentinel, caplog) assert f"{subproc_span_id}" in caplog.text, "subproc span id not found in logging output" assert ( f"resource {_do_things_in_subproc.__name__}_resource" in caplog.text), "subproc traced resource not found in logging output"
async def test_callback_error( tmp_path: Path, client: AsyncClient, respx_mock: respx.Router, caplog: LogCaptureFixture, ) -> None: """Test an error return from the OIDC token endpoint.""" config = await configure(tmp_path, "oidc") assert config.oidc return_url = "https://example.com/foo" r = await client.get("/login", params={"rd": return_url}) assert r.status_code == 307 url = urlparse(r.headers["Location"]) query = parse_qs(url.query) # Build an error response to return from the OIDC token URL and register # it as a result. response = { "error": "error_code", "error_description": "description", } respx_mock.post(config.oidc.token_url).respond(400, json=response) # Simulate the return from the OpenID Connect provider. caplog.clear() r = await client.get( "/oauth2/callback", params={ "code": "some-code", "state": query["state"][0] }, ) assert r.status_code == 403 assert "error_code: description" in r.text assert parse_log(caplog) == [ { "event": f"Retrieving ID token from {config.oidc.token_url}", "httpRequest": { "requestMethod": "GET", "requestUrl": ANY, "remoteIp": "127.0.0.1", }, "return_url": return_url, "severity": "info", }, { "error": "error_code: description", "event": "Authentication provider failed", "httpRequest": { "requestMethod": "GET", "requestUrl": ANY, "remoteIp": "127.0.0.1", }, "return_url": return_url, "severity": "warning", }, ] # Change the mock error response to not contain an error. We should then # internally raise the exception for the return status, which should # translate into an internal server error. respx_mock.post(config.oidc.token_url).respond(400, json={"foo": "bar"}) r = await client.get("/login", params={"rd": return_url}) query = parse_qs(urlparse(r.headers["Location"]).query) r = await client.get( "/oauth2/callback", params={ "code": "some-code", "state": query["state"][0] }, ) assert r.status_code == 403 assert "Cannot contact authentication provider" in r.text # Now try a reply that returns 200 but doesn't have the field we # need. respx_mock.post(config.oidc.token_url).respond(json={"foo": "bar"}) r = await client.get("/login", params={"rd": return_url}) query = parse_qs(urlparse(r.headers["Location"]).query) r = await client.get( "/oauth2/callback", params={ "code": "some-code", "state": query["state"][0] }, ) assert r.status_code == 403 assert "No id_token in token reply" in r.text # Return invalid JSON, which should raise an error during JSON decoding. respx_mock.post(config.oidc.token_url).respond(content=b"foo") r = await client.get("/login", params={"rd": return_url}) query = parse_qs(urlparse(r.headers["Location"]).query) r = await client.get( "/oauth2/callback", params={ "code": "some-code", "state": query["state"][0] }, ) assert r.status_code == 403 assert "not valid JSON" in r.text # Finally, return invalid JSON and an error reply. respx_mock.post(config.oidc.token_url).respond(400, content=b"foo") r = await client.get("/login", params={"rd": return_url}) query = parse_qs(urlparse(r.headers["Location"]).query) r = await client.get( "/oauth2/callback", params={ "code": "some-code", "state": query["state"][0] }, ) assert r.status_code == 403 assert "Cannot contact authentication provider" in r.text
def set_log_level_debug(caplog: LogCaptureFixture) -> None: # Set the post-test log level to DEBUG for failing tests. For all tests # (failing and successful), the live log level can be additionally set in # `setup.cfg`. It should be set to WARNING. caplog.set_level(logging.DEBUG)
async def test_invalid(setup: SetupTest, caplog: LogCaptureFixture) -> None: token_data = await setup.create_session_token() issuer = setup.factory.create_token_issuer() oidc_token = issuer.issue_token(token_data, jti="some-jti") caplog.clear() r = await setup.client.get( "/auth/userinfo", headers={"Authorization": f"token {oidc_token.encoded}"}, ) assert r.status_code == 400 authenticate = parse_www_authenticate(r.headers["WWW-Authenticate"]) assert isinstance(authenticate, AuthErrorChallenge) assert authenticate.auth_type == AuthType.Bearer assert authenticate.realm == setup.config.realm assert authenticate.error == AuthError.invalid_request assert authenticate.error_description == "Unknown Authorization type token" log = json.loads(caplog.record_tuples[0][2]) assert log == { "error": "Unknown Authorization type token", "event": "Invalid request", "level": "warning", "logger": "gafaelfawr", "method": "GET", "path": "/auth/userinfo", "remote": "127.0.0.1", "request_id": ANY, "user_agent": ANY, } r = await setup.client.get( "/auth/userinfo", headers={"Authorization": f"bearer{oidc_token.encoded}"}, ) assert r.status_code == 400 authenticate = parse_www_authenticate(r.headers["WWW-Authenticate"]) assert isinstance(authenticate, AuthErrorChallenge) assert authenticate.auth_type == AuthType.Bearer assert authenticate.realm == setup.config.realm assert authenticate.error == AuthError.invalid_request assert authenticate.error_description == "Malformed Authorization header" caplog.clear() r = await setup.client.get( "/auth/userinfo", headers={"Authorization": f"bearer XXX{oidc_token.encoded}"}, ) assert r.status_code == 401 authenticate = parse_www_authenticate(r.headers["WWW-Authenticate"]) assert isinstance(authenticate, AuthErrorChallenge) assert authenticate.auth_type == AuthType.Bearer assert authenticate.realm == setup.config.realm assert authenticate.error == AuthError.invalid_token assert authenticate.error_description log = json.loads(caplog.record_tuples[0][2]) assert log == { "error": ANY, "event": "Invalid token", "level": "warning", "logger": "gafaelfawr", "method": "GET", "path": "/auth/userinfo", "remote": "127.0.0.1", "request_id": ANY, "token_source": "bearer", "user_agent": ANY, }
async def test_token_errors( setup: SetupTest, caplog: LogCaptureFixture ) -> None: clients = [ OIDCClient(client_id="some-id", client_secret="some-secret"), OIDCClient(client_id="other-id", client_secret="other-secret"), ] setup.configure(oidc_clients=clients) token_data = await setup.create_session_token() token = token_data.token oidc_service = setup.factory.create_oidc_service() redirect_uri = f"https://{TEST_HOSTNAME}/app" code = await oidc_service.issue_code("some-id", redirect_uri, token) # Missing parameters. request: Dict[str, str] = {} caplog.clear() r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_request", "error_description": "Invalid token request", } log = json.loads(caplog.record_tuples[0][2]) assert log == { "error": "Invalid token request", "event": "Invalid request", "level": "warning", "logger": "gafaelfawr", "method": "POST", "path": "/auth/openid/token", "remote": "127.0.0.1", "request_id": ANY, "user_agent": ANY, } # Invalid grant type. request = { "grant_type": "bogus", "client_id": "other-client", "code": "nonsense", "redirect_uri": f"https://{TEST_HOSTNAME}/", } caplog.clear() r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "unsupported_grant_type", "error_description": "Invalid grant type bogus", } log = json.loads(caplog.record_tuples[0][2]) assert log == { "error": "Invalid grant type bogus", "event": "Unsupported grant type", "level": "warning", "logger": "gafaelfawr", "method": "POST", "path": "/auth/openid/token", "remote": "127.0.0.1", "request_id": ANY, "user_agent": ANY, } # Invalid code. request["grant_type"] = "authorization_code" r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_grant", "error_description": "Invalid authorization code", } # No client_secret. request["code"] = str(OIDCAuthorizationCode()) caplog.clear() r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_client", "error_description": "No client_secret provided", } log = json.loads(caplog.record_tuples[0][2]) assert log == { "error": "No client_secret provided", "event": "Unauthorized client", "level": "warning", "logger": "gafaelfawr", "method": "POST", "path": "/auth/openid/token", "remote": "127.0.0.1", "request_id": ANY, "user_agent": ANY, } # Incorrect client_id. request["client_secret"] = "other-secret" r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_client", "error_description": "Unknown client ID other-client", } # Incorrect client_secret. request["client_id"] = "some-id" r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_client", "error_description": "Invalid secret for some-id", } # No stored data. request["client_secret"] = "some-secret" bogus_code = OIDCAuthorizationCode() request["code"] = str(bogus_code) caplog.clear() r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_grant", "error_description": "Invalid authorization code", } log = json.loads(caplog.record_tuples[0][2]) assert log["event"] == "Invalid authorization code" assert log["error"] == f"Unknown authorization code {bogus_code.key}" # Corrupt stored data. await setup.redis.set(bogus_code.key, "XXXXXXX") r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_grant", "error_description": "Invalid authorization code", } # Correct code, but invalid client_id for that code. bogus_code = await oidc_service.issue_code("other-id", redirect_uri, token) request["code"] = str(bogus_code) r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_grant", "error_description": "Invalid authorization code", } # Correct code and client_id but invalid redirect_uri. request["code"] = str(code) r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_grant", "error_description": "Invalid authorization code", } # Delete the underlying token. token_service = setup.factory.create_token_service() await token_service.delete_token( token.key, token_data, token_data.username, ip_address="127.0.0.1" ) request["redirect_uri"] = redirect_uri r = await setup.client.post("/auth/openid/token", data=request) assert r.status_code == 400 assert r.json() == { "error": "invalid_grant", "error_description": "Invalid authorization code", }
async def test_login(setup: SetupTest, caplog: LogCaptureFixture) -> None: clients = [OIDCClient(client_id="some-id", client_secret="some-secret")] setup.configure(oidc_clients=clients) token_data = await setup.create_session_token() await setup.login(token_data.token) return_url = f"https://{TEST_HOSTNAME}:4444/foo?a=bar&b=baz" # Log in caplog.clear() r = await setup.client.get( "/auth/openid/login", params={ "response_type": "code", "scope": "openid", "client_id": "some-id", "state": "random-state", "redirect_uri": return_url, }, allow_redirects=False, ) assert r.status_code == 307 url = urlparse(r.headers["Location"]) assert url.scheme == "https" assert url.netloc == f"{TEST_HOSTNAME}:4444" assert url.path == "/foo" assert url.query query = parse_qs(url.query) assert query == { "a": ["bar"], "b": ["baz"], "code": [ANY], "state": ["random-state"], } code = query["code"][0] log = json.loads(caplog.record_tuples[0][2]) assert log == { "event": "Returned OpenID Connect authorization code", "level": "info", "logger": "gafaelfawr", "method": "GET", "path": "/auth/openid/login", "remote": "127.0.0.1", "request_id": ANY, "return_url": return_url, "scope": "user:token", "token": token_data.token.key, "token_source": "cookie", "user": token_data.username, "user_agent": ANY, } # Redeem the code for a token and check the result. caplog.clear() r = await setup.client.post( "/auth/openid/token", data={ "grant_type": "authorization_code", "client_id": "some-id", "client_secret": "some-secret", "code": code, "redirect_uri": return_url, }, ) assert r.status_code == 200 assert r.headers["Cache-Control"] == "no-store" assert r.headers["Pragma"] == "no-cache" data = r.json() assert data == { "access_token": ANY, "token_type": "Bearer", "expires_in": ANY, "id_token": ANY, } assert isinstance(data["expires_in"], int) exp_seconds = setup.config.issuer.exp_minutes * 60 assert exp_seconds - 5 <= data["expires_in"] <= exp_seconds assert data["access_token"] == data["id_token"] verifier = setup.factory.create_token_verifier() token = verifier.verify_internal_token(OIDCToken(encoded=data["id_token"])) assert token.claims == { "aud": setup.config.issuer.aud, "exp": ANY, "iat": ANY, "iss": setup.config.issuer.iss, "jti": OIDCAuthorizationCode.from_str(code).key, "name": token_data.name, "preferred_username": token_data.username, "scope": "openid", "sub": token_data.username, setup.config.issuer.username_claim: token_data.username, setup.config.issuer.uid_claim: token_data.uid, } now = time.time() expected_exp = now + setup.config.issuer.exp_minutes * 60 assert expected_exp - 5 <= token.claims["exp"] <= expected_exp assert now - 5 <= token.claims["iat"] <= now log = json.loads(caplog.record_tuples[0][2]) username = token_data.username assert log == { "event": f"Retrieved token for user {username} via OpenID Connect", "level": "info", "logger": "gafaelfawr", "method": "POST", "path": "/auth/openid/token", "remote": "127.0.0.1", "request_id": ANY, "token": OIDCAuthorizationCode.from_str(code).key, "user": username, "user_agent": ANY, }
async def test_login_errors( setup: SetupTest, caplog: LogCaptureFixture ) -> None: clients = [OIDCClient(client_id="some-id", client_secret="some-secret")] setup.configure(oidc_clients=clients) token_data = await setup.create_session_token() await setup.login(token_data.token) # No parameters at all. caplog.clear() r = await setup.client.get("/auth/openid/login", allow_redirects=False) assert r.status_code == 422 # Good client ID but missing redirect_uri. login_params = {"client_id": "some-id"} caplog.clear() r = await setup.client.get( "/auth/openid/login", params=login_params, allow_redirects=False ) assert r.status_code == 422 # Bad client ID. caplog.clear() login_params = { "client_id": "bad-client", "redirect_uri": f"https://{TEST_HOSTNAME}/", } r = await setup.client.get( "/auth/openid/login", params=login_params, allow_redirects=False ) assert r.status_code == 400 assert "Unknown client_id bad-client" in r.text log = json.loads(caplog.record_tuples[0][2]) assert log == { "error": "Unknown client_id bad-client in OpenID Connect request", "event": "Invalid request", "level": "warning", "logger": "gafaelfawr", "method": "GET", "path": "/auth/openid/login", "remote": "127.0.0.1", "request_id": ANY, "return_url": f"https://{TEST_HOSTNAME}/", "scope": "user:token", "token": ANY, "token_source": "cookie", "user": token_data.username, "user_agent": ANY, } # Bad redirect_uri. login_params["client_id"] = "some-id" login_params["redirect_uri"] = "https://foo.example.com/" r = await setup.client.get( "/auth/openid/login", params=login_params, allow_redirects=False ) assert r.status_code == 422 assert "URL is not at" in r.text # Valid redirect_uri but missing response_type. login_params["redirect_uri"] = f"https://{TEST_HOSTNAME}/app" caplog.clear() r = await setup.client.get( "/auth/openid/login", params=login_params, allow_redirects=False ) assert r.status_code == 307 url = urlparse(r.headers["Location"]) assert url.scheme == "https" assert url.netloc == TEST_HOSTNAME assert url.path == "/app" assert url.query query = parse_qs(url.query) assert query == { "error": ["invalid_request"], "error_description": ["Missing response_type parameter"], } log = json.loads(caplog.record_tuples[0][2]) assert log == { "error": "Missing response_type parameter", "event": "Invalid request", "level": "warning", "logger": "gafaelfawr", "method": "GET", "path": "/auth/openid/login", "remote": "127.0.0.1", "request_id": ANY, "return_url": login_params["redirect_uri"], "scope": "user:token", "token": ANY, "token_source": "cookie", "user": token_data.username, "user_agent": ANY, } # Invalid response_type. login_params["response_type"] = "bogus" r = await setup.client.get( "/auth/openid/login", params=login_params, allow_redirects=False ) assert r.status_code == 307 assert query_from_url(r.headers["Location"]) == { "error": ["invalid_request"], "error_description": ["code is the only supported response_type"], } # Valid response_type but missing scope. login_params["response_type"] = "code" r = await setup.client.get( "/auth/openid/login", params=login_params, allow_redirects=False ) assert r.status_code == 307 assert query_from_url(r.headers["Location"]) == { "error": ["invalid_request"], "error_description": ["Missing scope parameter"], } # Invalid scope. login_params["scope"] = "user:email" r = await setup.client.get( "/auth/openid/login", params=login_params, allow_redirects=False ) assert r.status_code == 307 assert query_from_url(r.headers["Location"]) == { "error": ["invalid_request"], "error_description": ["openid is the only supported scope"], }
async def test_create( factory: ComponentFactory, mock_kubernetes: MockKubernetesApi, caplog: LogCaptureFixture, ) -> None: await create_test_service_tokens(mock_kubernetes) kubernetes_service = factory.create_kubernetes_service(MagicMock()) await kubernetes_service.update_service_tokens() await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes) service_token = await mock_kubernetes.get_namespaced_custom_object( "gafaelfawr.lsst.io", "v1alpha1", "mobu", "gafaelfawrservicetokens", "gafaelfawr-secret", ) assert service_token["status"]["conditions"] == [ { "lastTransitionTime": ANY, "message": "Secret was created", "observedGeneration": 1, "reason": StatusReason.Created.value, "status": "True", "type": "SecretCreated", } ] service_token = await mock_kubernetes.get_namespaced_custom_object( "gafaelfawr.lsst.io", "v1alpha1", "nublado2", "gafaelfawrservicetokens", "gafaelfawr", ) assert service_token["status"]["conditions"] == [ { "lastTransitionTime": ANY, "message": "Secret was created", "observedGeneration": 45, "reason": StatusReason.Created.value, "status": "True", "type": "SecretCreated", } ] assert parse_log(caplog) == [ { "event": "Created new service token", "key": ANY, "severity": "info", "token_scope": "admin:token", "token_username": "******", }, { "event": "Created mobu/gafaelfawr-secret secret", "scopes": ["admin:token"], "severity": "info", "service": "mobu", }, { "event": "Created new service token", "key": ANY, "severity": "info", "token_scope": "", "token_username": "******", }, { "event": "Created nublado2/gafaelfawr secret", "scopes": [], "severity": "info", "service": "nublado-hub", }, ] # Running creation again should not change anything. caplog.clear() objects = mock_kubernetes.get_all_objects_for_test("Secret") await kubernetes_service.update_service_tokens() assert mock_kubernetes.get_all_objects_for_test("Secret") == objects assert caplog.record_tuples == []
def test_validate_with_none_if_default_is_valid(caplog: LogCaptureFixture, tmp_path: pathlib.Path): with caplog.at_level(logging.WARNING, rasa.cli.utils.logger.name): assert get_validated_path(None, "out", str(tmp_path)) == str(tmp_path) assert caplog.records == []
async def test_create_delete_modify( client: AsyncClient, factory: ComponentFactory, caplog: LogCaptureFixture ) -> None: user_info = TokenUserInfo( username="******", name="Example Person", email="*****@*****.**", uid=45613, groups=[TokenGroup(name="foo", id=12313)], ) token_service = factory.create_token_service() async with factory.session.begin(): session_token = await token_service.create_session_token( user_info, scopes=["read:all", "exec:admin", "user:token"], ip_address="127.0.0.1", ) csrf = await set_session_cookie(client, session_token) expires = current_datetime() + timedelta(days=100) r = await client.post( "/auth/api/v1/users/example/tokens", headers={"X-CSRF-Token": csrf}, json={ "token_name": "some token", "scopes": ["read:all"], "expires": int(expires.timestamp()), }, ) assert r.status_code == 201 assert r.json() == {"token": ANY} user_token = Token.from_str(r.json()["token"]) token_url = r.headers["Location"] assert token_url == f"/auth/api/v1/users/example/tokens/{user_token.key}" r = await client.get(token_url) assert r.status_code == 200 info = r.json() assert info == { "token": user_token.key, "username": "******", "token_name": "some token", "token_type": "user", "scopes": ["read:all"], "created": ANY, "expires": int(expires.timestamp()), } # Check that this is the same information as is returned by the token-info # route. This is a bit tricky to do since the cookie will take precedence # over the Authorization header, but we can't just delete the cookie since # we'll lose the CSRF token. Save the cookie and delete it, and then # later restore it. cookie = client.cookies.pop(COOKIE_NAME) r = await client.get( "/auth/api/v1/token-info", headers={"Authorization": f"bearer {user_token}"}, ) assert r.status_code == 200 assert r.json() == info client.cookies.set(COOKIE_NAME, cookie, domain=TEST_HOSTNAME) # Listing all tokens for this user should return the user token and a # session token. r = await client.get("/auth/api/v1/users/example/tokens") assert r.status_code == 200 data = r.json() # Adjust for sorting, which will be by creation date and then token. assert len(data) == 2 if data[0] == info: session_info = data[1] else: assert data[1] == info session_info = data[0] assert session_info == { "token": session_token.key, "username": "******", "token_type": "session", "scopes": ["exec:admin", "read:all", "user:token"], "created": ANY, "expires": ANY, } # Change the name, scope, and expiration of the token. caplog.clear() new_expires = current_datetime() + timedelta(days=200) r = await client.patch( token_url, headers={"X-CSRF-Token": csrf}, json={ "token_name": "happy token", "scopes": ["exec:admin"], "expires": int(new_expires.timestamp()), }, ) assert r.status_code == 201 assert r.json() == { "token": user_token.key, "username": "******", "token_name": "happy token", "token_type": "user", "scopes": ["exec:admin"], "created": ANY, "expires": int(new_expires.timestamp()), } # Check the logging. Regression test for a bug where new expirations # would be logged as raw datetime objects instead of timestamps. assert parse_log(caplog) == [ { "expires": int(new_expires.timestamp()), "event": "Modified token", "httpRequest": { "requestMethod": "PATCH", "requestUrl": f"https://{TEST_HOSTNAME}{token_url}", "remoteIp": "127.0.0.1", }, "key": user_token.key, "scope": "exec:admin read:all user:token", "severity": "info", "token": session_token.key, "token_name": "happy token", "token_scope": "exec:admin", "token_source": "cookie", "user": "******", } ] # Delete the token. r = await client.delete(token_url, headers={"X-CSRF-Token": csrf}) assert r.status_code == 204 r = await client.get(token_url) assert r.status_code == 404 # Deleting again should return 404. r = await client.delete(token_url, headers={"X-CSRF-Token": csrf}) assert r.status_code == 404 # This user should now have only one token. r = await client.get("/auth/api/v1/users/example/tokens") assert r.status_code == 200 assert len(r.json()) == 1 # We should be able to see the change history for the token. r = await client.get(token_url + "/change-history") assert r.status_code == 200 assert r.json() == [ { "token": user_token.key, "username": "******", "token_type": "user", "token_name": "happy token", "scopes": ["exec:admin"], "expires": int(new_expires.timestamp()), "actor": "example", "action": "revoke", "ip_address": "127.0.0.1", "event_time": ANY, }, { "token": user_token.key, "username": "******", "token_type": "user", "token_name": "happy token", "scopes": ["exec:admin"], "expires": int(new_expires.timestamp()), "actor": "example", "action": "edit", "old_token_name": "some token", "old_scopes": ["read:all"], "old_expires": int(expires.timestamp()), "ip_address": "127.0.0.1", "event_time": ANY, }, { "token": user_token.key, "username": "******", "token_type": "user", "token_name": "some token", "scopes": ["read:all"], "expires": int(expires.timestamp()), "actor": "example", "action": "create", "ip_address": "127.0.0.1", "event_time": ANY, }, ]
def test_load_exogenous_feature_with_database_disabled( feature_name: str, tmp_path: Path, monkeypatch: MonkeyPatch, caplog: LogCaptureFixture) -> None: caplog.set_level(logging.INFO) test_data = pd.DataFrame( [ [ "EF", "Project_1", 0, 0, "Contract_1", "EF", "Airframe_1", None, "Build Rate", "2015-01-01", 25.60 ], [ "EF", "Project_1", 0, 0, "Contract_1", "EF", "Airframe_1", None, "Build Rate 202001", "2015-01-01", 25.60 ], [ "EF", "Project_1", 0, 0, "Contract_1", "EF", "Airframe_1", None, "Build Rate", "2016-01-01", 25.60 ], [ "EF", "Project_1", 0, 0, "Contract_1", "EF", "Airframe_1", None, "Build Rate 202001", "2016-01-01", 25.60 ], ], columns=[ "binid", "projectid", "ship to", "branch", "contractid", "custmiscinfo", "airframe", "program", "periodicdatastream", "perioddate", "value", ], ) data_loader = DataLoader(internal_database=Mock(spec=Database, is_disabled=lambda: True), dsx_read_database=Mock(spec=Database)) dsx_exogenous_data_path = "01 Raw data/Test_DSX_exogenous_features.csv" absolute_dsx_exogenous_data_path = tmp_path / dsx_exogenous_data_path absolute_dsx_exogenous_data_path.parent.mkdir(parents=True) pd.DataFrame(test_data).to_csv(absolute_dsx_exogenous_data_path, index=False) monkeypatch.setattr(master_config, "dsx_exogenous_data_path", dsx_exogenous_data_path) monkeypatch.setattr(master_config, "default_data_loader_location", tmp_path) exogenous_data = data_loader.load_exogenous_feature( feature_name=feature_name, run_id=-1) assert_frame_equal(generate_exogenous_feature_data(feature_name), exogenous_data) assert ( f'Loaded exogenous feature for Periodic_Data_Stream "{feature_name}" ' f'with 2 lines from "{dsx_exogenous_data_path}"' in caplog.messages)
async def test_queue_monitor(producer: InternalProducer, caplog: LogCaptureFixture, fake_coroutine): """Ensure the queue monitor logs as we expect Note that something we implicitly test for here is that the monitor does not log lots of duplicate lines. Rather it only logs when something changes. """ producer.size_warning = 3 producer.monitor_interval = 0.01 caplog.set_level(logging.WARNING) # Start the producer running producer.start() # No logging yet assert not caplog.records # Add a couple of items to the queue (still under size_warning) producer.queue.put_nowait(None) producer.queue.put_nowait(None) await asyncio.sleep(0.05) # Still no logging yet assert not caplog.records # One more gets us up to the warning level producer.queue.put_nowait(None) await asyncio.sleep(0.05) # Now we have logging assert len(caplog.records) == 1 assert caplog.records[0].getMessage( ) == "Queue in InternalProducer now has 3 commands." caplog.clear() # Clear the log messages # Let's check we get another messages when the queue gets bigger again producer.queue.put_nowait(None) await asyncio.sleep(0.05) assert len(caplog.records) == 1 assert caplog.records[0].getMessage( ) == "Queue in InternalProducer now has 4 commands." caplog.clear() # Clear the log messages # Now check we get logging when the queue shrinks, but is still above the warning level producer.queue.get_nowait() await asyncio.sleep(0.05) assert len(caplog.records) == 1 assert caplog.records[0].getMessage() == ( "Queue in InternalProducer has shrunk back down to 3 commands.") caplog.clear() # Clear the log messages # Now check we get logging when the queue shrinks to BELOW the warning level producer.queue.get_nowait() await asyncio.sleep(0.05) assert len(caplog.records) == 1 assert caplog.records[0].getMessage() == ( "Queue in InternalProducer has shrunk back down to 2 commands. " "Queue is now at an OK size again.") caplog.clear() # Clear the log messages
def test_load_cleaning_input_data(caplog: LogCaptureFixture, monkeypatch: MonkeyPatch) -> None: caplog.set_level(logging.WARNING) def mock_load_csv(*args: Any, **kwargs: Any) -> pd.DataFrame: return pd.DataFrame( [ # this one will be kept after cleaning [ "Account_18", "Project_0", "Contract_0", 1, pd.Timestamp("2020-02-01"), 1.0, 1.0, "mn0" ], # this one will be removed due to adhoc contract in dsx data workaround in TODO FSC-371 [ "Account_18", "Project_0", "Contract_730", 1, pd.Timestamp("2020-02-01"), 1.0, 1.0, "mn0" ], # this one will be removed due to wrong days in dsx data workaround in TODO FSC-318 [ "Account_18", "Project_0", "Contract_0", 1, pd.Timestamp("2020-02-28"), 1.0, 1.0, "mn0" ], ], columns=[ "lowlevelcust", "projectid", "contractid", "shortid", "perioddate", "Cost", "Adjusted History", "masterpart", ], ) monkeypatch.setattr(DataLoader, "load_csv", mock_load_csv) internal_database = Database(DatabaseType.internal) dsx_read_database = Database(DatabaseType.dsx_read) dsx_read_database.is_disabled = lambda: True # type: ignore data_loader = DataLoader(internal_database, dsx_read_database) assert data_loader.load_cleaning_input_data().equals( pd.DataFrame( [ # this one will be kept after cleaning [ "Account_18", "Project_0", "Contract_0", 1, pd.Timestamp("2020-02-01"), 1.0, 1.0, "mn0" ] ], columns=[ "lowlevelcust", "projectid", "contractid", "shortid", "perioddate", "Cost", "Adjusted History", "masterpart", ], )) assert "Removed 1 rows with invalid dates." in caplog.messages assert "Removed 1 unexpected rows of adhoc contract." in caplog.messages
async def test_login( tmp_path: Path, client: AsyncClient, respx_mock: respx.Router, caplog: LogCaptureFixture, ) -> None: config = await configure(tmp_path, "oidc") token = await create_upstream_oidc_token(groups=["admin"], name="Some Person", email="*****@*****.**") await mock_oidc_provider_config(respx_mock, config.issuer.keypair) await mock_oidc_provider_token(respx_mock, "some-code", token) assert config.oidc return_url = "https://example.com:4444/foo?a=bar&b=baz" caplog.clear() r = await client.get("/login", params={"rd": return_url}) assert r.status_code == 307 assert r.headers["Location"].startswith(config.oidc.login_url) url = urlparse(r.headers["Location"]) assert url.query query = parse_qs(url.query) login_params = {p: [v] for p, v in config.oidc.login_params.items()} assert query == { "client_id": [config.oidc.client_id], "redirect_uri": [config.oidc.redirect_url], "response_type": ["code"], "scope": ["openid " + " ".join(config.oidc.scopes)], "state": [ANY], **login_params, } # Verify the logging. login_url = config.oidc.login_url assert parse_log(caplog) == [{ "event": f"Redirecting user to {login_url} for authentication", "httpRequest": { "requestMethod": "GET", "requestUrl": ANY, "remoteIp": "127.0.0.1", }, "return_url": return_url, "severity": "info", }] # Simulate the return from the provider. caplog.clear() r = await client.get("/login", params={ "code": "some-code", "state": query["state"][0] }) assert r.status_code == 307 assert r.headers["Location"] == return_url # Verify the logging. expected_scopes_set = set(config.issuer.group_mapping["admin"]) expected_scopes_set.add("user:token") expected_scopes = " ".join(sorted(expected_scopes_set)) event = f"Successfully authenticated user {token.username} ({token.uid})" assert parse_log(caplog) == [ { "event": f"Retrieving ID token from {config.oidc.token_url}", "httpRequest": { "requestMethod": "GET", "requestUrl": ANY, "remoteIp": "127.0.0.1", }, "return_url": return_url, "severity": "info", }, { "event": event, "httpRequest": { "requestMethod": "GET", "requestUrl": ANY, "remoteIp": "127.0.0.1", }, "return_url": return_url, "scope": expected_scopes, "severity": "info", "token": ANY, "user": token.username, }, ] # Check that the /auth route works and finds our token. r = await client.get("/auth", params={"scope": "exec:admin"}) assert r.status_code == 200 assert r.headers["X-Auth-Request-Token-Scopes"] == expected_scopes assert r.headers["X-Auth-Request-Scopes-Accepted"] == "exec:admin" assert r.headers["X-Auth-Request-Scopes-Satisfy"] == "all" assert r.headers["X-Auth-Request-User"] == token.username assert r.headers["X-Auth-Request-Email"] == "*****@*****.**" assert r.headers["X-Auth-Request-Uid"] == str(token.uid) assert r.headers["X-Auth-Request-Groups"] == "admin"
def test_jieba_load_and_persist_dictionary( tmp_path_factory: TempPathFactory, default_model_storage: ModelStorage, default_execution_context: ExecutionContext, caplog: LogCaptureFixture, ): dictionary_directory = tmp_path_factory.mktemp("dictionaries") dictionary_path = dictionary_directory / "dictionary_1" dictionary_contents = """ 创新办 3 i 云计算 5 凱特琳 nz 台中 """ dictionary_path.write_text(dictionary_contents, encoding="utf-8") component_config = {"dictionary_path": dictionary_directory} resource = Resource("jieba") tk = JiebaTokenizerGraphComponent.create( { **JiebaTokenizerGraphComponent.get_default_config(), **component_config }, default_model_storage, resource, default_execution_context, ) tk.process_training_data(TrainingData([Message(data={TEXT: ""})])) # The dictionary has not been persisted yet. with caplog.at_level(logging.WARN): JiebaTokenizerGraphComponent.load( { **JiebaTokenizerGraphComponent.get_default_config(), **component_config }, default_model_storage, resource, default_execution_context, ) assert any( "Failed to load JiebaTokenizerGraphComponent from model storage." in message for message in caplog.messages) tk.persist() # Check the persisted dictionary matches the original file. with default_model_storage.read_from(resource) as resource_dir: contents = (resource_dir / "dictionary_1").read_text(encoding="utf-8") assert contents == dictionary_contents # Delete original files to show that we read from the model storage. dictionary_path.unlink() dictionary_directory.rmdir() JiebaTokenizerGraphComponent.load( { **JiebaTokenizerGraphComponent.get_default_config(), **component_config }, default_model_storage, resource, default_execution_context, ) tk.process([Message(data={TEXT: ""})])