def test_is_testing_respects_environment_variable(): try: prev_env_var_value = os.environ.pop("MLFLOW_AUTOLOGGING_TESTING", None) assert not is_testing() os.environ["MLFLOW_AUTOLOGGING_TESTING"] = "false" assert not is_testing() os.environ["MLFLOW_AUTOLOGGING_TESTING"] = "true" assert is_testing() finally: if prev_env_var_value: os.environ["MLFLOW_AUTOLOGGING_TESTING"] = prev_env_var_value else: del os.environ["MLFLOW_AUTOLOGGING_TESTING"]
def test_safe_patch_validates_autologging_runs_when_necessary_in_test_mode( patch_destination, test_autologging_integration ): assert autologging_utils.is_testing() def no_tag_run_patch_impl(original, *args, **kwargs): with mlflow.start_run(nested=True): return original(*args, **kwargs) safe_patch(test_autologging_integration, patch_destination, "fn", no_tag_run_patch_impl) with mock.patch( "mlflow.utils.autologging_utils.safety._validate_autologging_run", wraps=_validate_autologging_run, ) as validate_run_mock: with pytest.raises( AssertionError, match="failed to set autologging tag with expected value" ): patch_destination.fn() assert validate_run_mock.call_count == 1 validate_run_mock.reset_mock() with mlflow.start_run(nested=True): # If a user-generated run existed prior to the autologged training session, we expect # that safe patch will not attempt to validate it patch_destination.fn() assert not validate_run_mock.called
def test_try_mlflow_log_propagates_exceptions_in_test_mode(): assert autologging_utils.is_testing() def throwing_function(): raise Exception("bad implementation") with pytest.raises(Exception, match="bad implementation"): try_mlflow_log(throwing_function)
def test_try_mlflow_log_emits_exceptions_as_warnings_in_standard_mode(): assert not autologging_utils.is_testing() def throwing_function(): raise Exception("bad implementation") with pytest.warns(UserWarning, match="bad implementation"): try_mlflow_log(throwing_function)
def test_mode_on(): try: prev_env_var_value = os.environ.pop(_AUTOLOGGING_TEST_MODE_ENV_VAR, None) os.environ[_AUTOLOGGING_TEST_MODE_ENV_VAR] = "true" assert is_testing() yield finally: if prev_env_var_value: os.environ[_AUTOLOGGING_TEST_MODE_ENV_VAR] = prev_env_var_value else: del os.environ[_AUTOLOGGING_TEST_MODE_ENV_VAR]
def test_safe_patch_does_not_throw_when_autologging_runs_are_leaked_in_standard_mode( patch_destination, test_autologging_integration ): assert not autologging_utils.is_testing() def leak_run_patch_impl(original, *args, **kwargs): mlflow.start_run(nested=True) safe_patch(test_autologging_integration, patch_destination, "fn", leak_run_patch_impl) patch_destination.fn() assert mlflow.active_run() # End the leaked run mlflow.end_run() assert not mlflow.active_run()
def test_exception_safe_class_exhibits_expected_behavior_in_test_mode(baseclass, metaclass): assert autologging_utils.is_testing() class NonThrowingClass(baseclass, metaclass=metaclass): def function(self): return 10 assert NonThrowingClass().function() == 10 exc_to_throw = Exception("function error") class ThrowingClass(baseclass, metaclass=metaclass): def function(self): raise exc_to_throw with pytest.raises(Exception) as exc: ThrowingClass().function() assert exc.value == exc_to_throw
def test_exception_safe_function_exhibits_expected_behavior_in_test_mode(): assert autologging_utils.is_testing() @exception_safe_function def non_throwing_function(): return 10 assert non_throwing_function() == 10 exc_to_throw = Exception("function error") @exception_safe_function def throwing_function(): raise exc_to_throw with pytest.raises(Exception) as exc: throwing_function() assert exc.value == exc_to_throw
def test_exception_safe_function_exhibits_expected_behavior_in_standard_mode(): assert not autologging_utils.is_testing() @exception_safe_function def non_throwing_function(): return 10 assert non_throwing_function() == 10 exc_to_throw = Exception("bad implementation") @exception_safe_function def throwing_function(): raise exc_to_throw with mock.patch("mlflow.utils.autologging_utils._logger.warning") as logger_mock: throwing_function() assert logger_mock.call_count == 1 message, formatting_arg = logger_mock.call_args[0] assert "unexpected error during autologging" in message assert formatting_arg == exc_to_throw
def test_exception_safe_class_exhibits_expected_behavior_in_standard_mode(baseclass, metaclass): assert not autologging_utils.is_testing() class NonThrowingClass(baseclass, metaclass=metaclass): def function(self): return 10 assert NonThrowingClass().function() == 10 exc_to_throw = Exception("function error") class ThrowingClass(baseclass, metaclass=metaclass): def function(self): raise exc_to_throw with mock.patch("mlflow.utils.autologging_utils._logger.warning") as logger_mock: ThrowingClass().function() assert logger_mock.call_count == 1 message, formatting_arg = logger_mock.call_args[0] assert "unexpected error during autologging" in message assert formatting_arg == exc_to_throw
def test_safe_patch_throws_when_autologging_runs_are_leaked_in_test_mode( patch_destination, test_autologging_integration ): assert autologging_utils.is_testing() def leak_run_patch_impl(original, *args, **kwargs): mlflow.start_run(nested=True) safe_patch(test_autologging_integration, patch_destination, "fn", leak_run_patch_impl) with pytest.raises(AssertionError, match="leaked an active run"): patch_destination.fn() # End the leaked run mlflow.end_run() with mlflow.start_run(): # If a user-generated run existed prior to the autologged training session, we expect # that safe patch will not throw a leaked run exception patch_destination.fn() # End the leaked nested run mlflow.end_run() assert not mlflow.active_run()
def test_safe_patch_does_not_validate_autologging_runs_in_standard_mode( patch_destination, test_autologging_integration ): assert not autologging_utils.is_testing() def no_tag_run_patch_impl(original, *args, **kwargs): with mlflow.start_run(nested=True): return original(*args, **kwargs) safe_patch(test_autologging_integration, patch_destination, "fn", no_tag_run_patch_impl) with mock.patch( "mlflow.utils.autologging_utils.safety._validate_autologging_run", wraps=_validate_autologging_run, ) as validate_run_mock: patch_destination.fn() with mlflow.start_run(nested=True): # If a user-generated run existed prior to the autologged training session, we expect # that safe patch will not attempt to validate it patch_destination.fn() assert not validate_run_mock.called