def test_universal_autolog_makes_expected_event_logging_calls():
class TestLogger(AutologgingEventLogger):
LoggerCall = namedtuple("LoggerCall", ["integration", "call_args", "call_kwargs"])
def __init__(self):
self.calls = []
def reset(self):
self.calls = []
def log_autolog_called(self, integration, call_args, call_kwargs):
self.calls.append(TestLogger.LoggerCall(integration, call_args, call_kwargs))
logger = TestLogger()
AutologgingEventLogger.set_logger(logger)
mlflow.autolog(exclusive=True, disable=True)
universal_autolog_event_logging_calls = [
call for call in logger.calls if call.integration == "mlflow"
]
assert len(universal_autolog_event_logging_calls) == 1
call = universal_autolog_event_logging_calls[0]
assert call.integration == "mlflow"
assert {"disable": True, "exclusive": True}.items() <= call.call_kwargs.items()
def test_autologging_integration_makes_expected_event_logging_calls():
@autologging_integration("test_success")
def autolog_success(foo, bar=7, disable=False, silent=False):
pass
@autologging_integration("test_failure")
def autolog_failure(biz, baz="val", disable=False, silent=False):
raise Exception("autolog failed")
class TestLogger(AutologgingEventLogger):
LoggerCall = namedtuple("LoggerCall",
["integration", "call_args", "call_kwargs"])
def __init__(self):
self.calls = []
def reset(self):
self.calls = []
def log_autolog_called(self, integration, call_args, call_kwargs):
self.calls.append(
TestLogger.LoggerCall(integration, call_args, call_kwargs))
logger = TestLogger()
AutologgingEventLogger.set_logger(logger)
autolog_success("a", bar=9, disable=True)
assert len(logger.calls) == 1
call = logger.calls[0]
assert call.integration == "test_success"
# NB: In MLflow > 1.13.1, the `call_args` argument to `log_autolog_called` is deprecated.
# Positional arguments passed to `autolog()` should be forwarded to `log_autolog_called`
# in keyword format
assert call.call_args == ()
assert call.call_kwargs == {
"foo": "a",
"bar": 9,
"disable": True,
"silent": False
}
logger.reset()
with pytest.raises(Exception, match="autolog failed"):
autolog_failure(82, disable=False, silent=True)
assert len(logger.calls) == 1
call = logger.calls[0]
assert call.integration == "test_failure"
# NB: In MLflow > 1.13.1, the `call_args` argument to `log_autolog_called` is deprecated.
# Positional arguments passed to `autolog()` should be forwarded to `log_autolog_called`
# in keyword format
assert call.call_args == ()
assert call.call_kwargs == {
"biz": 82,
"baz": "val",
"disable": False,
"silent": True
}
def mock_event_logger():
try:
prev_logger = AutologgingEventLogger.get_logger()
logger = MockEventLogger()
AutologgingEventLogger.set_logger(logger)
yield logger
finally:
AutologgingEventLogger.set_logger(prev_logger)
def test_autologging_event_logger_default_impl_warns_for_log_autolog_called_with_deprecated_args():
AutologgingEventLogger.set_logger(AutologgingEventLogger())
with pytest.warns(DeprecationWarning, match="Received 1 positional arguments"):
AutologgingEventLogger.get_logger().log_autolog_called(
"test_integration",
# call_args is deprecated in MLflow > 1.13.1; specifying a non-empty
# value for this parameter should emit a warning
call_args=("a"),
call_kwargs={"b": "c"},
)
def test_autologging_integration_succeeds_when_event_logging_throws_in_standard_mode():
@autologging_integration("test")
def autolog(disable=False, silent=False):
return "result"
class ThrowingLogger(AutologgingEventLogger):
def __init__(self):
self.logged_event = False
def log_autolog_called(self, integration, call_args, call_kwargs):
self.logged_event = True
raise Exception("autolog failed")
logger = ThrowingLogger()
AutologgingEventLogger.set_logger(logger)
assert autolog() == "result"
assert logger.logged_event
def test_autologging_integration_makes_expected_event_logging_calls():
@autologging_integration("test_success")
def autolog_success(foo, bar=7, disable=False):
pass
@autologging_integration("test_failure")
def autolog_failure(biz, baz="val", disable=False):
raise Exception("autolog failed")
class TestLogger(AutologgingEventLogger):
LoggerCall = namedtuple("LoggerCall", ["integration", "call_args", "call_kwargs"])
def __init__(self):
self.calls = []
def reset(self):
self.calls = []
def log_autolog_called(self, integration, call_args, call_kwargs):
self.calls.append(TestLogger.LoggerCall(integration, call_args, call_kwargs))
logger = TestLogger()
AutologgingEventLogger.set_logger(logger)
autolog_success("a", bar=9, disable=True)
assert len(logger.calls) == 1
call = logger.calls[0]
assert call.integration == "test_success"
assert call.call_args == ("a",)
assert call.call_kwargs == {"bar": 9, "disable": True}
logger.reset()
with pytest.raises(Exception, match="autolog failed"):
autolog_failure(82, baz="b", disable=False)
assert len(logger.calls) == 1
call = logger.calls[0]
assert call.integration == "test_failure"
assert call.call_args == (82,)
assert call.call_kwargs == {"baz": "b", "disable": False}
def test_autologging_event_logger_default_implementation_does_not_throw_for_valid_inputs():
AutologgingEventLogger.set_logger(AutologgingEventLogger())
class PatchObj:
def test_fn(self):
pass
# Test successful autologging workflow
AutologgingEventLogger.get_logger().log_autolog_called(
"test_integration", ("a"), {"b": 1, "c": "d"}
)
AutologgingEventLogger.get_logger().log_patch_function_start(
AutologgingSession("test_integration", "123"), PatchObj(), "test_fn", (1000), {"a": 2}
)
AutologgingEventLogger.get_logger().log_original_function_start(
AutologgingSession("test_integration", "123"), PatchObj(), "test_fn", (1000), {"a": 2}
)
AutologgingEventLogger.get_logger().log_original_function_success(
AutologgingSession("test_integration", "123"), PatchObj(), "test_fn", (1000), {"a": 2}
)
AutologgingEventLogger.get_logger().log_patch_function_success(
AutologgingSession("test_integration", "123"), PatchObj(), "test_fn", (1000), {"a": 2}
)
# Test patch function failure autologging workflow
AutologgingEventLogger.get_logger().log_patch_function_start(
AutologgingSession("test_integration", "123"), PatchObj(), "test_fn", (1000), {"a": 2}
)
AutologgingEventLogger.get_logger().log_patch_function_error(
AutologgingSession("test_integration", "123"),
PatchObj(),
"test_fn",
(1000),
{"a": 2},
Exception("patch error"),
)
# Test original function failure autologging workflow
AutologgingEventLogger.get_logger().log_patch_function_start(
AutologgingSession("test_integration", "123"), PatchObj(), "test_fn", (1000), {"a": 2}
)
AutologgingEventLogger.get_logger().log_original_function_start(
AutologgingSession("test_integration", "123"), PatchObj(), "test_fn", (1000), {"a": 2}
)
AutologgingEventLogger.get_logger().log_patch_function_error(
AutologgingSession("test_integration", "123"),
PatchObj(),
"test_fn",
(1000),
{"a": 2},
Exception("patch error"),
)
def test_safe_patch_succeeds_when_event_logging_throws_in_standard_mode(
patch_destination, test_autologging_integration,
):
patch_preamble_called = False
patch_postamble_called = False
def patch_impl(original, *args, **kwargs):
nonlocal patch_preamble_called
patch_preamble_called = True
original(*args, **kwargs)
nonlocal patch_postamble_called
patch_postamble_called = True
safe_patch(test_autologging_integration, patch_destination, "fn", patch_impl)
class ThrowingLogger(MockEventLogger):
def log_patch_function_start(
self, session, patch_obj, function_name, call_args, call_kwargs
):
super().log_patch_function_start(
session, patch_obj, function_name, call_args, call_kwargs
)
raise Exception("failed")
def log_patch_function_success(
self, session, patch_obj, function_name, call_args, call_kwargs
):
super().log_patch_function_success(
session, patch_obj, function_name, call_args, call_kwargs
)
raise Exception("failed")
def log_patch_function_error(
self, session, patch_obj, function_name, call_args, call_kwargs, exception
):
super().log_patch_function_error(
session, patch_obj, function_name, call_args, call_kwargs, exception
)
raise Exception("failed")
def log_original_function_start(
self, session, patch_obj, function_name, call_args, call_kwargs
):
super().log_original_function_start(
session, patch_obj, function_name, call_args, call_kwargs
)
raise Exception("failed")
def log_original_function_success(
self, session, patch_obj, function_name, call_args, call_kwargs
):
super().log_original_function_success(
session, patch_obj, function_name, call_args, call_kwargs
)
raise Exception("failed")
def log_original_function_error(
self, session, patch_obj, function_name, call_args, call_kwargs, exception
):
super().log_original_function_error(
session, patch_obj, function_name, call_args, call_kwargs, exception
)
raise Exception("failed")
logger = ThrowingLogger()
AutologgingEventLogger.set_logger(logger)
assert patch_destination.fn() == PATCH_DESTINATION_FN_DEFAULT_RESULT
assert patch_preamble_called
assert patch_postamble_called
expected_calls = ["patch_start", "original_start", "original_success", "patch_success"]
assert [call.method for call in logger.calls] == expected_calls
