def test_healthy():
with instance_for_test(
overrides={
"run_coordinator": {
"module":
"dagster.core.run_coordinator.queued_run_coordinator",
"class": "QueuedRunCoordinator",
},
}) as instance:
init_time = pendulum.now("UTC")
beyond_tolerated_time = init_time.float_timestamp + 100
controller = DagsterDaemonController(instance)
assert not all_daemons_healthy(
instance, curr_time_seconds=init_time.float_timestamp)
assert not all_daemons_live(
instance, curr_time_seconds=init_time.float_timestamp)
controller.run_iteration(init_time)
assert all_daemons_healthy(instance,
curr_time_seconds=init_time.float_timestamp)
assert all_daemons_live(instance,
curr_time_seconds=init_time.float_timestamp)
assert not all_daemons_healthy(instance,
curr_time_seconds=beyond_tolerated_time)
assert not all_daemons_live(instance,
curr_time_seconds=beyond_tolerated_time)
def liveness_check_command():
with DagsterInstance.get() as instance:
if all_daemons_live(instance):
click.echo("Daemon live")
else:
click.echo("Daemon(s) not running")
sys.exit(1)
def liveness_check_command():
with DagsterInstance.get() as instance:
if all_daemons_live(instance, heartbeat_tolerance_seconds=_get_heartbeat_tolerance()):
click.echo("Daemon live")
else:
click.echo("Daemon(s) not running")
sys.exit(1)
def test_healthy():
with instance_for_test(
overrides={
"run_coordinator": {
"module":
"dagster.core.run_coordinator.queued_run_coordinator",
"class": "QueuedRunCoordinator",
},
}) as instance:
init_time = pendulum.now("UTC")
assert not all_daemons_healthy(
instance, curr_time_seconds=init_time.float_timestamp)
assert not all_daemons_live(
instance, curr_time_seconds=init_time.float_timestamp)
with daemon_controller_from_instance(
instance, wait_for_processes_on_exit=True) as controller:
while True:
now = pendulum.now("UTC")
if all_daemons_healthy(
instance, curr_time_seconds=now.float_timestamp
) and all_daemons_live(instance,
curr_time_seconds=now.float_timestamp):
controller.check_daemons()
beyond_tolerated_time = now.float_timestamp + 100
assert not all_daemons_healthy(
instance, curr_time_seconds=beyond_tolerated_time)
assert not all_daemons_live(
instance, curr_time_seconds=beyond_tolerated_time)
break
if (now - init_time).total_seconds() > 10:
raise Exception(
"timed out waiting for instance to become healthy")
time.sleep(0.5)
def test_error_daemon(monkeypatch):
with instance_for_test() as instance:
from dagster.daemon.daemon import SensorDaemon
def run_iteration_error(_, _instance, _daemon_shutdown_event,
_grpc_server_registry):
raise DagsterInvariantViolationError("foobar")
yield # pylint: disable=unreachable
monkeypatch.setattr(SensorDaemon, "run_iteration", run_iteration_error)
init_time = pendulum.now("UTC")
with daemon_controller_from_instance(
instance, wait_for_processes_on_exit=True) as controller:
while True:
now = pendulum.now("UTC")
if all_daemons_live(instance):
# Despite error, daemon should still be running
controller.check_daemons()
status = get_daemon_status(instance,
SensorDaemon.daemon_type(),
now.float_timestamp)
assert status.healthy == False
assert len(status.last_heartbeat.errors) == 1
assert (
status.last_heartbeat.errors[0].message.strip() ==
"dagster.core.errors.DagsterInvariantViolationError: foobar"
)
assert not all_daemons_healthy(
instance, curr_time_seconds=now.float_timestamp)
assert all_daemons_live(
instance, curr_time_seconds=now.float_timestamp)
break
if (now - init_time).total_seconds() > 10:
raise Exception("timed out waiting for heartbeat error")
time.sleep(0.5)
def test_multiple_error_daemon(monkeypatch):
with instance_for_test() as instance:
from dagster.daemon.daemon import SensorDaemon
def run_loop_error(_, _instance, _workspace):
# ?message stack cls_name cause"
yield SerializableErrorInfo("foobar", None, None, None)
yield SerializableErrorInfo("bizbuz", None, None, None)
while True:
yield
time.sleep(0.5)
monkeypatch.setattr(SensorDaemon, "core_loop", run_loop_error)
init_time = pendulum.now("UTC")
heartbeat_interval_seconds = 1
with daemon_controller_from_instance(
instance,
workspace_load_target=EmptyWorkspaceTarget(),
heartbeat_interval_seconds=heartbeat_interval_seconds,
) as controller:
while True:
now = pendulum.now("UTC")
if all_daemons_live(
instance,
heartbeat_interval_seconds=heartbeat_interval_seconds):
# Despite error, daemon should still be running
controller.check_daemon_threads()
controller.check_daemon_heartbeats()
status = get_daemon_statuses(
instance, [SensorDaemon.daemon_type()],
now.float_timestamp)[SensorDaemon.daemon_type()]
if status.healthy == False and len(
status.last_heartbeat.errors) == 2:
assert status.last_heartbeat.errors[0].message.strip(
) == "bizbuz"
assert status.last_heartbeat.errors[1].message.strip(
) == "foobar"
break
if (now - init_time).total_seconds() > 10:
raise Exception("timed out waiting for heartbeat error")
time.sleep(0.5)
def test_healthy_with_different_daemons():
with instance_for_test() as instance:
with DagsterDaemonController.create_from_instance(instance):
with instance_for_test(
overrides={
"run_coordinator": {
"module": "dagster.core.run_coordinator.queued_run_coordinator",
"class": "QueuedRunCoordinator",
},
}
) as other_instance:
now = pendulum.now("UTC")
assert not all_daemons_healthy(
other_instance, curr_time_seconds=now.float_timestamp
)
assert not all_daemons_live(other_instance, curr_time_seconds=now.float_timestamp)
def test_multiple_error_daemon(monkeypatch):
with instance_for_test() as instance:
from dagster.daemon.daemon import SensorDaemon
def run_iteration_error(_, _instance, _daemon_shutdown_event_,
_grpc_server_registry):
# ?message stack cls_name cause"
yield SerializableErrorInfo("foobar", None, None, None)
yield SerializableErrorInfo("bizbuz", None, None, None)
monkeypatch.setattr(SensorDaemon, "run_iteration", run_iteration_error)
init_time = pendulum.now("UTC")
with daemon_controller_from_instance(
instance, wait_for_processes_on_exit=True) as controller:
while True:
now = pendulum.now("UTC")
if all_daemons_live(instance):
# Despite error, daemon should still be running
controller.check_daemons()
status = get_daemon_status(instance,
SensorDaemon.daemon_type(),
now.float_timestamp)
if status.healthy == False:
assert len(status.last_heartbeat.errors) == 2
assert status.last_heartbeat.errors[0].message.strip(
) == "foobar"
assert status.last_heartbeat.errors[1].message.strip(
) == "bizbuz"
break
if (now - init_time).total_seconds() > 10:
raise Exception("timed out waiting for heartbeat error")
time.sleep(0.5)
def test_error_daemon(monkeypatch):
with instance_for_test(overrides={}) as instance:
from dagster.daemon.daemon import SensorDaemon
def run_iteration_error(_):
raise DagsterInvariantViolationError("foobar")
yield # pylint: disable=unreachable
monkeypatch.setattr(SensorDaemon, "run_iteration", run_iteration_error)
controller = DagsterDaemonController(instance)
init_time = pendulum.now("UTC")
controller.run_iteration(init_time)
status = get_daemon_status(instance, SensorDaemon.daemon_type(),
init_time.float_timestamp)
assert status.healthy == False
assert len(status.last_heartbeat.errors) == 1
assert (status.last_heartbeat.errors[0].message.strip() ==
"dagster.core.errors.DagsterInvariantViolationError: foobar")
assert not all_daemons_healthy(
instance, curr_time_seconds=init_time.float_timestamp)
assert all_daemons_live(instance,
curr_time_seconds=init_time.float_timestamp)
def test_warn_multiple_daemons(capsys):
from dagster.daemon.daemon import SensorDaemon
with instance_for_test() as instance:
init_time = pendulum.now("UTC")
heartbeat_interval_seconds = 1
with daemon_controller_from_instance(
instance,
heartbeat_interval_seconds=heartbeat_interval_seconds):
while True:
now = pendulum.now("UTC")
if all_daemons_live(
instance,
heartbeat_interval_seconds=heartbeat_interval_seconds):
captured = capsys.readouterr()
assert "Taking over from another SENSOR daemon process" not in captured.out
break
if (now - init_time).total_seconds() > 10:
raise Exception("timed out waiting for daemon status")
time.sleep(0.5)
capsys.readouterr()
init_time = pendulum.now("UTC")
status = get_daemon_status(
instance,
SensorDaemon.daemon_type(),
now.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)
last_heartbeat_time = status.last_heartbeat.timestamp
# No warning when a second controller starts up again
with daemon_controller_from_instance(
instance,
heartbeat_interval_seconds=heartbeat_interval_seconds):
while True:
now = pendulum.now("UTC")
status = get_daemon_status(
instance,
SensorDaemon.daemon_type(),
now.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)
if status.last_heartbeat and status.last_heartbeat.timestamp != last_heartbeat_time:
captured = capsys.readouterr()
assert "Taking over from another SENSOR daemon process" not in captured.out
break
if (now - init_time).total_seconds() > 10:
raise Exception("timed out waiting for new daemon status")
time.sleep(0.5)
status = get_daemon_status(
instance,
SensorDaemon.daemon_type(),
now.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)
last_heartbeat_time = status.last_heartbeat.timestamp
# Starting up a controller while one is running produces the warning though
with daemon_controller_from_instance(
instance,
heartbeat_interval_seconds=heartbeat_interval_seconds):
# Wait for heartbeats while two controllers are running at once and there will
# be a warning
init_time = pendulum.now("UTC")
while True:
now = pendulum.now("UTC")
captured = capsys.readouterr()
if "Taking over from another SENSOR daemon process" in captured.out:
break
if (now - init_time).total_seconds() > 60:
raise Exception("timed out waiting for heartbeats")
time.sleep(5)
def test_healthy():
with instance_for_test(
overrides={
"run_coordinator": {
"module":
"dagster.core.run_coordinator.queued_run_coordinator",
"class": "QueuedRunCoordinator",
},
}) as instance:
init_time = pendulum.now("UTC")
heartbeat_interval_seconds = 1
assert not all_daemons_healthy(
instance,
curr_time_seconds=init_time.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)
assert not all_daemons_live(
instance,
curr_time_seconds=init_time.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)
with daemon_controller_from_instance(
instance, heartbeat_interval_seconds=heartbeat_interval_seconds
) as controller:
while True:
now = pendulum.now("UTC")
if all_daemons_healthy(
instance,
curr_time_seconds=now.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
) and all_daemons_live(
instance,
curr_time_seconds=now.float_timestamp,
heartbeat_interval_seconds=heartbeat_interval_seconds,
):
controller.check_daemon_threads()
controller.check_daemon_heartbeats()
beyond_tolerated_time = (
now.float_timestamp +
DEFAULT_DAEMON_HEARTBEAT_TOLERANCE_SECONDS + 1)
assert not all_daemons_healthy(
instance,
curr_time_seconds=beyond_tolerated_time,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)
assert not all_daemons_live(
instance,
curr_time_seconds=beyond_tolerated_time,
heartbeat_interval_seconds=heartbeat_interval_seconds,
)
break
if (now - init_time).total_seconds() > 10:
raise Exception(
"timed out waiting for instance to become healthy")
time.sleep(0.5)
