def run_command():
with capture_interrupts():
with DagsterInstance.get() as instance:
if instance.is_ephemeral:
raise Exception(
"dagster-daemon can't run using an in-memory instance. Make sure "
"the DAGSTER_HOME environment variable has been set correctly and that "
"you have created a dagster.yaml file there."
)
with daemon_controller_from_instance(instance) as controller:
start_time = pendulum.now("UTC")
while True:
# Wait until a daemon has been unhealthy for a long period of time
# before potentially restarting it due to a hanging or failed daemon
with raise_interrupts_as(KeyboardInterrupt):
time.sleep(1)
if (
pendulum.now("UTC") - start_time
).total_seconds() < 2 * DAEMON_HEARTBEAT_TOLERANCE_SECONDS:
continue
controller.check_daemons()
start_time = pendulum.now("UTC")
def check_daemon_loop(self):
start_time = pendulum.now("UTC")
while True:
# Wait until a daemon has been unhealthy for a long period of time
# before potentially restarting it due to a hanging or failed daemon
with raise_interrupts_as(KeyboardInterrupt):
time.sleep(5)
self.check_daemon_threads()
if (pendulum.now("UTC") - start_time
).total_seconds() < 2 * DAEMON_HEARTBEAT_TOLERANCE_SECONDS:
continue
self.check_daemon_heartbeats()
start_time = pendulum.now("UTC")
def check_daemon_loop(self):
start_time = time.time()
last_heartbeat_check_time = start_time
while True:
# Wait until a daemon has been unhealthy for a long period of time
# before potentially restarting it due to a hanging or failed daemon
with raise_interrupts_as(KeyboardInterrupt):
time.sleep(THREAD_CHECK_INTERVAL)
self.check_daemon_threads()
now = time.time()
# Give the daemon enough time to send an initial heartbeat before checking
if (
(now - start_time) < 2 * self._heartbeat_interval_seconds
or now - last_heartbeat_check_time < HEARTBEAT_CHECK_INTERVAL
):
continue
self.check_daemon_heartbeats()
last_heartbeat_check_time = time.time()
def raise_execution_interrupts():
with raise_interrupts_as(DagsterExecutionInterruptedError):
yield
