def run_command(): with capture_interrupts(): with DagsterInstance.get() as instance: if instance.is_ephemeral: raise Exception( "dagster-daemon can't run using an in-memory instance. Make sure " "the DAGSTER_HOME environment variable has been set correctly and that " "you have created a dagster.yaml file there." ) with daemon_controller_from_instance(instance) as controller: start_time = pendulum.now("UTC") while True: # Wait until a daemon has been unhealthy for a long period of time # before potentially restarting it due to a hanging or failed daemon with raise_interrupts_as(KeyboardInterrupt): time.sleep(1) if ( pendulum.now("UTC") - start_time ).total_seconds() < 2 * DAEMON_HEARTBEAT_TOLERANCE_SECONDS: continue controller.check_daemons() start_time = pendulum.now("UTC")
def check_daemon_loop(self): start_time = pendulum.now("UTC") while True: # Wait until a daemon has been unhealthy for a long period of time # before potentially restarting it due to a hanging or failed daemon with raise_interrupts_as(KeyboardInterrupt): time.sleep(5) self.check_daemon_threads() if (pendulum.now("UTC") - start_time ).total_seconds() < 2 * DAEMON_HEARTBEAT_TOLERANCE_SECONDS: continue self.check_daemon_heartbeats() start_time = pendulum.now("UTC")
def check_daemon_loop(self): start_time = time.time() last_heartbeat_check_time = start_time while True: # Wait until a daemon has been unhealthy for a long period of time # before potentially restarting it due to a hanging or failed daemon with raise_interrupts_as(KeyboardInterrupt): time.sleep(THREAD_CHECK_INTERVAL) self.check_daemon_threads() now = time.time() # Give the daemon enough time to send an initial heartbeat before checking if ( (now - start_time) < 2 * self._heartbeat_interval_seconds or now - last_heartbeat_check_time < HEARTBEAT_CHECK_INTERVAL ): continue self.check_daemon_heartbeats() last_heartbeat_check_time = time.time()
def raise_execution_interrupts(): with raise_interrupts_as(DagsterExecutionInterruptedError): yield