def execute_sensor_iteration_loop(instance, workspace, logger, until=None):
"""
Helper function that performs sensor evaluations on a tighter loop, while reusing grpc locations
within a given daemon interval. Rather than relying on the daemon machinery to run the
iteration loop every 30 seconds, sensors are continuously evaluated, every 5 seconds. We rely on
each sensor definition's min_interval to check that sensor evaluations are spaced appropriately.
"""
from dagster.daemon.daemon import CompletedIteration
manager_loaded_time = pendulum.now("UTC").timestamp()
RELOAD_LOCATION_MANAGER_INTERVAL = 60
start_time = pendulum.now("UTC").timestamp()
while True:
start_time = pendulum.now("UTC").timestamp()
if until and start_time >= until:
# provide a way of organically ending the loop to support test environment
break
if start_time - manager_loaded_time > RELOAD_LOCATION_MANAGER_INTERVAL:
workspace.cleanup()
manager_loaded_time = pendulum.now("UTC").timestamp()
yield from execute_sensor_iteration(instance, logger, workspace)
loop_duration = pendulum.now("UTC").timestamp() - start_time
sleep_time = max(0, MIN_INTERVAL_LOOP_TIME - loop_duration)
yield CompletedIteration()
time.sleep(sleep_time)
def execute_sensor_iteration_loop(
instance, grpc_server_registry, logger, daemon_shutdown_event, until=None
):
"""
Helper function that performs sensor evaluations on a tighter loop, while reusing grpc locations
within a given daemon interval. Rather than relying on the daemon machinery to run the
iteration loop every 30 seconds, sensors are continuously evaluated, every 5 seconds. We rely on
each sensor definition's min_interval to check that sensor evaluations are spaced appropriately.
"""
from dagster.daemon.daemon import CompletedIteration
handle_manager = None
manager_loaded_time = None
RELOAD_LOCATION_MANAGER_INTERVAL = 60
start_time = pendulum.now("UTC").timestamp()
with ExitStack() as stack:
while not daemon_shutdown_event or not daemon_shutdown_event.is_set():
start_time = pendulum.now("UTC").timestamp()
if until and start_time >= until:
# provide a way of organically ending the loop to support test environment
break
if (
not handle_manager
or (start_time - manager_loaded_time) > RELOAD_LOCATION_MANAGER_INTERVAL
):
stack.close() # remove the previous context
handle_manager = stack.enter_context(
RepositoryLocationHandleManager(grpc_server_registry)
)
manager_loaded_time = start_time
yield from execute_sensor_iteration(instance, logger, handle_manager)
loop_duration = pendulum.now("UTC").timestamp() - start_time
sleep_time = max(0, MIN_INTERVAL_LOOP_TIME - loop_duration)
yield CompletedIteration()
time.sleep(sleep_time)