def create_daemon_of_type(daemon_type):
if daemon_type == SchedulerDaemon.daemon_type():
return SchedulerDaemon.create_from_instance(DagsterInstance.get())
elif daemon_type == SensorDaemon.daemon_type():
return SensorDaemon.create_from_instance(DagsterInstance.get())
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
return QueuedRunCoordinatorDaemon.create_from_instance(
DagsterInstance.get())
else:
raise Exception("Unexpected daemon type {daemon_type}".format(
daemon_type=daemon_type))
def test_error_sensor_daemon(external_repo_context, monkeypatch):
freeze_datetime = to_timezone(
create_pendulum_time(year=2019, month=2, day=28, tz="UTC"),
"US/Central")
sleeps = []
def fake_sleep(s):
sleeps.append(s)
pendulum.set_test_now(pendulum.now().add(seconds=s))
monkeypatch.setattr(time, "sleep", fake_sleep)
with instance_with_sensors(
external_repo_context,
overrides={
"run_launcher": {
"module": "dagster.core.test_utils",
"class": "ExplodingRunLauncher",
},
},
) as (instance, workspace, _external_repo):
@contextmanager
def _gen_workspace(_instance):
yield workspace
with pendulum.test(freeze_datetime):
instance.add_job_state(
JobState(_get_unloadable_sensor_origin(), JobType.SENSOR,
JobStatus.RUNNING))
sensor_daemon = SensorDaemon.create_from_instance(instance)
daemon_shutdown_event = threading.Event()
sensor_daemon.run_loop(
"my_uuid",
daemon_shutdown_event,
_gen_workspace,
heartbeat_interval_seconds=DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
error_interval_seconds=DEFAULT_DAEMON_ERROR_INTERVAL_SECONDS,
until=freeze_datetime.add(seconds=65),
)
heartbeats = instance.get_daemon_heartbeats()
heartbeat = heartbeats["SENSOR"]
assert heartbeat
assert heartbeat.errors
assert len(heartbeat.errors) == DAEMON_HEARTBEAT_ERROR_LIMIT
def create_daemons_from_instance(instance):
daemon_types = required_daemons(instance)
daemons = []
# Separate instance for each daemon since each is in its own thread
for daemon_type in daemon_types:
if daemon_type == SchedulerDaemon.daemon_type():
daemons.append(
SchedulerDaemon.create_from_instance(DagsterInstance.get()))
elif daemon_type == SensorDaemon.daemon_type():
daemons.append(
SensorDaemon.create_from_instance(DagsterInstance.get()))
elif daemon_type == QueuedRunCoordinatorDaemon.daemon_type():
daemons.append(
QueuedRunCoordinatorDaemon.create_from_instance(
DagsterInstance.get()))
else:
raise Exception("Unexpected daemon type {daemon_type}".format(
daemon_type=daemon_type))
return daemons