def test_launch_failure(external_repo_context, capfd):
freeze_datetime = pendulum.datetime(
year=2019, month=2, day=27, hour=23, minute=59, second=59,
).in_tz("US/Central")
with instance_with_sensors(
external_repo_context,
overrides={
"run_launcher": {"module": "dagster.core.test_utils", "class": "ExplodingRunLauncher",},
},
) as (instance, external_repo):
with pendulum.test(freeze_datetime):
external_sensor = external_repo.get_external_sensor("always_on_sensor")
instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING)
)
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 0
execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))
assert instance.get_runs_count() == 1
run = instance.get_runs()[0]
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 1
validate_tick(
ticks[0], external_sensor, freeze_datetime, JobTickStatus.SUCCESS, run.run_id
)
captured = capfd.readouterr()
assert (
"Run {run_id} created successfully but failed to launch.".format(run_id=run.run_id)
) in captured.out
def test_error_sensor(external_repo_context, capfd):
freeze_datetime = pendulum.datetime(
year=2019, month=2, day=27, hour=23, minute=59, second=59,
).in_tz("US/Central")
with instance_with_sensors(external_repo_context) as (instance, external_repo):
with pendulum.test(freeze_datetime):
external_sensor = external_repo.get_external_sensor("error_sensor")
instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING)
)
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 0
execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 1
validate_tick(
ticks[0],
external_sensor,
freeze_datetime,
JobTickStatus.FAILURE,
None,
"Error occurred during the execution of evaluation_fn for sensor error_sensor",
)
captured = capfd.readouterr()
assert ("Failed to resolve sensor for error_sensor : ") in captured.out
assert (
"Error occurred during the execution of evaluation_fn for sensor error_sensor"
) in captured.out
def test_custom_interval_sensor(external_repo_context):
freeze_datetime = pendulum.datetime(year=2019, month=2, day=28).in_tz("US/Central")
with instance_with_sensors(external_repo_context) as (instance, external_repo):
with pendulum.test(freeze_datetime):
external_sensor = external_repo.get_external_sensor("custom_interval_sensor")
instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING)
)
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 0
list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 1
validate_tick(ticks[0], external_sensor, freeze_datetime, JobTickStatus.SKIPPED)
freeze_datetime = freeze_datetime.add(seconds=30)
with pendulum.test(freeze_datetime):
list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
# no additional tick created after 30 seconds
assert len(ticks) == 1
freeze_datetime = freeze_datetime.add(seconds=30)
with pendulum.test(freeze_datetime):
list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 2
expected_datetime = pendulum.datetime(year=2019, month=2, day=28, hour=0, minute=1)
validate_tick(ticks[0], external_sensor, expected_datetime, JobTickStatus.SKIPPED)
def test_simple_sensor(external_repo_context, capfd):
freeze_datetime = pendulum.datetime(
year=2019, month=2, day=27, hour=23, minute=59, second=59,
).in_tz("US/Central")
with instance_with_sensors(external_repo_context) as (instance, external_repo):
with pendulum.test(freeze_datetime):
external_sensor = external_repo.get_external_sensor("simple_sensor")
instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING)
)
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 0
list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 1
validate_tick(
ticks[0], external_sensor, freeze_datetime, JobTickStatus.SKIPPED,
)
captured = capfd.readouterr()
assert (
captured.out
== """2019-02-27 17:59:59 - SensorDaemon - INFO - Checking for new runs for the following sensors: simple_sensor
2019-02-27 17:59:59 - SensorDaemon - INFO - Sensor returned false for simple_sensor, skipping
"""
)
freeze_datetime = freeze_datetime.add(seconds=1)
with pendulum.test(freeze_datetime):
list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
wait_for_all_runs_to_start(instance)
assert instance.get_runs_count() == 1
run = instance.get_runs()[0]
validate_run_started(run)
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 2
expected_datetime = pendulum.datetime(year=2019, month=2, day=28)
validate_tick(
ticks[0], external_sensor, expected_datetime, JobTickStatus.SUCCESS, [run.run_id],
)
captured = capfd.readouterr()
assert (
captured.out
== """2019-02-27 18:00:00 - SensorDaemon - INFO - Checking for new runs for the following sensors: simple_sensor
2019-02-27 18:00:00 - SensorDaemon - INFO - Launching run for simple_sensor
2019-02-27 18:00:00 - SensorDaemon - INFO - Completed launch of run {run_id} for simple_sensor
""".format(
run_id=run.run_id
)
)
def test_launch_once(external_repo_context, capfd):
freeze_datetime = pendulum.datetime(
year=2019,
month=2,
day=27,
hour=23,
minute=59,
second=59,
).in_tz("US/Central")
with instance_with_sensors(external_repo_context) as (instance,
external_repo):
with pendulum.test(freeze_datetime):
external_sensor = external_repo.get_external_sensor(
"run_key_sensor")
instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR,
JobStatus.RUNNING))
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(
external_sensor.get_external_origin_id())
assert len(ticks) == 0
execute_sensor_iteration(instance,
get_default_daemon_logger("SensorDaemon"))
wait_for_all_runs_to_start(instance)
assert instance.get_runs_count() == 1
run = instance.get_runs()[0]
ticks = instance.get_job_ticks(
external_sensor.get_external_origin_id())
assert len(ticks) == 1
validate_tick(
ticks[0],
external_sensor,
freeze_datetime,
JobTickStatus.SUCCESS,
expected_run_ids=[run.run_id],
)
# run again, ensure
execute_sensor_iteration(instance,
get_default_daemon_logger("SensorDaemon"))
assert instance.get_runs_count() == 1
ticks = instance.get_job_ticks(
external_sensor.get_external_origin_id())
assert len(ticks) == 2
validate_tick(
ticks[0],
external_sensor,
freeze_datetime,
JobTickStatus.SKIPPED,
)
captured = capfd.readouterr()
assert f"Run {run.run_id} already completed with the run key `only_once` for run_key_sensor"
def test_wrong_config_sensor(external_repo_context, capfd):
freeze_datetime = pendulum.datetime(
year=2019,
month=2,
day=27,
hour=23,
minute=59,
second=59,
).in_tz("US/Central")
with instance_with_sensors(external_repo_context) as (instance, external_repo):
with pendulum.test(freeze_datetime):
external_sensor = external_repo.get_external_sensor("wrong_config_sensor")
instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING)
)
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 0
list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 1
validate_tick(
ticks[0],
external_sensor,
freeze_datetime,
JobTickStatus.FAILURE,
[],
"Error in config for pipeline the_pipeline",
)
captured = capfd.readouterr()
assert ("Error in config for pipeline the_pipeline") in captured.out
# Error repeats on subsequent ticks
list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 2
validate_tick(
ticks[0],
external_sensor,
freeze_datetime,
JobTickStatus.FAILURE,
[],
"Error in config for pipeline the_pipeline",
)
captured = capfd.readouterr()
assert ("Error in config for pipeline the_pipeline") in captured.out
def _test_launch_sensor_runs_in_subprocess(instance_ref, execution_datetime,
debug_crash_flags):
with DagsterInstance.from_ref(instance_ref) as instance:
try:
with pendulum.test(execution_datetime):
execute_sensor_iteration(
instance,
get_default_daemon_logger("SensorDaemon"),
debug_crash_flags=debug_crash_flags,
)
finally:
cleanup_test_instance(instance)
def _create_sensor_tick(instance):
with ProcessGrpcServerRegistry(wait_for_processes_on_exit=True) as grpc_server_registry:
with RepositoryLocationHandleManager(grpc_server_registry) as handle_manager:
list(
execute_sensor_iteration(
instance, get_default_daemon_logger("SensorDaemon"), handle_manager
)
)
def evaluate_sensors(instance, grpc_server_registry):
with RepositoryLocationHandleManager(
grpc_server_registry) as handle_manager:
list(
execute_sensor_iteration(
instance,
get_default_daemon_logger("SensorDaemon"),
handle_manager,
))
def test_sensor_next_ticks(graphql_context):
external_repository = graphql_context.get_repository_location(
main_repo_location_name()
).get_repository(main_repo_name())
graphql_context.instance.reconcile_scheduler_state(external_repository)
sensor_name = "always_no_config_sensor"
external_sensor = external_repository.get_external_sensor(sensor_name)
sensor_selector = infer_sensor_selector(graphql_context, sensor_name)
result = execute_dagster_graphql(
graphql_context, GET_SENSOR_QUERY, variables={"sensorSelector": sensor_selector}
)
# test default sensor off
assert result.data
assert result.data["sensorOrError"]["__typename"] == "Sensor"
next_tick = result.data["sensorOrError"]["nextTick"]
assert not next_tick
# test default sensor with no tick
graphql_context.instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING)
)
result = execute_dagster_graphql(
graphql_context, GET_SENSOR_QUERY, variables={"sensorSelector": sensor_selector}
)
assert result.data
assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 0
assert result.data["sensorOrError"]["__typename"] == "Sensor"
next_tick = result.data["sensorOrError"]["nextTick"]
assert not next_tick
# test default sensor with last tick
list(
execute_sensor_iteration(
graphql_context.instance, get_default_daemon_logger("SensorDaemon")
)
)
result = execute_dagster_graphql(
graphql_context, GET_SENSOR_QUERY, variables={"sensorSelector": sensor_selector}
)
assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 1
assert result.data
assert result.data["sensorOrError"]["__typename"] == "Sensor"
next_tick = result.data["sensorOrError"]["nextTick"]
assert next_tick
def _test_launch_sensor_runs_in_subprocess(instance_ref, execution_datetime,
debug_crash_flags):
with DagsterInstance.from_ref(instance_ref) as instance:
try:
with pendulum.test(execution_datetime), ProcessGrpcServerRegistry(
wait_for_processes_on_exit=True) as grpc_server_registry:
with RepositoryLocationHandleManager(
grpc_server_registry) as handle_manager:
list(
execute_sensor_iteration(
instance,
get_default_daemon_logger("SensorDaemon"),
handle_manager,
debug_crash_flags=debug_crash_flags,
))
finally:
cleanup_test_instance(instance)
def _create_tick(instance):
list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
def test_launch_once(external_repo_context, capfd):
freeze_datetime = to_timezone(
create_pendulum_time(
year=2019,
month=2,
day=27,
hour=23,
minute=59,
second=59,
tz="UTC",
),
"US/Central",
)
with instance_with_sensors(external_repo_context) as (instance,
external_repo):
with pendulum.test(freeze_datetime):
external_sensor = external_repo.get_external_sensor(
"run_key_sensor")
instance.add_job_state(
JobState(external_sensor.get_external_origin(), JobType.SENSOR,
JobStatus.RUNNING))
assert instance.get_runs_count() == 0
ticks = instance.get_job_ticks(
external_sensor.get_external_origin_id())
assert len(ticks) == 0
list(
execute_sensor_iteration(
instance, get_default_daemon_logger("SensorDaemon")))
wait_for_all_runs_to_start(instance)
assert instance.get_runs_count() == 1
run = instance.get_runs()[0]
ticks = instance.get_job_ticks(
external_sensor.get_external_origin_id())
assert len(ticks) == 1
validate_tick(
ticks[0],
external_sensor,
freeze_datetime,
JobTickStatus.SUCCESS,
expected_run_ids=[run.run_id],
)
# run again (after 30 seconds), to ensure that the run key maintains idempotence
freeze_datetime = freeze_datetime.add(seconds=30)
with pendulum.test(freeze_datetime):
list(
execute_sensor_iteration(
instance, get_default_daemon_logger("SensorDaemon")))
assert instance.get_runs_count() == 1
ticks = instance.get_job_ticks(
external_sensor.get_external_origin_id())
assert len(ticks) == 2
validate_tick(
ticks[0],
external_sensor,
freeze_datetime,
JobTickStatus.SKIPPED,
)
captured = capfd.readouterr()
assert (
f"Run {run.run_id} already completed with the run key `only_once` for run_key_sensor"
in captured.out)
launched_run = instance.get_runs()[0]
# Manually create a new run with the same tags
execute_pipeline(
the_pipeline,
run_config=launched_run.run_config,
tags=launched_run.tags,
instance=instance,
)
# Sensor loop still executes
freeze_datetime = freeze_datetime.add(seconds=30)
with pendulum.test(freeze_datetime):
list(
execute_sensor_iteration(
instance, get_default_daemon_logger("SensorDaemon")))
ticks = instance.get_job_ticks(
external_sensor.get_external_origin_id())
assert len(ticks) == 3
validate_tick(
ticks[0],
external_sensor,
freeze_datetime,
JobTickStatus.SKIPPED,
)
def run_iteration(self):
return execute_sensor_iteration(self._instance, self._logger)
