def test_launch_failure(external_repo_context, capfd): freeze_datetime = pendulum.datetime( year=2019, month=2, day=27, hour=23, minute=59, second=59, ).in_tz("US/Central") with instance_with_sensors( external_repo_context, overrides={ "run_launcher": {"module": "dagster.core.test_utils", "class": "ExplodingRunLauncher",}, }, ) as (instance, external_repo): with pendulum.test(freeze_datetime): external_sensor = external_repo.get_external_sensor("always_on_sensor") instance.add_job_state( JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING) ) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 0 execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")) assert instance.get_runs_count() == 1 run = instance.get_runs()[0] ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 1 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.SUCCESS, run.run_id ) captured = capfd.readouterr() assert ( "Run {run_id} created successfully but failed to launch.".format(run_id=run.run_id) ) in captured.out
def test_error_sensor(external_repo_context, capfd): freeze_datetime = pendulum.datetime( year=2019, month=2, day=27, hour=23, minute=59, second=59, ).in_tz("US/Central") with instance_with_sensors(external_repo_context) as (instance, external_repo): with pendulum.test(freeze_datetime): external_sensor = external_repo.get_external_sensor("error_sensor") instance.add_job_state( JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING) ) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 0 execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 1 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.FAILURE, None, "Error occurred during the execution of evaluation_fn for sensor error_sensor", ) captured = capfd.readouterr() assert ("Failed to resolve sensor for error_sensor : ") in captured.out assert ( "Error occurred during the execution of evaluation_fn for sensor error_sensor" ) in captured.out
def test_custom_interval_sensor(external_repo_context): freeze_datetime = pendulum.datetime(year=2019, month=2, day=28).in_tz("US/Central") with instance_with_sensors(external_repo_context) as (instance, external_repo): with pendulum.test(freeze_datetime): external_sensor = external_repo.get_external_sensor("custom_interval_sensor") instance.add_job_state( JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING) ) ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 0 list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))) ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 1 validate_tick(ticks[0], external_sensor, freeze_datetime, JobTickStatus.SKIPPED) freeze_datetime = freeze_datetime.add(seconds=30) with pendulum.test(freeze_datetime): list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))) ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) # no additional tick created after 30 seconds assert len(ticks) == 1 freeze_datetime = freeze_datetime.add(seconds=30) with pendulum.test(freeze_datetime): list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))) ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 2 expected_datetime = pendulum.datetime(year=2019, month=2, day=28, hour=0, minute=1) validate_tick(ticks[0], external_sensor, expected_datetime, JobTickStatus.SKIPPED)
def test_simple_sensor(external_repo_context, capfd): freeze_datetime = pendulum.datetime( year=2019, month=2, day=27, hour=23, minute=59, second=59, ).in_tz("US/Central") with instance_with_sensors(external_repo_context) as (instance, external_repo): with pendulum.test(freeze_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") instance.add_job_state( JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING) ) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 0 list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 1 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.SKIPPED, ) captured = capfd.readouterr() assert ( captured.out == """2019-02-27 17:59:59 - SensorDaemon - INFO - Checking for new runs for the following sensors: simple_sensor 2019-02-27 17:59:59 - SensorDaemon - INFO - Sensor returned false for simple_sensor, skipping """ ) freeze_datetime = freeze_datetime.add(seconds=1) with pendulum.test(freeze_datetime): list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))) wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 1 run = instance.get_runs()[0] validate_run_started(run) ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 2 expected_datetime = pendulum.datetime(year=2019, month=2, day=28) validate_tick( ticks[0], external_sensor, expected_datetime, JobTickStatus.SUCCESS, [run.run_id], ) captured = capfd.readouterr() assert ( captured.out == """2019-02-27 18:00:00 - SensorDaemon - INFO - Checking for new runs for the following sensors: simple_sensor 2019-02-27 18:00:00 - SensorDaemon - INFO - Launching run for simple_sensor 2019-02-27 18:00:00 - SensorDaemon - INFO - Completed launch of run {run_id} for simple_sensor """.format( run_id=run.run_id ) )
def test_launch_once(external_repo_context, capfd): freeze_datetime = pendulum.datetime( year=2019, month=2, day=27, hour=23, minute=59, second=59, ).in_tz("US/Central") with instance_with_sensors(external_repo_context) as (instance, external_repo): with pendulum.test(freeze_datetime): external_sensor = external_repo.get_external_sensor( "run_key_sensor") instance.add_job_state( JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING)) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks( external_sensor.get_external_origin_id()) assert len(ticks) == 0 execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")) wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 1 run = instance.get_runs()[0] ticks = instance.get_job_ticks( external_sensor.get_external_origin_id()) assert len(ticks) == 1 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.SUCCESS, expected_run_ids=[run.run_id], ) # run again, ensure execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")) assert instance.get_runs_count() == 1 ticks = instance.get_job_ticks( external_sensor.get_external_origin_id()) assert len(ticks) == 2 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.SKIPPED, ) captured = capfd.readouterr() assert f"Run {run.run_id} already completed with the run key `only_once` for run_key_sensor"
def test_wrong_config_sensor(external_repo_context, capfd): freeze_datetime = pendulum.datetime( year=2019, month=2, day=27, hour=23, minute=59, second=59, ).in_tz("US/Central") with instance_with_sensors(external_repo_context) as (instance, external_repo): with pendulum.test(freeze_datetime): external_sensor = external_repo.get_external_sensor("wrong_config_sensor") instance.add_job_state( JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING) ) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 0 list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 1 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.FAILURE, [], "Error in config for pipeline the_pipeline", ) captured = capfd.readouterr() assert ("Error in config for pipeline the_pipeline") in captured.out # Error repeats on subsequent ticks list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon"))) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 2 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.FAILURE, [], "Error in config for pipeline the_pipeline", ) captured = capfd.readouterr() assert ("Error in config for pipeline the_pipeline") in captured.out
def _test_launch_sensor_runs_in_subprocess(instance_ref, execution_datetime, debug_crash_flags): with DagsterInstance.from_ref(instance_ref) as instance: try: with pendulum.test(execution_datetime): execute_sensor_iteration( instance, get_default_daemon_logger("SensorDaemon"), debug_crash_flags=debug_crash_flags, ) finally: cleanup_test_instance(instance)
def _create_sensor_tick(instance): with ProcessGrpcServerRegistry(wait_for_processes_on_exit=True) as grpc_server_registry: with RepositoryLocationHandleManager(grpc_server_registry) as handle_manager: list( execute_sensor_iteration( instance, get_default_daemon_logger("SensorDaemon"), handle_manager ) )
def evaluate_sensors(instance, grpc_server_registry): with RepositoryLocationHandleManager( grpc_server_registry) as handle_manager: list( execute_sensor_iteration( instance, get_default_daemon_logger("SensorDaemon"), handle_manager, ))
def test_sensor_next_ticks(graphql_context): external_repository = graphql_context.get_repository_location( main_repo_location_name() ).get_repository(main_repo_name()) graphql_context.instance.reconcile_scheduler_state(external_repository) sensor_name = "always_no_config_sensor" external_sensor = external_repository.get_external_sensor(sensor_name) sensor_selector = infer_sensor_selector(graphql_context, sensor_name) result = execute_dagster_graphql( graphql_context, GET_SENSOR_QUERY, variables={"sensorSelector": sensor_selector} ) # test default sensor off assert result.data assert result.data["sensorOrError"]["__typename"] == "Sensor" next_tick = result.data["sensorOrError"]["nextTick"] assert not next_tick # test default sensor with no tick graphql_context.instance.add_job_state( JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING) ) result = execute_dagster_graphql( graphql_context, GET_SENSOR_QUERY, variables={"sensorSelector": sensor_selector} ) assert result.data assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 0 assert result.data["sensorOrError"]["__typename"] == "Sensor" next_tick = result.data["sensorOrError"]["nextTick"] assert not next_tick # test default sensor with last tick list( execute_sensor_iteration( graphql_context.instance, get_default_daemon_logger("SensorDaemon") ) ) result = execute_dagster_graphql( graphql_context, GET_SENSOR_QUERY, variables={"sensorSelector": sensor_selector} ) assert len(result.data["sensorOrError"]["sensorState"]["ticks"]) == 1 assert result.data assert result.data["sensorOrError"]["__typename"] == "Sensor" next_tick = result.data["sensorOrError"]["nextTick"] assert next_tick
def _test_launch_sensor_runs_in_subprocess(instance_ref, execution_datetime, debug_crash_flags): with DagsterInstance.from_ref(instance_ref) as instance: try: with pendulum.test(execution_datetime), ProcessGrpcServerRegistry( wait_for_processes_on_exit=True) as grpc_server_registry: with RepositoryLocationHandleManager( grpc_server_registry) as handle_manager: list( execute_sensor_iteration( instance, get_default_daemon_logger("SensorDaemon"), handle_manager, debug_crash_flags=debug_crash_flags, )) finally: cleanup_test_instance(instance)
def _create_tick(instance): list(execute_sensor_iteration(instance, get_default_daemon_logger("SensorDaemon")))
def test_launch_once(external_repo_context, capfd): freeze_datetime = to_timezone( create_pendulum_time( year=2019, month=2, day=27, hour=23, minute=59, second=59, tz="UTC", ), "US/Central", ) with instance_with_sensors(external_repo_context) as (instance, external_repo): with pendulum.test(freeze_datetime): external_sensor = external_repo.get_external_sensor( "run_key_sensor") instance.add_job_state( JobState(external_sensor.get_external_origin(), JobType.SENSOR, JobStatus.RUNNING)) assert instance.get_runs_count() == 0 ticks = instance.get_job_ticks( external_sensor.get_external_origin_id()) assert len(ticks) == 0 list( execute_sensor_iteration( instance, get_default_daemon_logger("SensorDaemon"))) wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 1 run = instance.get_runs()[0] ticks = instance.get_job_ticks( external_sensor.get_external_origin_id()) assert len(ticks) == 1 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.SUCCESS, expected_run_ids=[run.run_id], ) # run again (after 30 seconds), to ensure that the run key maintains idempotence freeze_datetime = freeze_datetime.add(seconds=30) with pendulum.test(freeze_datetime): list( execute_sensor_iteration( instance, get_default_daemon_logger("SensorDaemon"))) assert instance.get_runs_count() == 1 ticks = instance.get_job_ticks( external_sensor.get_external_origin_id()) assert len(ticks) == 2 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.SKIPPED, ) captured = capfd.readouterr() assert ( f"Run {run.run_id} already completed with the run key `only_once` for run_key_sensor" in captured.out) launched_run = instance.get_runs()[0] # Manually create a new run with the same tags execute_pipeline( the_pipeline, run_config=launched_run.run_config, tags=launched_run.tags, instance=instance, ) # Sensor loop still executes freeze_datetime = freeze_datetime.add(seconds=30) with pendulum.test(freeze_datetime): list( execute_sensor_iteration( instance, get_default_daemon_logger("SensorDaemon"))) ticks = instance.get_job_ticks( external_sensor.get_external_origin_id()) assert len(ticks) == 3 validate_tick( ticks[0], external_sensor, freeze_datetime, JobTickStatus.SKIPPED, )
def run_iteration(self): return execute_sensor_iteration(self._instance, self._logger)