def test_sensor_invocation_args(): # Test no arg invocation @sensor(pipeline_name="foo_pipeline") def basic_sensor_no_arg(): return RunRequest(run_key=None, run_config={}, tags={}) assert basic_sensor_no_arg().run_config == {} # Test underscore name @sensor(pipeline_name="foo_pipeline") def basic_sensor(_): return RunRequest(run_key=None, run_config={}, tags={}) assert basic_sensor(build_sensor_context()).run_config == {} assert basic_sensor(None).run_config == {} # Test sensor arbitrary arg name @sensor(pipeline_name="foo_pipeline") def basic_sensor_with_context(_arbitrary_context): return RunRequest(run_key=None, run_config={}, tags={}) context = build_sensor_context() # Pass context as positional arg assert basic_sensor_with_context(context).run_config == {} # pass context as kwarg assert basic_sensor_with_context( _arbitrary_context=context).run_config == {} # pass context as wrong kwarg with pytest.raises( DagsterInvalidInvocationError, match="Sensor invocation expected argument '_arbitrary_context'.", ): basic_sensor_with_context( # pylint: disable=unexpected-keyword-arg,no-value-for-parameter bad_context=context) # pass context with no args with pytest.raises( DagsterInvalidInvocationError, match= "Sensor decorated function has context argument, but no context argument was " "provided when invoking.", ): basic_sensor_with_context() # pylint: disable=no-value-for-parameter # pass context with too many args with pytest.raises( DagsterInvalidInvocationError, match= "Sensor invocation received multiple arguments. Only a first positional context " "parameter should be provided when invoking.", ): basic_sensor_with_context( # pylint: disable=redundant-keyword-arg context, _arbitrary_context=None)
def test_instance_access_built_sensor(): with pytest.raises( DagsterInvariantViolationError, match="Attempted to initialize dagster instance, but no instance reference was provided.", ): build_sensor_context().instance # pylint: disable=expression-not-assigned with instance_for_test() as instance: assert isinstance(build_sensor_context(instance).instance, DagsterInstance)
def test_first_events(mock_event_records): mock_event_records.side_effect = get_mock_event_records([("comments", 1), ("stories", 2)]) with instance_for_test() as instance: context = build_sensor_context(instance=instance) result = make_hn_tables_updated_sensor(job=GraphDefinition("test")).evaluate_tick(context) assert len(result.run_requests) == 1 assert result.cursor == json.dumps({"comments": 1, "stories": 2})
def test_no_runs_for_different_mode(): with tempfile.TemporaryDirectory() as temp_dir: instance = DagsterInstance.local_temp(temp_dir) instance.add_run( PipelineRun(status=PipelineRunStatus.SUCCESS, mode="xyz", pipeline_name="download_pipeline")) run_requests = list( dbt_on_hn_download_finished( build_sensor_context(instance=instance))) assert len(run_requests) == 0
def test_no_runs_for_failed_run(): with tempfile.TemporaryDirectory() as temp_dir: instance = DagsterInstance.local_temp(temp_dir) instance.add_run( PipelineRun( status=PipelineRunStatus.FAILURE, mode="prod", pipeline_name="download_pipeline", run_config={"resources": DEFAULT_PARTITION_RESOURCE_CONFIG}, )) run_requests = list( dbt_on_hn_download_finished( build_sensor_context(instance=instance))) assert len(run_requests) == 0
def test_no_runs_for_invalid_config(): with tempfile.TemporaryDirectory() as temp_dir: instance = DagsterInstance.local_temp(temp_dir) instance.add_run( PipelineRun( status=PipelineRunStatus.FAILURE, mode="prod", pipeline_name="download_pipeline", run_config={"I'm some config": { "that is not": "valid" }}, )) run_requests = list( dbt_on_hn_download_finished( build_sensor_context(instance=instance))) assert len(run_requests) == 0
def test_pipeline_failure_sensor_has_no_request(): with instance_for_test() as instance: execute_pipeline( reconstructable(your_pipeline_name), run_config={"solids": { "foo": { "config": { "fail": False } } }}, instance=instance, ) context = build_sensor_context(instance) requests = pipeline_failure_sensor.get_execution_data(context) assert len(requests) == 0
def test_multiple_runs_for_successful_runs(): def get_should_launch_run(): return PipelineRun( run_id=str(uuid.uuid4()), status=PipelineRunStatus.SUCCESS, mode="prod", pipeline_name="download_pipeline", run_config={"resources": DEFAULT_PARTITION_RESOURCE_CONFIG}, ) with tempfile.TemporaryDirectory() as temp_dir: instance = DagsterInstance.local_temp(temp_dir) for run in [ get_should_launch_run(), PipelineRun(status=PipelineRunStatus.FAILURE, mode="prod", pipeline_name="download_pipeline"), PipelineRun(status=PipelineRunStatus.SUCCESS, mode="dev", pipeline_name="weird_pipeline"), PipelineRun(status=PipelineRunStatus.SUCCESS, mode="test", pipeline_name="download_pipeline"), PipelineRun(status=PipelineRunStatus.SUCCESS, mode="prod", pipeline_name="other"), get_should_launch_run(), get_should_launch_run(), get_should_launch_run(), ]: instance.add_run(run) run_requests = list( dbt_on_hn_download_finished( build_sensor_context(instance=instance))) assert len(run_requests) == 4 for run_request in run_requests: assert validate_run_config(dbt_pipeline, run_request.run_config)
def test_no_runs(): run_requests = list( dbt_on_hn_download_finished( build_sensor_context(instance=DagsterInstance.local_temp()))) assert len(run_requests) == 0
def test_my_directory_sensor_cursor(): context = build_sensor_context(cursor="0") for run_request in my_directory_sensor_cursor(context): assert validate_run_config(log_file_pipeline, run_request.run_config)
def test_instance_access_with_mock(): mock_instance = mock.MagicMock(spec=DagsterInstance) assert build_sensor_context( instance=mock_instance).instance == mock_instance