def test_sensor_invocation_args():
# Test no arg invocation
@sensor(pipeline_name="foo_pipeline")
def basic_sensor_no_arg():
return RunRequest(run_key=None, run_config={}, tags={})
assert basic_sensor_no_arg().run_config == {}
# Test underscore name
@sensor(pipeline_name="foo_pipeline")
def basic_sensor(_):
return RunRequest(run_key=None, run_config={}, tags={})
assert basic_sensor(build_sensor_context()).run_config == {}
assert basic_sensor(None).run_config == {}
# Test sensor arbitrary arg name
@sensor(pipeline_name="foo_pipeline")
def basic_sensor_with_context(_arbitrary_context):
return RunRequest(run_key=None, run_config={}, tags={})
context = build_sensor_context()
# Pass context as positional arg
assert basic_sensor_with_context(context).run_config == {}
# pass context as kwarg
assert basic_sensor_with_context(
_arbitrary_context=context).run_config == {}
# pass context as wrong kwarg
with pytest.raises(
DagsterInvalidInvocationError,
match="Sensor invocation expected argument '_arbitrary_context'.",
):
basic_sensor_with_context( # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
bad_context=context)
# pass context with no args
with pytest.raises(
DagsterInvalidInvocationError,
match=
"Sensor decorated function has context argument, but no context argument was "
"provided when invoking.",
):
basic_sensor_with_context() # pylint: disable=no-value-for-parameter
# pass context with too many args
with pytest.raises(
DagsterInvalidInvocationError,
match=
"Sensor invocation received multiple arguments. Only a first positional context "
"parameter should be provided when invoking.",
):
basic_sensor_with_context( # pylint: disable=redundant-keyword-arg
context,
_arbitrary_context=None)
def test_instance_access_built_sensor():
with pytest.raises(
DagsterInvariantViolationError,
match="Attempted to initialize dagster instance, but no instance reference was provided.",
):
build_sensor_context().instance # pylint: disable=expression-not-assigned
with instance_for_test() as instance:
assert isinstance(build_sensor_context(instance).instance, DagsterInstance)
def test_first_events(mock_event_records):
mock_event_records.side_effect = get_mock_event_records([("comments", 1), ("stories", 2)])
with instance_for_test() as instance:
context = build_sensor_context(instance=instance)
result = make_hn_tables_updated_sensor(job=GraphDefinition("test")).evaluate_tick(context)
assert len(result.run_requests) == 1
assert result.cursor == json.dumps({"comments": 1, "stories": 2})
def test_no_runs_for_different_mode():
with tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
instance.add_run(
PipelineRun(status=PipelineRunStatus.SUCCESS,
mode="xyz",
pipeline_name="download_pipeline"))
run_requests = list(
dbt_on_hn_download_finished(
build_sensor_context(instance=instance)))
assert len(run_requests) == 0
def test_no_runs_for_failed_run():
with tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
instance.add_run(
PipelineRun(
status=PipelineRunStatus.FAILURE,
mode="prod",
pipeline_name="download_pipeline",
run_config={"resources": DEFAULT_PARTITION_RESOURCE_CONFIG},
))
run_requests = list(
dbt_on_hn_download_finished(
build_sensor_context(instance=instance)))
assert len(run_requests) == 0
def test_no_runs_for_invalid_config():
with tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
instance.add_run(
PipelineRun(
status=PipelineRunStatus.FAILURE,
mode="prod",
pipeline_name="download_pipeline",
run_config={"I'm some config": {
"that is not": "valid"
}},
))
run_requests = list(
dbt_on_hn_download_finished(
build_sensor_context(instance=instance)))
assert len(run_requests) == 0
def test_pipeline_failure_sensor_has_no_request():
with instance_for_test() as instance:
execute_pipeline(
reconstructable(your_pipeline_name),
run_config={"solids": {
"foo": {
"config": {
"fail": False
}
}
}},
instance=instance,
)
context = build_sensor_context(instance)
requests = pipeline_failure_sensor.get_execution_data(context)
assert len(requests) == 0
def test_multiple_runs_for_successful_runs():
def get_should_launch_run():
return PipelineRun(
run_id=str(uuid.uuid4()),
status=PipelineRunStatus.SUCCESS,
mode="prod",
pipeline_name="download_pipeline",
run_config={"resources": DEFAULT_PARTITION_RESOURCE_CONFIG},
)
with tempfile.TemporaryDirectory() as temp_dir:
instance = DagsterInstance.local_temp(temp_dir)
for run in [
get_should_launch_run(),
PipelineRun(status=PipelineRunStatus.FAILURE,
mode="prod",
pipeline_name="download_pipeline"),
PipelineRun(status=PipelineRunStatus.SUCCESS,
mode="dev",
pipeline_name="weird_pipeline"),
PipelineRun(status=PipelineRunStatus.SUCCESS,
mode="test",
pipeline_name="download_pipeline"),
PipelineRun(status=PipelineRunStatus.SUCCESS,
mode="prod",
pipeline_name="other"),
get_should_launch_run(),
get_should_launch_run(),
get_should_launch_run(),
]:
instance.add_run(run)
run_requests = list(
dbt_on_hn_download_finished(
build_sensor_context(instance=instance)))
assert len(run_requests) == 4
for run_request in run_requests:
assert validate_run_config(dbt_pipeline, run_request.run_config)
def test_no_runs():
run_requests = list(
dbt_on_hn_download_finished(
build_sensor_context(instance=DagsterInstance.local_temp())))
assert len(run_requests) == 0
def test_my_directory_sensor_cursor():
context = build_sensor_context(cursor="0")
for run_request in my_directory_sensor_cursor(context):
assert validate_run_config(log_file_pipeline, run_request.run_config)
def test_instance_access_with_mock():
mock_instance = mock.MagicMock(spec=DagsterInstance)
assert build_sensor_context(
instance=mock_instance).instance == mock_instance
