def test_great_expectations_operator__validation_failure_raises_exc():
operator = GreatExpectationsOperator(
task_id="task_id",
data_context_root_dir=ge_root_dir,
checkpoint_name="taxi.fail.chk",
)
with pytest.raises(AirflowException):
operator.execute(context={})
def test_great_expectations_operator__checkpoint_config_with_substituted_batch_request_works_and_fails(
in_memory_data_context_config, in_memory_checkpoint_config):
failing_batch_request = BatchRequest(
**{
"datasource_name": "my_datasource",
"data_connector_name": "default_inferred_data_connector_name",
"data_asset_name": "yellow_tripdata_sample_2019-02.csv",
"data_connector_query": {
"index": -1
},
})
operator = GreatExpectationsOperator(
task_id="task_id",
data_context_config=in_memory_data_context_config,
checkpoint_config=in_memory_checkpoint_config,
checkpoint_kwargs={
"validations": [{
"batch_request": failing_batch_request
}]
},
fail_task_on_validation_failure=False,
)
result = operator.execute(context={}) # should fail the suite
logger.info(result)
assert result["success"] is False
def test_great_expectations_operator__return_json_dict():
operator = GreatExpectationsOperator(task_id="task_id",
data_context_root_dir=ge_root_dir,
checkpoint_name="taxi.pass.chk",
return_json_dict=True)
result = operator.execute(context={})
logger.info(result)
assert isinstance(result, dict)
assert result[
"_success"] # TODO: Update to "success" upon changes to `to_json_dict` in core GE
def test_great_expectations_operator__data_context_config_and_checkpoint_config_pass(
in_memory_data_context_config, in_memory_checkpoint_config):
operator = GreatExpectationsOperator(
task_id="task_id",
data_context_config=in_memory_data_context_config,
checkpoint_config=in_memory_checkpoint_config,
)
result = operator.execute(context={})
logger.info(result)
assert result["success"]
def test_great_expectations_operator__context_root_dir_and_checkpoint_name_pass(
):
operator = GreatExpectationsOperator(
task_id="task_id",
data_context_root_dir=ge_root_dir,
checkpoint_name="taxi.pass.chk",
)
result = operator.execute(context={})
logger.info(result)
assert result["success"]
def test_great_expectations_operator__validation_failure_callback():
my_callback = mock.MagicMock()
operator = GreatExpectationsOperator(
task_id="task_id",
data_context_root_dir=ge_root_dir,
checkpoint_name="taxi.fail.chk",
fail_task_on_validation_failure=False,
validation_failure_callback=my_callback,
)
result = operator.execute(context={})
assert result["success"] is False
my_callback.assert_called_once_with(result)
def test_great_expectations_operator__checkpoint_config_with_substituted_expectation_suite_works_and_fails(
in_memory_data_context_config, in_memory_checkpoint_config):
operator = GreatExpectationsOperator(
task_id="task_id",
data_context_config=in_memory_data_context_config,
checkpoint_config=in_memory_checkpoint_config,
checkpoint_kwargs={"expectation_suite_name": "taxi.demo_fail"},
fail_task_on_validation_failure=False,
)
result = operator.execute(context={}) # should fail the suite
logger.info(result)
assert result["success"] is False
def test_great_expectations_operator__validation_failure_logs_warning(caplog):
operator = GreatExpectationsOperator(
task_id="task_id",
data_context_root_dir=ge_root_dir,
checkpoint_name="taxi.fail.chk",
fail_task_on_validation_failure=False,
)
operator._log = logging.getLogger("my_test_logger")
caplog.set_level(level="WARNING", logger="my_test_logger")
caplog.clear()
result = operator.execute(context={})
assert result["success"] is False
assert ("my_test_logger", logging.WARNING) in ((r.name, r.levelno)
for r in caplog.records)