def test_great_expectations_operator__validation_failure_raises_exc(): operator = GreatExpectationsOperator( task_id="task_id", data_context_root_dir=ge_root_dir, checkpoint_name="taxi.fail.chk", ) with pytest.raises(AirflowException): operator.execute(context={})
def test_great_expectations_operator__checkpoint_config_with_substituted_batch_request_works_and_fails( in_memory_data_context_config, in_memory_checkpoint_config): failing_batch_request = BatchRequest( **{ "datasource_name": "my_datasource", "data_connector_name": "default_inferred_data_connector_name", "data_asset_name": "yellow_tripdata_sample_2019-02.csv", "data_connector_query": { "index": -1 }, }) operator = GreatExpectationsOperator( task_id="task_id", data_context_config=in_memory_data_context_config, checkpoint_config=in_memory_checkpoint_config, checkpoint_kwargs={ "validations": [{ "batch_request": failing_batch_request }] }, fail_task_on_validation_failure=False, ) result = operator.execute(context={}) # should fail the suite logger.info(result) assert result["success"] is False
def test_great_expectations_operator__return_json_dict(): operator = GreatExpectationsOperator(task_id="task_id", data_context_root_dir=ge_root_dir, checkpoint_name="taxi.pass.chk", return_json_dict=True) result = operator.execute(context={}) logger.info(result) assert isinstance(result, dict) assert result[ "_success"] # TODO: Update to "success" upon changes to `to_json_dict` in core GE
def test_great_expectations_operator__data_context_config_and_checkpoint_config_pass( in_memory_data_context_config, in_memory_checkpoint_config): operator = GreatExpectationsOperator( task_id="task_id", data_context_config=in_memory_data_context_config, checkpoint_config=in_memory_checkpoint_config, ) result = operator.execute(context={}) logger.info(result) assert result["success"]
def test_great_expectations_operator__context_root_dir_and_checkpoint_name_pass( ): operator = GreatExpectationsOperator( task_id="task_id", data_context_root_dir=ge_root_dir, checkpoint_name="taxi.pass.chk", ) result = operator.execute(context={}) logger.info(result) assert result["success"]
def test_great_expectations_operator__validation_failure_callback(): my_callback = mock.MagicMock() operator = GreatExpectationsOperator( task_id="task_id", data_context_root_dir=ge_root_dir, checkpoint_name="taxi.fail.chk", fail_task_on_validation_failure=False, validation_failure_callback=my_callback, ) result = operator.execute(context={}) assert result["success"] is False my_callback.assert_called_once_with(result)
def test_great_expectations_operator__checkpoint_config_with_substituted_expectation_suite_works_and_fails( in_memory_data_context_config, in_memory_checkpoint_config): operator = GreatExpectationsOperator( task_id="task_id", data_context_config=in_memory_data_context_config, checkpoint_config=in_memory_checkpoint_config, checkpoint_kwargs={"expectation_suite_name": "taxi.demo_fail"}, fail_task_on_validation_failure=False, ) result = operator.execute(context={}) # should fail the suite logger.info(result) assert result["success"] is False
def test_great_expectations_operator__validation_failure_logs_warning(caplog): operator = GreatExpectationsOperator( task_id="task_id", data_context_root_dir=ge_root_dir, checkpoint_name="taxi.fail.chk", fail_task_on_validation_failure=False, ) operator._log = logging.getLogger("my_test_logger") caplog.set_level(level="WARNING", logger="my_test_logger") caplog.clear() result = operator.execute(context={}) assert result["success"] is False assert ("my_test_logger", logging.WARNING) in ((r.name, r.levelno) for r in caplog.records)