def _verify_checkpoint_does_not_exist(context: DataContext, checkpoint: str,
usage_event: str) -> None:
if checkpoint in context.list_checkpoints():
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
f"A checkpoint named `{checkpoint}` already exists. Please choose a new name.",
)
def test_checkpoint_new_specify_datasource_with_ge_config_v2(
mock_emit,
caplog,
titanic_data_context_stats_enabled_config_version_2,
titanic_expectation_suite,
):
context = titanic_data_context_stats_enabled_config_version_2
root_dir = context.root_directory
assert context.list_checkpoints() == []
context.save_expectation_suite(titanic_expectation_suite)
assert context.list_expectation_suite_names() == ["Titanic.warning"]
mock_emit.reset_mock()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
f"checkpoint new passengers Titanic.warning -d {root_dir} --datasource mydatasource",
input="1\n1\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "A checkpoint named `passengers` was added to your project" in stdout
assert mock_emit.call_count == 2
assert mock_emit.call_args_list == [
mock.call({
"event_payload": {},
"event": "data_context.__init__",
"success": True
}),
mock.call({
"event": "cli.checkpoint.new",
"event_payload": {
"api_version": "v2"
},
"success": True,
}),
]
expected_checkpoint = os.path.join(
root_dir,
DataContextConfigDefaults.CHECKPOINTS_BASE_DIRECTORY.value,
"passengers.yml",
)
assert os.path.isfile(expected_checkpoint)
# Newup a context for additional assertions
context = DataContext(root_dir)
assert context.list_checkpoints() == ["passengers"]
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
allowed_deprecation_message=
LEGACY_CONFIG_DEFAULT_CHECKPOINT_STORE_MESSAGE,
)
def test_checkpoint_new_works_if_checkpoints_directory_is_missing(
mock_emit, caplog, titanic_data_context_stats_enabled,
titanic_expectation_suite):
context = titanic_data_context_stats_enabled
root_dir = context.root_directory
checkpoints_dir = os.path.join(
root_dir, DataContextConfigDefaults.CHECKPOINTS_BASE_DIRECTORY.value)
shutil.rmtree(checkpoints_dir)
assert not os.path.isdir(checkpoints_dir)
assert context.list_checkpoints() == []
context.save_expectation_suite(titanic_expectation_suite)
assert context.list_expectation_suite_names() == ["Titanic.warning"]
mock_emit.reset_mock()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
f"checkpoint new passengers Titanic.warning -d {root_dir}",
input="1\n1\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "A checkpoint named `passengers` was added to your project" in stdout
assert mock_emit.call_count == 2
assert mock_emit.call_args_list == [
mock.call({
"event_payload": {},
"event": "data_context.__init__",
"success": True
}),
mock.call({
"event": "cli.checkpoint.new",
"event_payload": {},
"success": True
}),
]
expected_checkpoint = os.path.join(
root_dir,
DataContextConfigDefaults.CHECKPOINTS_BASE_DIRECTORY.value,
"passengers.yml",
)
assert os.path.isfile(expected_checkpoint)
# Newup a context for additional assertions
context = DataContext(root_dir)
assert context.list_checkpoints() == ["passengers"]
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
allowed_deprecation_message=VALIDATION_OPERATORS_DEPRECATION_MESSAGE,
)
def _verify_checkpoint_does_not_exist(context: DataContext, checkpoint: str,
usage_event: str) -> None:
try:
if checkpoint in context.list_checkpoints():
toolkit.exit_with_failure_message_and_stats(
context,
usage_event,
f"A checkpoint named `{checkpoint}` already exists. Please choose a new name.",
)
except InvalidTopLevelConfigKeyError as e:
toolkit.exit_with_failure_message_and_stats(context, usage_event,
f"<red>{e}</red>")
def test_checkpoint_new_specify_datasource(mock_emit, caplog,
titanic_data_context_stats_enabled,
titanic_expectation_suite):
context = titanic_data_context_stats_enabled
root_dir = context.root_directory
assert context.list_checkpoints() == []
context.save_expectation_suite(titanic_expectation_suite)
assert context.list_expectation_suite_names() == ["Titanic.warning"]
mock_emit.reset_mock()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
f"checkpoint new passengers Titanic.warning -d {root_dir} --datasource mydatasource",
input="1\n1\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "A checkpoint named `passengers` was added to your project" in stdout
assert mock_emit.call_count == 2
assert mock_emit.call_args_list == [
mock.call({
"event_payload": {},
"event": "data_context.__init__",
"success": True
}),
mock.call({
"event": "cli.checkpoint.new",
"event_payload": {},
"success": True
}),
]
expected_checkpoint = os.path.join(root_dir, context.CHECKPOINTS_DIR,
"passengers.yml")
assert os.path.isfile(expected_checkpoint)
# Newup a context for additional assertions
context = DataContext(root_dir)
assert context.list_checkpoints() == ["passengers"]
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_checkpoint_new_happy_path_generates_checkpoint_yml_with_comments_with_ge_config_v2(
mock_emit,
caplog,
titanic_data_context_stats_enabled_config_version_2,
titanic_expectation_suite,
):
context = titanic_data_context_stats_enabled_config_version_2
root_dir = context.root_directory
assert context.list_checkpoints() == []
context.save_expectation_suite(titanic_expectation_suite)
assert context.list_expectation_suite_names() == ["Titanic.warning"]
mock_emit.reset_mock()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
f"checkpoint new passengers Titanic.warning -d {root_dir}",
input="1\n1\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "A checkpoint named `passengers` was added to your project" in stdout
assert mock_emit.call_count == 2
assert mock_emit.call_args_list == [
mock.call({
"event_payload": {},
"event": "data_context.__init__",
"success": True
}),
mock.call({
"event": "cli.checkpoint.new",
"event_payload": {
"api_version": "v2"
},
"success": True,
}),
]
expected_checkpoint = os.path.join(
root_dir,
DataContextConfigDefaults.CHECKPOINTS_BASE_DIRECTORY.value,
"passengers.yml",
)
assert os.path.isfile(expected_checkpoint)
# Newup a context for additional assertions
context = DataContext(root_dir)
assert context.list_checkpoints() == ["passengers"]
with open(expected_checkpoint) as f:
obs_file = f.read()
# This is snapshot-ish to prove that comments remain in place
# TODO: <Alex>ALEX</Alex>
# assert (
# """\
# # This checkpoint was created by the command `great_expectations checkpoint new`.
# #
# # A checkpoint is a list of one or more batches paired with one or more
# # Expectation Suites and a configurable Validation Operator.
# #
# # It can be run with the `great_expectations checkpoint run` command.
# # You can edit this file to add batches of data and expectation suites.
# #
# # For more details please see
# # https://docs.greatexpectations.io/en/latest/guides/how_to_guides/validation/how_to_add_validations_data_or_suites_to_a_checkpoint.html
# validation_operator_name: action_list_operator
# # Batches are a list of batch_kwargs paired with a list of one or more suite
# # names. A checkpoint can have one or more batches. This makes deploying
# # Great Expectations in your pipelines easy!
# batches:
# - batch_kwargs:"""
# in obs_file
# )
assert ("""\
batches:
- batch_kwargs:""" in obs_file)
assert "/data/Titanic.csv" in obs_file
assert ("""datasource: mydatasource
data_asset_name: Titanic
expectation_suite_names:
- Titanic.warning
""" in obs_file)
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
allowed_deprecation_message=
LEGACY_CONFIG_DEFAULT_CHECKPOINT_STORE_MESSAGE,
)
def test_checkpoint_new_happy_path_generates_checkpoint_yml_with_comments(
mock_emit, caplog, titanic_data_context_stats_enabled,
titanic_expectation_suite):
context = titanic_data_context_stats_enabled
root_dir = context.root_directory
assert context.list_checkpoints() == []
context.save_expectation_suite(titanic_expectation_suite)
assert context.list_expectation_suite_names() == ["Titanic.warning"]
mock_emit.reset_mock()
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
f"checkpoint new passengers Titanic.warning -d {root_dir}",
input="1\n1\n",
catch_exceptions=False,
)
stdout = result.stdout
assert result.exit_code == 0
assert "A checkpoint named `passengers` was added to your project" in stdout
assert mock_emit.call_count == 2
assert mock_emit.call_args_list == [
mock.call({
"event_payload": {},
"event": "data_context.__init__",
"success": True
}),
mock.call({
"event": "cli.checkpoint.new",
"event_payload": {},
"success": True
}),
]
expected_checkpoint = os.path.join(root_dir, context.CHECKPOINTS_DIR,
"passengers.yml")
assert os.path.isfile(expected_checkpoint)
# Newup a context for additional assertions
context = DataContext(root_dir)
assert context.list_checkpoints() == ["passengers"]
with open(expected_checkpoint, "r") as f:
obs_file = f.read()
# This is snapshot-ish to prove that comments remain in place
assert ("""\
# This checkpoint was created by the command `great_expectations checkpoint new`.
#
# A checkpoint is a list of one or more batches paired with one or more
# Expectation Suites and a configurable Validation Operator.
#
# It can be run with the `great_expectations checkpoint run` command.
# You can edit this file to add batches of data and expectation suites.
#
# For more details please see
# https://docs.greatexpectations.io/en/latest/command_line.html#great-expectations-checkpoint-new-checkpoint-suite
validation_operator_name: action_list_operator
# Batches are a list of batch_kwargs paired with a list of one or more suite
# names. A checkpoint can have one or more batches. This makes deploying
# Great Expectations in your pipelines easy!
batches:
- batch_kwargs:""" in obs_file)
assert "/data/Titanic.csv" in obs_file
assert ("""datasource: mydatasource
expectation_suite_names: # one or more suites may validate against a single batch
- Titanic.warning
""" in obs_file)
assert_no_logging_messages_or_tracebacks(caplog, result)