def test_cli_init_spark_without_library_installed_instructs_user(
caplog, tmp_path_factory):
basedir = tmp_path_factory.mktemp("test_cli_init_diff")
basedir = str(basedir)
os.chdir(basedir)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(cli, ["init", "--no-view"],
input="\n\n1\n2\n",
catch_exceptions=False)
stdout = result.output
assert "Always know what to expect from your data" in stdout
assert "What data would you like Great Expectations to connect to" in stdout
assert "What are you processing your files with" in stdout
assert "Great Expectations relies on the library `pyspark`" in stdout
assert "Please `pip install pyspark` before trying again" in stdout
assert "Profiling" not in stdout
assert "Building" not in stdout
assert "Data Docs" not in stdout
assert "Great Expectations is now set up" not in stdout
assert result.exit_code == 1
assert os.path.isdir(os.path.join(basedir, "great_expectations"))
config_path = os.path.join(basedir,
"great_expectations/great_expectations.yml")
assert os.path.isfile(config_path)
config = yaml.load(open(config_path, "r"))
assert config["datasources"] == dict()
obs_tree = gen_directory_tree_str(
os.path.join(basedir, "great_expectations"))
assert (obs_tree == """\
great_expectations/
.gitignore
great_expectations.yml
checkpoints/
expectations/
notebooks/
pandas/
validation_playground.ipynb
spark/
validation_playground.ipynb
sql/
validation_playground.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
validations/
""")
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_cli_init_on_new_project(
mock_webbrowser, caplog, tmp_path_factory, titanic_sqlite_db_file
):
project_dir = str(tmp_path_factory.mktemp("test_cli_init_diff"))
ge_dir = os.path.join(project_dir, "great_expectations")
database_path = os.path.join(project_dir, "titanic.db")
shutil.copy(titanic_sqlite_db_file, database_path)
engine = create_engine("sqlite:///{}".format(database_path))
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["init", "-d", project_dir],
input="\n\n2\n6\ntitanic\n{}\n\n\n1\nwarning\n\n\n\n".format(engine.url),
catch_exceptions=False,
)
stdout = result.output
assert len(stdout) < 6000, "CLI output is unreasonably long."
assert "Always know what to expect from your data" in stdout
assert "What data would you like Great Expectations to connect to" in stdout
assert "Which database backend are you using" in stdout
assert "Give your new Datasource a short name" in stdout
assert "What is the url/connection string for the sqlalchemy connection" in stdout
assert "Attempting to connect to your database." in stdout
assert "Great Expectations connected to your database" in stdout
assert "Which table would you like to use?" in stdout
assert "Name the new Expectation Suite [main.titanic.warning]" in stdout
assert (
"Great Expectations will choose a couple of columns and generate expectations about them"
in stdout
)
assert "Generating example Expectation Suite..." in stdout
assert "Building" in stdout
assert "Data Docs" in stdout
assert "Great Expectations is now set up" in stdout
context = DataContext(ge_dir)
assert len(context.list_datasources()) == 1
assert context.list_datasources()[0]["class_name"] == "SqlAlchemyDatasource"
assert context.list_datasources()[0]["name"] == "titanic"
first_suite = context.list_expectation_suites()[0]
suite = context.get_expectation_suite(first_suite.expectation_suite_name)
assert len(suite.expectations) == 14
assert os.path.isdir(ge_dir)
config_path = os.path.join(project_dir, "great_expectations/great_expectations.yml")
assert os.path.isfile(config_path)
config = yaml.load(open(config_path, "r"))
data_source_class = config["datasources"]["titanic"]["data_asset_type"][
"class_name"
]
assert data_source_class == "SqlAlchemyDataset"
obs_tree = gen_directory_tree_str(ge_dir)
# Instead of monkey patching guids, just regex out the guids
guid_safe_obs_tree = re.sub(
r"[a-z0-9]{32}(?=\.(json|html))", "foobarbazguid", obs_tree
)
# print(guid_safe_obs_tree)
assert (
guid_safe_obs_tree
== """\
great_expectations/
.gitignore
great_expectations.yml
checkpoints/
expectations/
warning.json
notebooks/
pandas/
validation_playground.ipynb
spark/
validation_playground.ipynb
sql/
validation_playground.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
local_site/
index.html
expectations/
warning.html
static/
fonts/
HKGrotesk/
HKGrotesk-Bold.otf
HKGrotesk-BoldItalic.otf
HKGrotesk-Italic.otf
HKGrotesk-Light.otf
HKGrotesk-LightItalic.otf
HKGrotesk-Medium.otf
HKGrotesk-MediumItalic.otf
HKGrotesk-Regular.otf
HKGrotesk-SemiBold.otf
HKGrotesk-SemiBoldItalic.otf
images/
favicon.ico
glossary_scroller.gif
iterative-dev-loop.png
logo-long-vector.svg
logo-long.png
short-logo-vector.svg
short-logo.png
validation_failed_unexpected_values.gif
styles/
data_docs_custom_styles_template.css
data_docs_default_styles.css
validations/
warning/
20190926T134241.000000Z/
20190926T134241.000000Z/
foobarbazguid.html
validations/
warning/
20190926T134241.000000Z/
20190926T134241.000000Z/
foobarbazguid.json
"""
)
assert_no_logging_messages_or_tracebacks(caplog, result)
assert result.exit_code == 0
assert mock_webbrowser.call_count == 1
assert (
"{}/great_expectations/uncommitted/data_docs/local_site/validations/warning/".format(
project_dir
)
in mock_webbrowser.call_args[0][0]
)
def _library_not_loaded_test(tmp_path_factory, cli_input, library_name,
library_import_name, my_caplog):
"""
This test requires that a library is NOT installed. It tests that:
- a helpful error message is returned to install the missing library
- the expected tree structure is in place
- the config yml contains an empty dict in its datasource entry
"""
basedir = tmp_path_factory.mktemp("test_cli_init_diff")
basedir = str(basedir)
os.chdir(basedir)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(cli, ["init", "--no-view"],
input=cli_input,
catch_exceptions=False)
stdout = result.output
assert "Always know what to expect from your data" in stdout
assert "What data would you like Great Expectations to connect to" in stdout
assert "Which database backend are you using" in stdout
assert "Give your new Datasource a short name" in stdout
assert (
"""Next, we will configure database credentials and store them in the `my_db` section
of this config file: great_expectations/uncommitted/config_variables.yml"""
in stdout)
assert ("Great Expectations relies on the library `{}`".format(
library_import_name) in stdout)
assert "Please `pip install {}` before trying again".format(
library_name) in stdout
assert "Profiling" not in stdout
assert "Building" not in stdout
assert "Data Docs" not in stdout
assert "Great Expectations is now set up" not in stdout
assert result.exit_code == 1
assert os.path.isdir(os.path.join(basedir, "great_expectations"))
config_path = os.path.join(basedir,
"great_expectations/great_expectations.yml")
assert os.path.isfile(config_path)
config = yaml.load(open(config_path, "r"))
assert config["datasources"] == dict()
obs_tree = gen_directory_tree_str(
os.path.join(basedir, "great_expectations"))
assert (obs_tree == """\
great_expectations/
.gitignore
great_expectations.yml
checkpoints/
expectations/
notebooks/
pandas/
validation_playground.ipynb
spark/
validation_playground.ipynb
sql/
validation_playground.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
validations/
""")
assert_no_logging_messages_or_tracebacks(my_caplog, result)
def test_cli_init_on_new_project(mock_emit, mock_webbrowser, caplog,
tmp_path_factory, monkeypatch):
monkeypatch.delenv("GE_USAGE_STATS",
raising=False) # Undo the project-wide test default
project_dir = str(tmp_path_factory.mktemp("test_cli_init_diff"))
os.makedirs(os.path.join(project_dir, "data"))
data_folder_path = os.path.join(project_dir, "data")
data_path = os.path.join(project_dir, "data", "Titanic.csv")
fixture_path = file_relative_path(__file__, "../test_sets/Titanic.csv")
shutil.copy(fixture_path, data_path)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["init", "-d", project_dir],
input="\n\n1\n1\n{}\n\n\n\n2\n{}\n\n\n\n".format(
data_folder_path, data_path),
catch_exceptions=False,
)
stdout = result.output
assert mock_webbrowser.call_count == 1
assert (
"{}/great_expectations/uncommitted/data_docs/local_site/validations/Titanic/warning/"
.format(project_dir) in mock_webbrowser.call_args[0][0])
assert len(stdout) < 6000, "CLI output is unreasonably long."
assert "Always know what to expect from your data" in stdout
assert "What data would you like Great Expectations to connect to" in stdout
assert "What are you processing your files with" in stdout
assert (
"Enter the path of a data file (relative or absolute, s3a:// and gs:// paths are ok too)"
in stdout)
assert "Name the new Expectation Suite [Titanic.warning]" in stdout
assert (
"Great Expectations will choose a couple of columns and generate expectations about them"
in stdout)
assert "Generating example Expectation Suite..." in stdout
assert "Building" in stdout
assert "Data Docs" in stdout
assert "Done generating example Expectation Suite" in stdout
assert "Great Expectations is now set up" in stdout
assert os.path.isdir(os.path.join(project_dir, "great_expectations"))
config_path = os.path.join(project_dir,
"great_expectations/great_expectations.yml")
assert os.path.isfile(config_path)
config = yaml.load(open(config_path))
data_source_class = config["datasources"]["data__dir"]["data_asset_type"][
"class_name"]
assert data_source_class == "PandasDataset"
obs_tree = gen_directory_tree_str(
os.path.join(project_dir, "great_expectations"))
# Instead of monkey patching guids, just regex out the guids
guid_safe_obs_tree = re.sub(r"[a-z0-9]{32}(?=\.(json|html))",
"foobarbazguid", obs_tree)
# print(guid_safe_obs_tree)
assert (guid_safe_obs_tree == """great_expectations/
.gitignore
great_expectations.yml
checkpoints/
expectations/
.ge_store_backend_id
Titanic/
warning.json
notebooks/
pandas/
validation_playground.ipynb
spark/
validation_playground.ipynb
sql/
validation_playground.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
local_site/
index.html
expectations/
Titanic/
warning.html
static/
fonts/
HKGrotesk/
HKGrotesk-Bold.otf
HKGrotesk-BoldItalic.otf
HKGrotesk-Italic.otf
HKGrotesk-Light.otf
HKGrotesk-LightItalic.otf
HKGrotesk-Medium.otf
HKGrotesk-MediumItalic.otf
HKGrotesk-Regular.otf
HKGrotesk-SemiBold.otf
HKGrotesk-SemiBoldItalic.otf
images/
favicon.ico
glossary_scroller.gif
iterative-dev-loop.png
logo-long-vector.svg
logo-long.png
short-logo-vector.svg
short-logo.png
validation_failed_unexpected_values.gif
styles/
data_docs_custom_styles_template.css
data_docs_default_styles.css
validations/
Titanic/
warning/
20190926T134241.000000Z/
20190926T134241.000000Z/
foobarbazguid.html
validations/
.ge_store_backend_id
Titanic/
warning/
20190926T134241.000000Z/
20190926T134241.000000Z/
foobarbazguid.json
""")
assert mock_emit.call_count == 9
assert mock_emit.call_args_list[1] == mock.call({
"event_payload": {},
"event": "cli.init.create",
"success": True
})
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_ValidationsStore_with_TupleFileSystemStoreBackend(tmp_path_factory):
path = str(
tmp_path_factory.mktemp(
"test_ValidationResultStore_with_TupleFileSystemStoreBackend__dir"
)
)
project_path = str(tmp_path_factory.mktemp("my_dir"))
my_store = ValidationsStore(
store_backend={
"module_name": "great_expectations.data_context.store",
"class_name": "TupleFilesystemStoreBackend",
"base_directory": "my_store/",
},
runtime_environment={"root_directory": path},
)
with pytest.raises(TypeError):
my_store.get("not_a_ValidationResultIdentifier")
ns_1 = ValidationResultIdentifier(
expectation_suite_identifier=ExpectationSuiteIdentifier("asset.quarantine"),
run_id="prod-100",
batch_identifier="batch_id",
)
my_store.set(ns_1, ExpectationSuiteValidationResult(success=True))
assert my_store.get(ns_1) == ExpectationSuiteValidationResult(
success=True, statistics={}, results=[]
)
ns_2 = ValidationResultIdentifier.from_tuple(
(
"asset",
"quarantine",
"prod-20",
datetime.datetime.now(datetime.timezone.utc),
"batch_id",
)
)
my_store.set(ns_2, ExpectationSuiteValidationResult(success=False))
assert my_store.get(ns_2) == ExpectationSuiteValidationResult(
success=False, statistics={}, results=[]
)
print(my_store.list_keys())
assert set(my_store.list_keys()) == {
ns_1,
ns_2,
}
print(gen_directory_tree_str(path))
assert (
gen_directory_tree_str(path)
== """\
test_ValidationResultStore_with_TupleFileSystemStoreBackend__dir0/
my_store/
asset/
quarantine/
prod-100/
20190926T134241.000000Z/
batch_id.json
prod-20/
20190926T134241.000000Z/
batch_id.json
"""
)
def test_project_upgrade_with_exception(v10_project_directory, caplog):
# test project upgrade that requires manual steps
# copy v2 yml
shutil.copy(
file_relative_path(
__file__,
"../../test_fixtures/upgrade_helper/great_expectations_v1_basic_with_exception.yml",
),
os.path.join(v10_project_directory, "great_expectations.yml"),
)
runner: CliRunner = CliRunner(mix_stderr=False)
result: Result = runner.invoke(
cli,
["-c", v10_project_directory, "--v3-api", "project", "upgrade"],
input="\n",
catch_exceptions=False,
)
stdout: str = result.stdout
with open(
file_relative_path(
__file__,
"../../test_fixtures/upgrade_helper/test_project_upgrade_with_exception_expected_stdout.fixture",
)) as f:
expected_stdout: str = f.read()
expected_stdout = expected_stdout.replace("GE_PROJECT_DIR",
v10_project_directory)
assert stdout == expected_stdout
expected_project_tree_str: str = """\
great_expectations/
.gitignore
great_expectations.yml
checkpoints/
.gitkeep
expectations/
.ge_store_backend_id
.gitkeep
notebooks/
.gitkeep
plugins/
custom_store_backends/
__init__.py
my_custom_store_backend.py
uncommitted/
config_variables.yml
data_docs/
local_site/
expectations/
.gitkeep
static/
.gitkeep
validations/
diabetic_data/
warning/
20200430T191246.763896Z/
20200430T191246.763896Z/
c3b4c5df224fef4b1a056a0f3b93aba5.html
logs/
project_upgrades/
UpgradeHelperV11_20190926T134241.000000Z.json
validations/
.ge_store_backend_id
diabetic_data/
warning/
20200430T191246.763896Z/
20200430T191246.763896Z/
c3b4c5df224fef4b1a056a0f3b93aba5.json
"""
obs_project_tree_str: str = gen_directory_tree_str(
startpath=v10_project_directory)
assert obs_project_tree_str == expected_project_tree_str
# make sure config number not incremented
assert (DataContext.get_ge_config_version(
context_root_dir=v10_project_directory) == 1.0)
with open(
file_relative_path(
__file__,
"../../test_fixtures/upgrade_helper/UpgradeHelperV11_basic_upgrade_with_exception_log.json",
)) as f:
expected_upgrade_log_dict: dict = json.load(f)
expected_upgrade_log_str: str = json.dumps(expected_upgrade_log_dict)
expected_upgrade_log_str = expected_upgrade_log_str.replace(
"GE_PROJECT_DIR", v10_project_directory)
expected_upgrade_log_str = expected_upgrade_log_str.replace(
"GE_PATH",
os.path.split(great_expectations.__file__)[0])
expected_upgrade_log_dict = json.loads(expected_upgrade_log_str)
with open(
f"{v10_project_directory}/uncommitted/logs/project_upgrades/UpgradeHelperV11_20190926T134241.000000Z.json"
) as f:
obs_upgrade_log_dict: dict = json.load(f)
obs_upgrade_log_dict["exceptions"][0]["exception_message"] = ""
assert obs_upgrade_log_dict == expected_upgrade_log_dict
def test_render_full_static_site_from_empty_project(tmp_path_factory, filesystem_csv_3):
# TODO : Use a standard test fixture
# TODO : Have that test fixture copy a directory, rather than building a new one from scratch
base_dir = str(tmp_path_factory.mktemp("project_dir"))
project_dir = os.path.join(base_dir, "project_path")
os.mkdir(project_dir)
os.makedirs(os.path.join(project_dir, "data"))
os.makedirs(os.path.join(project_dir, "data/titanic"))
shutil.copy(
file_relative_path(__file__, "../test_sets/Titanic.csv"),
str(os.path.join(project_dir, "data/titanic/Titanic.csv"))
)
os.makedirs(os.path.join(project_dir, "data/random"))
shutil.copy(
os.path.join(filesystem_csv_3, "f1.csv"),
str(os.path.join(project_dir, "data/random/f1.csv"))
)
shutil.copy(
os.path.join(filesystem_csv_3, "f2.csv"),
str(os.path.join(project_dir, "data/random/f2.csv"))
)
assert gen_directory_tree_str(project_dir) == """\
project_path/
data/
random/
f1.csv
f2.csv
titanic/
Titanic.csv
"""
context = DataContext.create(project_dir)
ge_directory = os.path.join(project_dir, "great_expectations")
context.add_datasource("titanic",
module_name="great_expectations.datasource",
class_name="PandasDatasource",
generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": os.path.join(project_dir, "data/titanic/")
}
}
)
context.add_datasource("random",
module_name="great_expectations.datasource",
class_name="PandasDatasource",
generators={
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": os.path.join(project_dir, "data/random/")
}
}
)
context.profile_datasource("titanic")
# Replicate the batch id of the batch that will be profiled in order to generate the file path of the
# validation result
titanic_profiled_batch_id = PathBatchKwargs({
'path': os.path.join(project_dir, 'data/titanic/Titanic.csv'),
'datasource': 'titanic'}
).to_id()
tree_str = gen_directory_tree_str(project_dir)
assert tree_str == """project_path/
data/
random/
f1.csv
f2.csv
titanic/
Titanic.csv
great_expectations/
.gitignore
great_expectations.yml
expectations/
titanic/
subdir_reader/
Titanic/
BasicDatasetProfiler.json
notebooks/
pandas/
validation_playground.ipynb
spark/
validation_playground.ipynb
sql/
validation_playground.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
validations/
titanic/
subdir_reader/
Titanic/
BasicDatasetProfiler/
profiling/
{}.json
""".format(titanic_profiled_batch_id)
context.profile_datasource("random")
context.build_data_docs()
f1_profiled_batch_id = PathBatchKwargs({
'path': os.path.join(project_dir, 'data/random/f1.csv'),
'datasource': 'random'}
).to_id()
f2_profiled_batch_id = PathBatchKwargs({
'path': os.path.join(project_dir, 'data/random/f2.csv'),
'datasource': 'random'}
).to_id()
data_docs_dir = os.path.join(project_dir, "great_expectations/uncommitted/data_docs")
observed = gen_directory_tree_str(data_docs_dir)
assert observed == """\
data_docs/
local_site/
index.html
expectations/
random/
subdir_reader/
f1/
BasicDatasetProfiler.html
f2/
BasicDatasetProfiler.html
titanic/
subdir_reader/
Titanic/
BasicDatasetProfiler.html
static/
fonts/
HKGrotesk/
HKGrotesk-Bold.otf
HKGrotesk-BoldItalic.otf
HKGrotesk-Italic.otf
HKGrotesk-Light.otf
HKGrotesk-LightItalic.otf
HKGrotesk-Medium.otf
HKGrotesk-MediumItalic.otf
HKGrotesk-Regular.otf
HKGrotesk-SemiBold.otf
HKGrotesk-SemiBoldItalic.otf
images/
favicon.ico
glossary_scroller.gif
iterative-dev-loop.png
logo-long-vector.svg
logo-long.png
short-logo-vector.svg
short-logo.png
validation_failed_unexpected_values.gif
styles/
data_docs_custom_styles_template.css
data_docs_default_styles.css
validations/
random/
subdir_reader/
f1/
BasicDatasetProfiler/
profiling/
{0:s}.html
f2/
BasicDatasetProfiler/
profiling/
{1:s}.html
titanic/
subdir_reader/
Titanic/
BasicDatasetProfiler/
profiling/
{2:s}.html
""".format(f1_profiled_batch_id, f2_profiled_batch_id, titanic_profiled_batch_id)
# save data_docs locally
safe_mmkdir("./tests/data_context/output")
safe_mmkdir("./tests/data_context/output/data_docs")
if os.path.isdir("./tests/data_context/output/data_docs"):
shutil.rmtree("./tests/data_context/output/data_docs")
shutil.copytree(
os.path.join(
ge_directory,
"uncommitted/data_docs/"
),
"./tests/data_context/output/data_docs"
)
def test_v3_configuration_store(tmp_path_factory):
root_directory_path: str = "test_v3_configuration_store"
root_directory: str = str(tmp_path_factory.mktemp(root_directory_path))
base_directory: str = str(Path(root_directory) / "some_store_config_dir")
config_0: SampleConfig = SampleConfig(some_param_0="test_str_0",
some_param_1=65)
store_name_0: str = "test_config_store_0"
configuration_name_0: str = "test_config_name_0"
with pytest.raises(FileNotFoundError):
save_config_to_filesystem(
configuration_store_class_name="unknown_class",
configuration_store_module_name="unknown_module",
store_name=store_name_0,
base_directory=base_directory,
configuration_key=configuration_name_0,
configuration=config_0,
)
save_config_to_filesystem(
configuration_store_class_name="SampleConfigurationStore",
configuration_store_module_name=SampleConfigurationStore.__module__,
store_name=store_name_0,
base_directory=base_directory,
configuration_key=configuration_name_0,
configuration=config_0,
)
dir_tree: str = gen_directory_tree_str(startpath=base_directory)
assert (dir_tree == """some_store_config_dir/
.ge_store_backend_id
test_config_name_0.yml
""")
assert (len([
path for path in Path(base_directory).iterdir()
if str(path).find(".ge_store_backend_id") == (-1)
]) == 1)
stored_file_name_0: str = Path(
base_directory) / f"{configuration_name_0}.yml"
with open(stored_file_name_0) as f:
config: CommentedMap = yaml.load(f)
expected_config: CommentedMap = CommentedMap({
"some_param_0": "test_str_0",
"some_param_1": 65
})
assert config == expected_config
with pytest.raises(ValueError):
# noinspection PyUnusedLocal
loaded_config: BaseYamlConfig = load_config_from_filesystem(
configuration_store_class_name="SampleConfigurationStore",
configuration_store_module_name=SampleConfigurationStore.
__module__,
store_name=store_name_0,
base_directory="unknown_base_directory",
configuration_key=configuration_name_0,
)
with pytest.raises(ge_exceptions.InvalidKeyError):
# noinspection PyUnusedLocal
loaded_config: BaseYamlConfig = load_config_from_filesystem(
configuration_store_class_name="SampleConfigurationStore",
configuration_store_module_name=SampleConfigurationStore.
__module__,
store_name=store_name_0,
base_directory=base_directory,
configuration_key="unknown_configuration",
)
loaded_config: BaseYamlConfig = load_config_from_filesystem(
configuration_store_class_name="SampleConfigurationStore",
configuration_store_module_name=SampleConfigurationStore.__module__,
store_name=store_name_0,
base_directory=base_directory,
configuration_key=configuration_name_0,
)
assert loaded_config.to_json_dict() == config_0.to_json_dict()
config_1: SampleConfig = SampleConfig(some_param_0="test_str_1",
some_param_1=26)
store_name_1: str = "test_config_store_1"
configuration_name_1: str = "test_config_name_1"
save_config_to_filesystem(
configuration_store_class_name="SampleConfigurationStore",
configuration_store_module_name=SampleConfigurationStore.__module__,
store_name=store_name_1,
base_directory=base_directory,
configuration_key=configuration_name_1,
configuration=config_1,
)
dir_tree: str = gen_directory_tree_str(startpath=base_directory)
assert (dir_tree == """some_store_config_dir/
.ge_store_backend_id
test_config_name_0.yml
test_config_name_1.yml
""")
assert (len([
path for path in Path(base_directory).iterdir()
if str(path).find(".ge_store_backend_id") == (-1)
]) == 2)
stored_file_name_1: str = Path(
base_directory) / f"{configuration_name_1}.yml"
with open(stored_file_name_1) as f:
config: CommentedMap = yaml.load(f)
expected_config: CommentedMap = CommentedMap({
"some_param_0": "test_str_1",
"some_param_1": 26
})
assert config == expected_config
loaded_config: BaseYamlConfig = load_config_from_filesystem(
configuration_store_class_name="SampleConfigurationStore",
configuration_store_module_name=SampleConfigurationStore.__module__,
store_name=store_name_1,
base_directory=base_directory,
configuration_key=configuration_name_1,
)
assert loaded_config.to_json_dict() == config_1.to_json_dict()
delete_config_from_filesystem(
configuration_store_class_name="SampleConfigurationStore",
configuration_store_module_name=SampleConfigurationStore.__module__,
store_name=store_name_0,
base_directory=base_directory,
configuration_key=configuration_name_0,
)
assert (len([
path for path in Path(base_directory).iterdir()
if str(path).find(".ge_store_backend_id") == (-1)
]) == 1)
delete_config_from_filesystem(
configuration_store_class_name="SampleConfigurationStore",
configuration_store_module_name=SampleConfigurationStore.__module__,
store_name=store_name_1,
base_directory=base_directory,
configuration_key=configuration_name_1,
)
assert (len([
path for path in Path(base_directory).iterdir()
if str(path).find(".ge_store_backend_id") == (-1)
]) == 0)
def test_cli_init_connection_string_non_working_db_connection_instructs_user_and_leaves_entries_in_config_files_for_debugging(
mock_webbrowser, caplog, monkeypatch, tmp_path_factory, sa):
root_dir = tmp_path_factory.mktemp("bad_con_string_test")
root_dir = str(root_dir)
os.chdir(root_dir)
runner = CliRunner(mix_stderr=False)
monkeypatch.chdir(root_dir)
result = runner.invoke(
cli,
["--v3-api", "init"],
input="\n\n2\n6\nmy_db\nsqlite:////not_a_real.db\n\nn\n",
catch_exceptions=False,
)
stdout = result.output
assert mock_webbrowser.call_count == 0
assert "Always know what to expect from your data" in stdout
assert "What data would you like Great Expectations to connect to" in stdout
assert "Which database backend are you using" in stdout
assert "What is the url/connection string for the sqlalchemy connection" in stdout
assert "Give your new Datasource a short name" in stdout
assert "Attempting to connect to your database. This may take a moment" in stdout
assert "Cannot connect to the database" in stdout
assert "Profiling" not in stdout
assert "Building" not in stdout
assert "Data Docs" not in stdout
assert "Great Expectations is now set up" not in stdout
assert result.exit_code == 1
ge_dir = os.path.join(root_dir, DataContext.GE_DIR)
assert os.path.isdir(ge_dir)
config_path = os.path.join(ge_dir, DataContext.GE_YML)
assert os.path.isfile(config_path)
config = yaml.load(open(config_path))
assert config["datasources"] == {
"my_db": {
"data_asset_type": {
"module_name": None,
"class_name": "SqlAlchemyDataset",
},
"credentials": "${my_db}",
"class_name": "SqlAlchemyDatasource",
"module_name": "great_expectations.datasource",
}
}
config_path = os.path.join(ge_dir, DataContext.GE_UNCOMMITTED_DIR,
"config_variables.yml")
config = yaml.load(open(config_path))
assert config["my_db"] == {"url": "sqlite:////not_a_real.db"}
obs_tree = gen_directory_tree_str(
os.path.join(root_dir, "great_expectations"))
assert (obs_tree == """\
great_expectations/
.gitignore
great_expectations.yml
checkpoints/
expectations/
.ge_store_backend_id
notebooks/
pandas/
validation_playground.ipynb
spark/
validation_playground.ipynb
sql/
validation_playground.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
validations/
.ge_store_backend_id
""")
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_cli_init_on_new_project(tmp_path_factory, filesystem_csv_2):
try:
basedir = tmp_path_factory.mktemp("test_cli_init_diff")
basedir = str(basedir)
os.makedirs(os.path.join(basedir, "data"))
curdir = os.path.abspath(os.getcwd())
shutil.copy("./tests/test_sets/Titanic.csv",
str(os.path.join(basedir, "data/Titanic.csv")))
os.chdir(basedir)
runner = CliRunner()
result = runner.invoke(cli, ["init"],
input="Y\n1\n%s\n\nn\n\n" %
str(os.path.join(basedir, "data")))
print(result.output)
print("result.output length:", len(result.output))
assert len(result.output) < 10000, "CLI output is unreasonably long."
assert len(
re.findall("{", result.output)
) < 100, "CLI contains way more '{' than we would reasonably expect."
assert """Always know what to expect from your data""" in result.output
assert """Let's add Great Expectations to your project""" in result.output
assert """open a tutorial notebook""" in result.output
assert os.path.isdir(os.path.join(basedir, "great_expectations"))
assert os.path.isfile(
os.path.join(basedir, "great_expectations/great_expectations.yml"))
config = yaml.load(
open(
os.path.join(basedir,
"great_expectations/great_expectations.yml"),
"r"))
assert config["datasources"]["data__dir"][
"class_name"] == "PandasDatasource"
print(
gen_directory_tree_str(os.path.join(basedir,
"great_expectations")))
assert gen_directory_tree_str(
os.path.join(basedir, "great_expectations")) == """\
great_expectations/
.gitignore
great_expectations.yml
datasources/
expectations/
data__dir/
default/
Titanic/
BasicDatasetProfiler.json
notebooks/
create_expectations.ipynb
integrate_validation_into_pipeline.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
local_site/
index.html
expectations/
data__dir/
default/
Titanic/
BasicDatasetProfiler.html
validations/
profiling/
data__dir/
default/
Titanic/
BasicDatasetProfiler.html
samples/
validations/
profiling/
data__dir/
default/
Titanic/
BasicDatasetProfiler.json
"""
assert os.path.isfile(
os.path.join(
basedir,
"great_expectations/expectations/data__dir/default/Titanic/BasicDatasetProfiler.json"
))
fnames = []
path = os.path.join(
basedir,
"great_expectations/uncommitted/validations/profiling/data__dir/default/Titanic"
)
for (dirpath, dirnames, filenames) in os.walk(path):
for filename in filenames:
fnames.append(filename)
assert fnames == ["BasicDatasetProfiler.json"]
assert os.path.isfile(
os.path.join(
basedir,
"great_expectations/uncommitted/data_docs/local_site/validations/profiling/data__dir/default/Titanic/BasicDatasetProfiler.html"
))
assert os.path.getsize(
os.path.join(
basedir,
"great_expectations/uncommitted/data_docs/local_site/validations/profiling/data__dir/default/Titanic/BasicDatasetProfiler.html"
)) > 0
print(result)
except:
raise
finally:
os.chdir(curdir)
def test_cli_init_on_existing_ge_yml_with_missing_uncommitted_dirs_and_missing_config_variables_yml(
tmp_path_factory):
"""
This test walks through an onboarding experience.
The user just is missing some uncommitted dirs and is missing
config_variables.yml
"""
tmp_dir = str(tmp_path_factory.mktemp("more_stuff"))
ge_dir = os.path.join(tmp_dir, "great_expectations")
curdir = os.path.abspath(os.getcwd())
os.chdir(tmp_dir)
runner = CliRunner()
runner.invoke(cli, ["init"], input="Y\n4\n")
# mangle setup
uncommitted_dir = os.path.join(ge_dir, "uncommitted")
shutil.rmtree(os.path.join(uncommitted_dir, "data_docs"))
config_var_path = os.path.join(uncommitted_dir, "config_variables.yml")
os.remove(config_var_path)
# sanity check
assert not os.path.isfile(config_var_path)
try:
result = runner.invoke(cli, ["init"], input="Y\n")
# check dir structure
dir_structure = gen_directory_tree_str(ge_dir)
assert dir_structure == """\
great_expectations/
.gitignore
great_expectations.yml
datasources/
expectations/
notebooks/
create_expectations.ipynb
integrate_validation_into_pipeline.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
samples/
validations/
"""
# check config_variables.yml
with open(config_var_path, 'r') as f:
obs_yml = f.read()
assert obs_yml == CONFIG_VARIABLES_TEMPLATE
# Check CLI output
obs = result.output
assert "To run locally, we need some files that are not in source control." in obs
assert "You may see new files in" in obs
assert "Let's add Great Expectations to your project, by scaffolding" not in obs
assert "open a tutorial notebook" not in obs
except:
raise
finally:
os.chdir(curdir)
def test_cli_init_on_new_project(tmp_path_factory):
try:
basedir = tmp_path_factory.mktemp("test_cli_init_diff")
basedir = str(basedir)
os.makedirs(os.path.join(basedir, "data"))
curdir = os.path.abspath(os.getcwd())
shutil.copy("./tests/test_sets/Titanic.csv",
str(os.path.join(basedir, "data/Titanic.csv")))
os.chdir(basedir)
runner = CliRunner()
result = runner.invoke(cli, ["init", "--no-view"],
input="Y\n1\n%s\n\nn\n\n" %
str(os.path.join(basedir, "data")))
print(result.output)
print("result.output length:", len(result.output))
assert len(result.output) < 10000, "CLI output is unreasonably long."
assert len(
re.findall("{", result.output)
) < 100, "CLI contains way more '{' than we would reasonably expect."
assert """Always know what to expect from your data""" in result.output
assert """Let's add Great Expectations to your project""" in result.output
assert os.path.isdir(os.path.join(basedir, "great_expectations"))
assert os.path.isfile(
os.path.join(basedir, "great_expectations/great_expectations.yml"))
config = yaml.load(
open(
os.path.join(basedir,
"great_expectations/great_expectations.yml"),
"r"))
assert config["datasources"]["data__dir"][
"class_name"] == "PandasDatasource"
print(
gen_directory_tree_str(os.path.join(basedir,
"great_expectations")))
assert gen_directory_tree_str(
os.path.join(basedir, "great_expectations")) == """\
great_expectations/
.gitignore
great_expectations.yml
datasources/
expectations/
notebooks/
pandas/
create_expectations.ipynb
validation_playground.ipynb
spark/
create_expectations.ipynb
validation_playground.ipynb
sql/
create_expectations.ipynb
validation_playground.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
samples/
validations/
"""
except:
raise
finally:
os.chdir(curdir)
def test_render_full_static_site_from_empty_project(tmp_path_factory,
filesystem_csv_3):
# TODO : Use a standard test fixture
# TODO : Have that test fixture copy a directory, rather than building a new one from scratch
base_dir = str(tmp_path_factory.mktemp("project_dir"))
project_dir = os.path.join(base_dir, "project_path")
os.mkdir(project_dir)
os.makedirs(os.path.join(project_dir, "data"))
os.makedirs(os.path.join(project_dir, "data/titanic"))
shutil.copy("./tests/test_sets/Titanic.csv",
str(os.path.join(project_dir, "data/titanic/Titanic.csv")))
os.makedirs(os.path.join(project_dir, "data/random"))
shutil.copy(os.path.join(filesystem_csv_3, "f1.csv"),
str(os.path.join(project_dir, "data/random/f1.csv")))
shutil.copy(os.path.join(filesystem_csv_3, "f2.csv"),
str(os.path.join(project_dir, "data/random/f2.csv")))
assert gen_directory_tree_str(project_dir) == """\
project_path/
data/
random/
f1.csv
f2.csv
titanic/
Titanic.csv
"""
context = DataContext.create(project_dir)
ge_directory = os.path.join(project_dir, "great_expectations")
context.add_datasource("titanic",
module_name="great_expectations.datasource",
class_name="PandasDatasource",
base_directory=os.path.join(project_dir,
"data/titanic/"))
context.add_datasource("random",
module_name="great_expectations.datasource",
class_name="PandasDatasource",
base_directory=os.path.join(project_dir,
"data/random/"))
context.profile_datasource("titanic")
assert gen_directory_tree_str(project_dir) == """\
project_path/
data/
random/
f1.csv
f2.csv
titanic/
Titanic.csv
great_expectations/
.gitignore
great_expectations.yml
datasources/
expectations/
titanic/
default/
Titanic/
BasicDatasetProfiler.json
notebooks/
create_expectations.ipynb
integrate_validation_into_pipeline.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
samples/
validations/
profiling/
titanic/
default/
Titanic/
BasicDatasetProfiler.json
"""
context.profile_datasource("random")
context.build_data_docs()
data_docs_dir = os.path.join(project_dir,
"great_expectations/uncommitted/data_docs")
observed = gen_directory_tree_str(data_docs_dir)
print(observed)
assert observed == """\
data_docs/
local_site/
index.html
expectations/
random/
default/
f1/
BasicDatasetProfiler.html
f2/
BasicDatasetProfiler.html
titanic/
default/
Titanic/
BasicDatasetProfiler.html
validations/
profiling/
random/
default/
f1/
BasicDatasetProfiler.html
f2/
BasicDatasetProfiler.html
titanic/
default/
Titanic/
BasicDatasetProfiler.html
"""
# save data_docs locally
safe_mmkdir("./tests/data_context/output")
safe_mmkdir("./tests/data_context/output/data_docs")
if os.path.isdir("./tests/data_context/output/data_docs"):
shutil.rmtree("./tests/data_context/output/data_docs")
shutil.copytree(os.path.join(ge_directory, "uncommitted/data_docs/"),
"./tests/data_context/output/data_docs")
def test_v2_to_v3_project_upgrade_without_manual_steps(
v20_project_directory_with_v30_configuration_and_no_checkpoints,
caplog):
runner: CliRunner = CliRunner(mix_stderr=False)
result: Result = runner.invoke(
cli,
[
"-c",
v20_project_directory_with_v30_configuration_and_no_checkpoints,
"--v3-api",
"project",
"upgrade",
],
input="\n",
catch_exceptions=False,
)
stdout: str = result.stdout
with open(
file_relative_path(
__file__,
"../../test_fixtures/upgrade_helper/test_v2_to_v3_project_upgrade_without_manual_steps_expected_stdout.fixture",
)) as f:
expected_stdout: str = f.read()
expected_stdout = expected_stdout.replace(
"GE_PROJECT_DIR",
v20_project_directory_with_v30_configuration_and_no_checkpoints,
)
assert stdout == expected_stdout
expected_project_tree_str: str = """\
great_expectations/
.gitignore
great_expectations.yml
expectations/
.ge_store_backend_id
.gitkeep
notebooks/
.gitkeep
pandas/
validation_playground.ipynb
spark/
validation_playground.ipynb
sql/
validation_playground.ipynb
plugins/
custom_data_docs/
styles/
data_docs_custom_styles.css
uncommitted/
config_variables.yml
data_docs/
local_site/
expectations/
.gitkeep
static/
.gitkeep
validations/
diabetic_data/
warning/
20200430T191246.763896Z/
c3b4c5df224fef4b1a056a0f3b93aba5.html
logs/
project_upgrades/
UpgradeHelperV13_20210119T132639.000000Z.json
validations/
.ge_store_backend_id
diabetic_data/
warning/
20200430T191246.763896Z/
c3b4c5df224fef4b1a056a0f3b93aba5.json
"""
obs_project_tree_str: str = gen_directory_tree_str(
startpath=
v20_project_directory_with_v30_configuration_and_no_checkpoints)
assert obs_project_tree_str == expected_project_tree_str
# make sure config number incremented
assert (DataContext.get_ge_config_version(
context_root_dir=
v20_project_directory_with_v30_configuration_and_no_checkpoints) == 3.0
)
with open(
file_relative_path(
__file__,
"../../test_fixtures/upgrade_helper/UpgradeHelperV13_upgrade_without_manual_steps_log.json",
)) as f:
expected_upgrade_log_dict: dict = json.load(f)
expected_upgrade_log_str: str = json.dumps(expected_upgrade_log_dict)
expected_upgrade_log_str = expected_upgrade_log_str.replace(
"GE_PROJECT_DIR",
v20_project_directory_with_v30_configuration_and_no_checkpoints,
)
expected_upgrade_log_dict = json.loads(expected_upgrade_log_str)
with open(
f"{v20_project_directory_with_v30_configuration_and_no_checkpoints}/uncommitted/logs/project_upgrades/UpgradeHelperV13_20210119T132639.000000Z.json"
) as f:
obs_upgrade_log_dict: dict = json.load(f)
assert obs_upgrade_log_dict == expected_upgrade_log_dict
def test_cli_init_on_new_project_with_broken_excel_file_try_again_with_different_file(
mock_webbrowser, caplog, tmp_path_factory):
project_dir = str(tmp_path_factory.mktemp("test_cli_init_diff"))
os.makedirs(os.path.join(project_dir, "data"))
data_path = os.path.join(project_dir, "data", "broken_excel_file.xls")
fixture_path = file_relative_path(__file__,
"../test_sets/broken_excel_file.xls")
data_path_2 = os.path.join(project_dir, "data", "Titanic.csv")
fixture_path_2 = file_relative_path(__file__, "../test_sets/Titanic.csv")
shutil.copy(fixture_path, data_path)
shutil.copy(fixture_path_2, data_path_2)
runner = CliRunner(mix_stderr=False)
result = runner.invoke(
cli,
["init", "-d", project_dir],
input="Y\n1\n1\n{}\n\n{}\n".format(data_path,
data_path_2,
catch_exceptions=False),
)
stdout = result.output
assert mock_webbrowser.call_count == 1
assert (
"{}/great_expectations/uncommitted/data_docs/local_site/validations/Titanic/warning/"
.format(project_dir) in mock_webbrowser.call_args[0][0])
assert len(stdout) < 3000, "CLI output is unreasonably long."
assert "Always know what to expect from your data" in stdout
assert "What data would you like Great Expectations to connect to" in stdout
assert "What are you processing your files with" in stdout
assert "Enter the path (relative or absolute) of a data file" in stdout
assert "Cannot load file." in stdout
assert (
"- Please check the file and try again or select a different data file."
in stdout)
assert (
"- Error: Unsupported format, or corrupt file: Expected BOF record; found b'PRODUCTI'"
in stdout)
assert "Try again? [Y/n]:" in stdout
assert "[{}]:".format(data_path) in stdout
assert "Name the new expectation suite [Titanic.warning]" in stdout
assert (
"Great Expectations will choose a couple of columns and generate expectations about them"
in stdout)
assert "Generating example Expectation Suite..." in stdout
assert "Building" in stdout
assert "Data Docs" in stdout
assert (
"A new Expectation suite 'Titanic.warning' was added to your project"
in stdout)
assert "Great Expectations is now set up" in stdout
assert os.path.isdir(os.path.join(project_dir, "great_expectations"))
config_path = os.path.join(project_dir,
"great_expectations/great_expectations.yml")
assert os.path.isfile(config_path)
config = yaml.load(open(config_path, "r"))
data_source_class = config["datasources"]["files_datasource"][
"data_asset_type"]["class_name"]
assert data_source_class == "PandasDataset"
obs_tree = gen_directory_tree_str(
os.path.join(project_dir, "great_expectations"))
# Instead of monkey patching datetime, just regex out the time directories
date_safe_obs_tree = re.sub(r"\d*T\d*\.\d*Z", "9999.9999", obs_tree)
# Instead of monkey patching guids, just regex out the guids
guid_safe_obs_tree = re.sub(r"[a-z0-9]{32}(?=\.(json|html))",
"foobarbazguid", date_safe_obs_tree)
assert (guid_safe_obs_tree == """great_expectations/
.gitignore
great_expectations.yml
checkpoints/
expectations/
Titanic/
warning.json
notebooks/
pandas/
validation_playground.ipynb
spark/
validation_playground.ipynb
sql/
validation_playground.ipynb
plugins/
custom_data_docs/
renderers/
styles/
data_docs_custom_styles.css
views/
uncommitted/
config_variables.yml
data_docs/
local_site/
index.html
expectations/
Titanic/
warning.html
static/
fonts/
HKGrotesk/
HKGrotesk-Bold.otf
HKGrotesk-BoldItalic.otf
HKGrotesk-Italic.otf
HKGrotesk-Light.otf
HKGrotesk-LightItalic.otf
HKGrotesk-Medium.otf
HKGrotesk-MediumItalic.otf
HKGrotesk-Regular.otf
HKGrotesk-SemiBold.otf
HKGrotesk-SemiBoldItalic.otf
images/
favicon.ico
glossary_scroller.gif
iterative-dev-loop.png
logo-long-vector.svg
logo-long.png
short-logo-vector.svg
short-logo.png
validation_failed_unexpected_values.gif
styles/
data_docs_custom_styles_template.css
data_docs_default_styles.css
validations/
Titanic/
warning/
9999.9999/
foobarbazguid.html
validations/
Titanic/
warning/
9999.9999/
foobarbazguid.json
""")
assert_no_logging_messages_or_tracebacks(caplog, result)
def test_HtmlSiteStore_filesystem_backend(tmp_path_factory):
path = str(
tmp_path_factory.mktemp(
"test_HtmlSiteStore_with_TupleFileSystemStoreBackend__dir"))
my_store = HtmlSiteStore(
store_backend={
"class_name": "TupleFilesystemStoreBackend",
"base_directory": "my_store",
},
runtime_environment={"root_directory": path},
)
with pytest.raises(TypeError):
my_store.get("not_a_ValidationResultIdentifier")
with pytest.raises(ValidationError):
my_store.get(validationResultIdentifierSchema.load({}))
ns_1 = SiteSectionIdentifier(
site_section_name="validations",
resource_identifier=ValidationResultIdentifier.from_tuple((
"a",
"b",
"c",
"quarantine",
datetime.datetime.now(datetime.timezone.utc),
"prod-100",
)),
)
my_store.set(ns_1, "aaa")
# assert my_store.get(ns_1) == "aaa"
ns_2 = SiteSectionIdentifier(
site_section_name="validations",
resource_identifier=ValidationResultIdentifier.from_tuple((
"a",
"b",
"c",
"quarantine",
datetime.datetime.now(datetime.timezone.utc),
"prod-20",
)),
)
my_store.set(ns_2, "bbb")
# assert my_store.get(ns_2) == {"B": "bbb"}
print(my_store.list_keys())
# WARNING: OBSERVE THAT SITE_SECTION_NAME IS LOST IN THE CALL TO LIST_KEYS
assert set(my_store.list_keys()) == {
ns_1.resource_identifier,
ns_2.resource_identifier,
}
print(gen_directory_tree_str(path))
assert (gen_directory_tree_str(path) == """\
test_HtmlSiteStore_with_TupleFileSystemStoreBackend__dir0/
my_store/
validations/
a/
b/
c/
quarantine/
20190926T134241.000000Z/
prod-100.html
prod-20.html
""")
def test_profiler_store_integration(
empty_data_context: DataContext,
profiler_store_name: str,
profiler_name: str,
profiler_config_with_placeholder_args: RuleBasedProfilerConfig,
):
base_directory: str = str(
Path(empty_data_context.root_directory) / "profilers")
profiler_store: ProfilerStore = build_profiler_store_using_filesystem(
store_name=profiler_store_name,
base_directory=base_directory,
overwrite_existing=True,
)
dir_tree: str
dir_tree = gen_directory_tree_str(startpath=base_directory)
assert (dir_tree == """profilers/
.ge_store_backend_id
""")
key: ConfigurationIdentifier = ConfigurationIdentifier(
configuration_key=profiler_name)
profiler_store.set(key=key, value=profiler_config_with_placeholder_args)
dir_tree = gen_directory_tree_str(startpath=base_directory)
assert (dir_tree == """profilers/
.ge_store_backend_id
my_first_profiler.yml
""")
assert len(profiler_store.list_keys()) == 1
profiler_store.remove_key(key=key)
assert len(profiler_store.list_keys()) == 0
data: dict = profiler_store.self_check()
self_check_report: dict = convert_to_json_serializable(data=data)
# Drop dynamic value to ensure appropriate assert
self_check_report["config"]["store_backend"].pop("base_directory")
assert self_check_report == {
"config": {
"class_name": "ProfilerStore",
"module_name":
"great_expectations.data_context.store.profiler_store",
"overwrite_existing": True,
"store_backend": {
"class_name": "TupleFilesystemStoreBackend",
"filepath_suffix": ".yml",
"fixed_length_key": False,
"module_name":
"great_expectations.data_context.store.tuple_store_backend",
"platform_specific_separator": True,
"suppress_store_backend_id": False,
},
"store_name": "profiler_store",
},
"keys": [],
"len_keys": 0,
}
def test_upgrade_helper_intervention_on_cli_command(v10_project_directory,
caplog, monkeypatch):
# test if cli detects out of date project and asks to run upgrade helper
# decline upgrade and ensure config version was not modified
runner: CliRunner = CliRunner(mix_stderr=False)
monkeypatch.chdir(os.path.dirname(v10_project_directory))
result: Result = runner.invoke(
cli,
[
"--v3-api",
"checkpoint",
"list",
],
input="n\n",
catch_exceptions=False,
)
stdout: str = result.stdout
assert (
"Your project appears to have an out-of-date config version (1.0) - the version number must be at least 3."
in stdout)
assert "In order to proceed, your project must be upgraded." in stdout
assert (
"Would you like to run the Upgrade Helper to bring your project up-to-date? [Y/n]:"
in stdout)
assert (
"Ok, exiting now. To upgrade at a later time, use the following command: [36mgreat_expectations project "
"upgrade[0m" in stdout)
assert (
"To learn more about the upgrade process, visit ["
"36mhttps://docs.greatexpectations.io/docs/guides/miscellaneous/migration_guide#migrating-to-the-batch-request-v3-api"
in stdout)
assert_no_logging_messages_or_tracebacks(
my_caplog=caplog,
click_result=result,
)
# make sure config version unchanged
assert (DataContext.get_ge_config_version(
context_root_dir=v10_project_directory) == 1.0)
expected_project_tree_str: str = """\
great_expectations/
.gitignore
great_expectations.yml
checkpoints/
.gitkeep
expectations/
.gitkeep
notebooks/
.gitkeep
plugins/
custom_store_backends/
__init__.py
my_custom_store_backend.py
uncommitted/
config_variables.yml
data_docs/
local_site/
expectations/
.gitkeep
static/
.gitkeep
validations/
diabetic_data/
warning/
20200430T191246.763896Z/
c3b4c5df224fef4b1a056a0f3b93aba5.html
validations/
diabetic_data/
warning/
20200430T191246.763896Z/
c3b4c5df224fef4b1a056a0f3b93aba5.json
"""
obs_project_tree_str: str = gen_directory_tree_str(
startpath=v10_project_directory)
assert obs_project_tree_str == expected_project_tree_str
