def test_BasicDatasetProfiler(mock_emit):
toy_dataset = PandasDataset({"x": [1, 2, 3]}, )
assert (len(
toy_dataset.get_expectation_suite(
suppress_warnings=True).expectations) == 0)
expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset)
assert (len(
toy_dataset.get_expectation_suite(suppress_warnings=True).expectations)
> 0)
assert "BasicDatasetProfiler" in expectations_config.meta
assert set(expectations_config.meta["BasicDatasetProfiler"].keys()) == {
"created_by",
"created_at",
"batch_kwargs",
}
assert "notes" in expectations_config.meta
assert set(
expectations_config.meta["notes"].keys()) == {"format", "content"}
assert "To add additional notes" in expectations_config.meta["notes"][
"content"][0]
added_expectations = set()
for exp in expectations_config.expectations:
added_expectations.add(exp.expectation_type)
assert "BasicDatasetProfiler" in exp.meta
assert "confidence" in exp.meta["BasicDatasetProfiler"]
expected_expectations = {
"expect_table_row_count_to_be_between",
"expect_table_columns_to_match_ordered_list",
"expect_column_values_to_be_in_set",
"expect_column_unique_value_count_to_be_between",
"expect_column_proportion_of_unique_values_to_be_between",
"expect_column_values_to_not_be_null",
"expect_column_values_to_be_in_type_list",
"expect_column_values_to_be_unique",
}
assert expected_expectations.issubset(added_expectations)
# Note 20211209 - Currently the only method called by the Profiler that is instrumented for usage_statistics
# is ExpectationSuite's add_expectation(). It will not send a usage_stats event when called from a Profiler.
# this number can change in the future our instrumentation changes.
assert mock_emit.call_count == 0
assert mock_emit.call_args_list == []
def test_BasicDatasetProfiler_null_column():
"""
The profiler should determine that null columns are of null cardinality and of null type and
not to generate expectations specific to types and cardinality categories.
We verify this by running the basic profiler on a Pandas dataset with an empty column
and asserting the number of successful results for the empty columns.
"""
toy_dataset = PandasDataset({"x": [1, 2, 3], "y": [None, None, None]})
assert (len(
toy_dataset.get_expectation_suite(
suppress_warnings=True).expectations) == 0)
expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset)
# TODO: assert set - specific expectations
assert (len([
result for result in evr_config["results"]
if result.expectation_config["kwargs"].get("column") == "y"
and result.success
]) == 4)
assert len([
result for result in evr_config["results"]
if result.expectation_config["kwargs"].get("column") == "y"
and result.success
]) < len([
result for result in evr_config["results"]
if result.expectation_config["kwargs"].get("column") == "x"
and result.success
])
def test_BasicDatasetProfiler_null_column():
"""
The profiler should determine that null columns are of null cardinality and of null type and
not to generate expectations specific to types and cardinality categories.
We verify this by running the basic profiler on a Pandas dataset with an empty column
and asserting the number of successful results for the empty columns.
"""
toy_dataset = PandasDataset({
"x": [1, 2, 3],
"y": [None, None, None]
},
data_asset_name="toy_dataset")
assert len(
toy_dataset.get_expectation_suite(
suppress_warnings=True)["expectations"]) == 0
expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset)
# TODO: assert set - specific expectations
assert len([
result for result in evr_config['results']
if result['expectation_config']['kwargs'].get('column') == 'y'
and result['success']
]) == 4
assert len([result for result in evr_config['results'] if
result['expectation_config']['kwargs'].get('column') == 'y' and result['success']]) < \
len([result for result in evr_config['results'] if
result['expectation_config']['kwargs'].get('column') == 'x' and result['success']])
def test_BasicDatasetProfiler():
toy_dataset = PandasDataset({"x": [1, 2, 3]},
data_asset_name="toy_dataset")
assert len(
toy_dataset.get_expectation_suite(
suppress_warnings=True)["expectations"]) == 0
expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset)
# print(json.dumps(expectations_config, indent=2))
assert len(
toy_dataset.get_expectation_suite(
suppress_warnings=True)["expectations"]) > 0
assert expectations_config["data_asset_name"] == "toy_dataset"
assert "BasicDatasetProfiler" in expectations_config["meta"]
assert set(expectations_config["meta"]["BasicDatasetProfiler"].keys()) == {
"created_by", "created_at"
}
assert "notes" in expectations_config["meta"]
assert set(
expectations_config["meta"]["notes"].keys()) == {"format", "content"}
assert "To add additional notes" in expectations_config["meta"]["notes"][
"content"][0]
added_expectations = set()
for exp in expectations_config["expectations"]:
added_expectations.add(exp["expectation_type"])
assert "BasicDatasetProfiler" in exp["meta"]
assert "confidence" in exp["meta"]["BasicDatasetProfiler"]
expected_expectations = {
'expect_table_row_count_to_be_between',
'expect_table_columns_to_match_ordered_list',
'expect_column_values_to_be_in_set',
'expect_column_unique_value_count_to_be_between',
'expect_column_proportion_of_unique_values_to_be_between',
'expect_column_values_to_not_be_null',
'expect_column_values_to_be_in_type_list',
'expect_column_values_to_be_unique'
}
assert expected_expectations.issubset(added_expectations)
def test_BasicDatasetProfiler():
toy_dataset = PandasDataset({"x": [1, 2, 3]}, )
assert (len(
toy_dataset.get_expectation_suite(
suppress_warnings=True).expectations) == 0)
expectations_config, evr_config = BasicDatasetProfiler.profile(toy_dataset)
assert (len(
toy_dataset.get_expectation_suite(suppress_warnings=True).expectations)
> 0)
assert "BasicDatasetProfiler" in expectations_config.meta
assert set(expectations_config.meta["BasicDatasetProfiler"].keys()) == {
"created_by",
"created_at",
"batch_kwargs",
}
assert "notes" in expectations_config.meta
assert set(
expectations_config.meta["notes"].keys()) == {"format", "content"}
assert "To add additional notes" in expectations_config.meta["notes"][
"content"][0]
added_expectations = set()
for exp in expectations_config.expectations:
added_expectations.add(exp.expectation_type)
assert "BasicDatasetProfiler" in exp.meta
assert "confidence" in exp.meta["BasicDatasetProfiler"]
expected_expectations = {
"expect_table_row_count_to_be_between",
"expect_table_columns_to_match_ordered_list",
"expect_column_values_to_be_in_set",
"expect_column_unique_value_count_to_be_between",
"expect_column_proportion_of_unique_values_to_be_between",
"expect_column_values_to_not_be_null",
"expect_column_values_to_be_in_type_list",
"expect_column_values_to_be_unique",
}
assert expected_expectations.issubset(added_expectations)