def add_datasource_usage_statistics(
data_context: "DataContext",
name: str,
**kwargs # noqa: F821
) -> dict:
if not data_context._usage_statistics_handler:
return {}
try:
data_context_id = data_context.data_context_id
except AttributeError:
data_context_id = None
from great_expectations.core.usage_statistics.anonymizers.datasource_anonymizer import (
DatasourceAnonymizer, )
aggregate_anonymizer = Anonymizer(salt=data_context_id)
datasource_anonymizer = DatasourceAnonymizer(
salt=data_context_id, aggregate_anonymizer=aggregate_anonymizer)
payload = {}
# noinspection PyBroadException
try:
payload = datasource_anonymizer._anonymize_datasource_info(
name, kwargs)
except Exception as e:
logger.debug(
f"{UsageStatsExceptionPrefix.EMIT_EXCEPTION.value}: {e} type: {type(e)}, add_datasource_usage_statistics: Unable to create add_datasource_usage_statistics payload field"
)
return payload
def test_anonymize_datasource_info_v2_api_core_ge_class(
datasource_anonymizer: DatasourceAnonymizer, ):
name = "test_pandas_datasource"
config = {
"name": name,
"class_name": "PandasDatasource",
"module_name": "great_expectations.datasource",
"data_asset_type": {
"module_name": "custom_pandas_dataset",
"class_name": "CustomPandasDataset",
},
"batch_kwargs_generators": {
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": "some_path",
}
},
}
anonymized_datasource = datasource_anonymizer._anonymize_datasource_info(
name=name, config=config)
assert anonymized_datasource == {
"anonymized_name": "2642802d79d90ce6d147b0f9f61c3569",
"parent_class": "PandasDatasource",
}
def test_anonymize_datasource_info_v3_api_core_ge_class(
datasource_anonymizer: DatasourceAnonymizer, ):
name = "test_pandas_datasource"
yaml_config = """
class_name: Datasource
module_name: great_expectations.datasource
execution_engine:
class_name: PandasExecutionEngine
module_name: great_expectations.execution_engine
data_connectors:
my_filesystem_data_connector:
class_name: InferredAssetFilesystemDataConnector
module_name: great_expectations.datasource.data_connector
"""
config: CommentedMap = yaml.load(yaml_config)
anonymized_datasource = datasource_anonymizer._anonymize_datasource_info(
name=name, config=config)
assert anonymized_datasource == {
"anonymized_data_connectors": [{
"anonymized_name":
"42af601aeb8a03d76bf468a462cb62f6",
"parent_class":
"InferredAssetFilesystemDataConnector",
}],
"anonymized_execution_engine": {
"anonymized_name": "6b8f8c12352592a69083f958369c7151",
"parent_class": "PandasExecutionEngine",
},
"anonymized_name":
"2642802d79d90ce6d147b0f9f61c3569",
"parent_class":
"Datasource",
}
def test_datasource_anonymizer(datasource_anonymizer: DatasourceAnonymizer):
n1 = datasource_anonymizer._anonymize_datasource_info(
name="test_datasource",
config={
"name": "test_datasource",
"class_name": "PandasDatasource",
"module_name": "great_expectations.datasource",
},
)
assert n1 == {
"anonymized_name": "04bf89e1fb7495b0904bbd5ae478fbe0",
"parent_class": "PandasDatasource",
}
n2 = datasource_anonymizer._anonymize_datasource_info(
name="test_datasource",
config={
"name": "test_datasource",
"class_name": "CustomDatasource",
"module_name": "tests.datasource.test_datasource_anonymizer",
},
)
datasource_anonymizer_2 = DatasourceAnonymizer(
aggregate_anonymizer=Anonymizer())
n3 = datasource_anonymizer_2._anonymize_datasource_info(
name="test_datasource",
config={
"name": "test_datasource",
"class_name": "CustomDatasource",
"module_name": "tests.datasource.test_datasource_anonymizer",
},
)
assert n2["parent_class"] == "PandasDatasource"
assert n3["parent_class"] == "PandasDatasource"
print(n3)
assert len(n3["anonymized_class"]) == 32
assert n2["anonymized_class"] != n3["anonymized_class"]
# Same anonymizer *does* produce the same result
n4 = datasource_anonymizer._anonymize_datasource_info(
name="test_datasource",
config={
"name": "test_datasource",
"class_name": "CustomDatasource",
"module_name": "tests.datasource.test_datasource_anonymizer",
},
)
assert n4["anonymized_class"] == n2["anonymized_class"]
def test_anonymize_datasource_info_v2_api_custom_subclass(
datasource_anonymizer: DatasourceAnonymizer, ):
"""
What does this test and why?
We should be able to discern the GE parent class for a custom type and construct
a useful usage stats event message.
Custom v2 API Datasources should continue to be supported.
"""
name = "test_pandas_datasource"
yaml_config = """
module_name: tests.data_context.fixtures.plugins.my_custom_v2_api_datasource
class_name: MyCustomV2ApiDatasource
"""
config: CommentedMap = yaml.load(yaml_config)
anonymized_datasource = datasource_anonymizer._anonymize_datasource_info(
name=name, config=config)
assert anonymized_datasource == {
"anonymized_class": "c454ace824bf401ea42815c84d0f5717",
"anonymized_name": "2642802d79d90ce6d147b0f9f61c3569",
"parent_class": "PandasDatasource",
}
def test_anonymize_datasource_info_v3_api_custom_subclass(
datasource_anonymizer: DatasourceAnonymizer, ):
name = "test_pandas_datasource"
yaml_config = """
module_name: tests.data_context.fixtures.plugins.my_custom_v3_api_datasource
class_name: MyCustomV3ApiDatasource
execution_engine:
class_name: PandasExecutionEngine
module_name: great_expectations.execution_engine
data_connectors:
my_filesystem_data_connector:
class_name: InferredAssetFilesystemDataConnector
module_name: great_expectations.datasource.data_connector
"""
config: CommentedMap = yaml.load(yaml_config)
anonymized_datasource = datasource_anonymizer._anonymize_datasource_info(
name=name, config=config)
assert anonymized_datasource == {
"anonymized_name":
"2642802d79d90ce6d147b0f9f61c3569",
"anonymized_class":
"ae74d1b58a67f5a944bb9cda16a62472",
"parent_class":
"Datasource",
"anonymized_execution_engine": {
"anonymized_name": "6b8f8c12352592a69083f958369c7151",
"parent_class": "PandasExecutionEngine",
},
"anonymized_data_connectors": [{
"anonymized_name":
"42af601aeb8a03d76bf468a462cb62f6",
"parent_class":
"InferredAssetFilesystemDataConnector",
}],
}