def test_anonymize_datasource_info_v3_api_core_ge_class():
name = "test_pandas_datasource"
yaml_config = f"""
class_name: Datasource
module_name: great_expectations.datasource
execution_engine:
class_name: PandasExecutionEngine
module_name: great_expectations.execution_engine
data_connectors:
my_filesystem_data_connector:
class_name: InferredAssetFilesystemDataConnector
module_name: great_expectations.datasource.data_connector
"""
config: CommentedMap = yaml.load(yaml_config)
datasource_anonymizer = DatasourceAnonymizer(salt=CONSISTENT_SALT)
anonymized_datasource = datasource_anonymizer.anonymize_datasource_info(
name=name, config=config)
assert anonymized_datasource == {
"anonymized_data_connectors": [{
"anonymized_name":
"42af601aeb8a03d76bf468a462cb62f6",
"parent_class":
"InferredAssetFilesystemDataConnector",
}],
"anonymized_execution_engine": {
"anonymized_name": "6b8f8c12352592a69083f958369c7151",
"parent_class": "PandasExecutionEngine",
},
"anonymized_name":
"2642802d79d90ce6d147b0f9f61c3569",
"parent_class":
"Datasource",
}
def test_anonymize_datasource_info_v2_api_core_ge_class():
name = "test_pandas_datasource"
config = {
"name": name,
"class_name": "PandasDatasource",
"module_name": "great_expectations.datasource",
"data_asset_type": {
"module_name": "custom_pandas_dataset",
"class_name": "CustomPandasDataset",
},
"batch_kwargs_generators": {
"subdir_reader": {
"class_name": "SubdirReaderBatchKwargsGenerator",
"base_directory": "some_path",
}
},
}
datasource_anonymizer = DatasourceAnonymizer(salt=CONSISTENT_SALT)
anonymized_datasource = datasource_anonymizer.anonymize_datasource_info(
name=name, config=config)
assert anonymized_datasource == {
"anonymized_name": "2642802d79d90ce6d147b0f9f61c3569",
"parent_class": "PandasDatasource",
}
def add_datasource_usage_statistics(
data_context: "DataContext",
name: str,
**kwargs # noqa: F821
) -> dict:
if not data_context._usage_statistics_handler:
return {}
try:
data_context_id = data_context.data_context_id
except AttributeError:
data_context_id = None
# noinspection PyBroadException
try:
datasource_anonymizer = (
data_context._usage_statistics_handler._datasource_anonymizer)
except Exception:
datasource_anonymizer = DatasourceAnonymizer(data_context_id)
payload = {}
# noinspection PyBroadException
try:
payload = datasource_anonymizer.anonymize_datasource_info(name, kwargs)
except Exception as e:
logger.debug(
f"{UsageStatsExceptionPrefix.EMIT_EXCEPTION.value}: {e} type: {type(e)}, add_datasource_usage_statistics: Unable to create add_datasource_usage_statistics payload field"
)
return payload
def test_datasource_anonymizer():
datasource_anonymizer = DatasourceAnonymizer(salt=CONSISTENT_SALT)
n1 = datasource_anonymizer.anonymize_datasource_info(
name="test_datasource",
config={
"name": "test_datasource",
"class_name": "PandasDatasource",
"module_name": "great_expectations.datasource",
},
)
assert n1 == {
"anonymized_name": "04bf89e1fb7495b0904bbd5ae478fbe0",
"parent_class": "PandasDatasource",
}
n2 = datasource_anonymizer.anonymize_datasource_info(
name="test_datasource",
config={
"name": "test_datasource",
"class_name": "CustomDatasource",
"module_name": "tests.datasource.test_datasource_anonymizer",
},
)
datasource_anonymizer_2 = DatasourceAnonymizer()
n3 = datasource_anonymizer_2.anonymize_datasource_info(
name="test_datasource",
config={
"name": "test_datasource",
"class_name": "CustomDatasource",
"module_name": "tests.datasource.test_datasource_anonymizer",
},
)
assert n2["parent_class"] == "PandasDatasource"
assert n3["parent_class"] == "PandasDatasource"
print(n3)
assert len(n3["anonymized_class"]) == 32
assert n2["anonymized_class"] != n3["anonymized_class"]
# Same anonymizer *does* produce the same result
n4 = datasource_anonymizer.anonymize_datasource_info(
name="test_datasource",
config={
"name": "test_datasource",
"class_name": "CustomDatasource",
"module_name": "tests.datasource.test_datasource_anonymizer",
},
)
assert n4["anonymized_class"] == n2["anonymized_class"]
def add_datasource_usage_statistics(data_context, name, **kwargs):
try:
data_context_id = data_context.data_context_id
except AttributeError:
data_context_id = None
try:
datasource_anonymizer = (
data_context._usage_statistics_handler._datasource_anonymizer)
except Exception:
datasource_anonymizer = DatasourceAnonymizer(data_context_id)
payload = {}
try:
payload = datasource_anonymizer.anonymize_datasource_info(name, kwargs)
except Exception:
logger.debug(
"add_datasource_usage_statistics: Unable to create add_datasource_usage_statistics payload field"
)
return payload
def test_anonymize_datasource_info_v2_api_custom_subclass():
"""
What does this test and why?
We should be able to discern the GE parent class for a custom type and construct
a useful usage stats event message.
Custom v2 API Datasources should continue to be supported.
"""
name = "test_pandas_datasource"
yaml_config = f"""
module_name: tests.data_context.fixtures.plugins.my_custom_v2_api_datasource
class_name: MyCustomV2ApiDatasource
"""
config: CommentedMap = yaml.load(yaml_config)
datasource_anonymizer = DatasourceAnonymizer(salt=CONSISTENT_SALT)
anonymized_datasource = datasource_anonymizer.anonymize_datasource_info(
name=name, config=config)
assert anonymized_datasource == {
"anonymized_class": "c454ace824bf401ea42815c84d0f5717",
"anonymized_name": "2642802d79d90ce6d147b0f9f61c3569",
"parent_class": "PandasDatasource",
}
class UsageStatisticsHandler(object):
def __init__(self, data_context, data_context_id, usage_statistics_url):
self._url = usage_statistics_url
self._data_context_id = data_context_id
self._data_context_instance_id = data_context.instance_id
self._data_context = data_context
self._ge_version = ge_version
self._message_queue = Queue()
self._worker = threading.Thread(target=self._requests_worker,
daemon=True)
self._worker.start()
self._datasource_anonymizer = DatasourceAnonymizer(data_context_id)
self._store_anonymizer = StoreAnonymizer(data_context_id)
self._validation_operator_anonymizer = ValidationOperatorAnonymizer(
data_context_id)
self._data_docs_sites_anonymizer = DataDocsSiteAnonymizer(
data_context_id)
self._batch_anonymizer = BatchAnonymizer(data_context_id)
self._expectation_suite_anonymizer = ExpectationSuiteAnonymizer(
data_context_id)
self._sigterm_handler = signal.signal(signal.SIGTERM, self._teardown)
self._sigint_handler = signal.signal(signal.SIGINT, self._teardown)
atexit.register(self._close_worker)
def _teardown(self, signum: int, frame):
self._close_worker()
if signum == signal.SIGTERM:
self._sigterm_handler(signum, frame)
if signum == signal.SIGINT:
self._sigint_handler(signum, frame)
def _close_worker(self):
self._message_queue.put(STOP_SIGNAL)
self._worker.join()
def _requests_worker(self):
session = requests.Session()
while True:
message = self._message_queue.get()
if message == STOP_SIGNAL:
self._message_queue.task_done()
return
try:
res = session.post(self._url, json=message, timeout=2)
logger.debug("Posted usage stats: message status " +
str(res.status_code))
if res.status_code != 201:
logger.debug("Server rejected message: ",
json.dumps(message, indent=2))
except requests.exceptions.Timeout:
logger.debug("Timeout while sending usage stats message.")
except Exception as e:
logger.debug("Unexpected error posting message: " + str(e))
finally:
self._message_queue.task_done()
def send_usage_message(self, event, event_payload=None, success=None):
"""send a usage statistics message."""
try:
message = {
"event": event,
"event_payload": event_payload or {},
"success": success,
}
self.emit(message)
except Exception:
pass
def build_init_payload(self):
"""Adds information that may be available only after full data context construction, but is useful to
calculate only one time (for example, anonymization)."""
expectation_suites = [
self._data_context.get_expectation_suite(expectation_suite_name)
for expectation_suite_name in
self._data_context.list_expectation_suite_names()
]
return {
"platform.system":
platform.system(),
"platform.release":
platform.release(),
"version_info":
str(sys.version_info),
"anonymized_datasources": [
self._datasource_anonymizer.anonymize_datasource_info(
datasource_name, datasource_config)
for datasource_name, datasource_config in self._data_context.
_project_config_with_variables_substituted.datasources.items()
],
"anonymized_stores": [
self._store_anonymizer.anonymize_store_info(
store_name, store_obj)
for store_name, store_obj in self._data_context.stores.items()
],
"anonymized_validation_operators": [
self._validation_operator_anonymizer.
anonymize_validation_operator_info(
validation_operator_name=validation_operator_name,
validation_operator_obj=validation_operator_obj)
for validation_operator_name, validation_operator_obj in
self._data_context.validation_operators.items()
],
"anonymized_data_docs_sites": [
self._data_docs_sites_anonymizer.anonymize_data_docs_site_info(
site_name=site_name, site_config=site_config)
for site_name, site_config in
self._data_context._project_config_with_variables_substituted.
data_docs_sites.items()
],
"anonymized_expectation_suites": [
self._expectation_suite_anonymizer.
anonymize_expectation_suite_info(expectation_suite)
for expectation_suite in expectation_suites
]
}
def build_envelope(self, message):
message["version"] = "1.0.0"
message["event_time"] = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
message["data_context_id"] = self._data_context_id
message["data_context_instance_id"] = self._data_context_instance_id
message["ge_version"] = self._ge_version
return message
def validate_message(self, message, schema):
try:
jsonschema.validate(message, schema=schema)
return True
except jsonschema.ValidationError as e:
logger.debug("invalid message: " + str(e))
return False
def emit(self, message):
"""
Emit a message.
"""
try:
if message["event"] == "data_context.__init__":
message["event_payload"] = self.build_init_payload()
message = self.build_envelope(message)
if not self.validate_message(
message, schema=usage_statistics_record_schema):
return
self._message_queue.put(message)
# noinspection PyBroadException
except Exception as e:
# We *always* tolerate *any* error in usage statistics
logger.debug(e)
class UsageStatisticsHandler:
def __init__(
self,
data_context: "DataContext", # noqa: F821
data_context_id: str,
usage_statistics_url: str,
):
self._url = usage_statistics_url
self._data_context_id = data_context_id
self._data_context_instance_id = data_context.instance_id
self._data_context = data_context
self._ge_version = ge_version
self._message_queue = Queue()
self._worker = threading.Thread(target=self._requests_worker,
daemon=True)
self._worker.start()
self._datasource_anonymizer = DatasourceAnonymizer(data_context_id)
self._execution_engine_anonymizer = ExecutionEngineAnonymizer(
data_context_id)
self._store_anonymizer = StoreAnonymizer(data_context_id)
self._validation_operator_anonymizer = ValidationOperatorAnonymizer(
data_context_id)
self._data_docs_sites_anonymizer = DataDocsSiteAnonymizer(
data_context_id)
self._batch_request_anonymizer = BatchRequestAnonymizer(
data_context_id)
self._batch_anonymizer = BatchAnonymizer(data_context_id)
self._expectation_suite_anonymizer = ExpectationSuiteAnonymizer(
data_context_id)
self._checkpoint_run_anonymizer = CheckpointRunAnonymizer(
data_context_id)
try:
self._sigterm_handler = signal.signal(signal.SIGTERM,
self._teardown)
except ValueError:
# if we are not the main thread, we don't get to ask for signal handling.
self._sigterm_handler = None
try:
self._sigint_handler = signal.signal(signal.SIGINT, self._teardown)
except ValueError:
# if we are not the main thread, we don't get to ask for signal handling.
self._sigint_handler = None
atexit.register(self._close_worker)
def _teardown(self, signum: int, frame: Optional[FrameType]) -> None:
self._close_worker()
if signum == signal.SIGTERM and self._sigterm_handler:
self._sigterm_handler(signum, frame)
if signum == signal.SIGINT and self._sigint_handler:
self._sigint_handler(signum, frame)
def _close_worker(self) -> None:
self._message_queue.put(STOP_SIGNAL)
self._worker.join()
def _requests_worker(self) -> None:
session = requests.Session()
while True:
message = self._message_queue.get()
if message == STOP_SIGNAL:
self._message_queue.task_done()
return
try:
res = session.post(self._url, json=message, timeout=2)
logger.debug("Posted usage stats: message status " +
str(res.status_code))
if res.status_code != 201:
logger.debug("Server rejected message: ",
json.dumps(message, indent=2))
except requests.exceptions.Timeout:
logger.debug("Timeout while sending usage stats message.")
except Exception as e:
logger.debug("Unexpected error posting message: " + str(e))
finally:
self._message_queue.task_done()
def build_init_payload(self) -> dict:
"""Adds information that may be available only after full data context construction, but is useful to
calculate only one time (for example, anonymization)."""
expectation_suites = [
self._data_context.get_expectation_suite(expectation_suite_name)
for expectation_suite_name in
self._data_context.list_expectation_suite_names()
]
return {
"platform.system":
platform.system(),
"platform.release":
platform.release(),
"version_info":
str(sys.version_info),
"anonymized_datasources": [
self._datasource_anonymizer.anonymize_datasource_info(
datasource_name, datasource_config)
for datasource_name, datasource_config in self._data_context.
project_config_with_variables_substituted.datasources.items()
],
"anonymized_stores": [
self._store_anonymizer.anonymize_store_info(
store_name, store_obj)
for store_name, store_obj in self._data_context.stores.items()
],
"anonymized_validation_operators": [
self._validation_operator_anonymizer.
anonymize_validation_operator_info(
validation_operator_name=validation_operator_name,
validation_operator_obj=validation_operator_obj,
) for validation_operator_name, validation_operator_obj in
self._data_context.validation_operators.items()
],
"anonymized_data_docs_sites": [
self._data_docs_sites_anonymizer.anonymize_data_docs_site_info(
site_name=site_name, site_config=site_config)
for site_name, site_config in
self._data_context.project_config_with_variables_substituted.
data_docs_sites.items()
],
"anonymized_expectation_suites": [
self._expectation_suite_anonymizer.
anonymize_expectation_suite_info(expectation_suite)
for expectation_suite in expectation_suites
],
}
def build_envelope(self, message: dict) -> dict:
message["version"] = "1.0.0"
message["ge_version"] = self._ge_version
message["data_context_id"] = self._data_context_id
message["data_context_instance_id"] = self._data_context_instance_id
message["event_time"] = (datetime.datetime.now(
datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z")
event_duration_property_name: str = f'{message["event"]}.duration'.replace(
".", "_")
if hasattr(self, event_duration_property_name):
delta_t: int = getattr(self, event_duration_property_name)
message["event_duration"] = delta_t
return message
@staticmethod
def validate_message(message: dict, schema: dict) -> bool:
try:
jsonschema.validate(message, schema=schema)
return True
except jsonschema.ValidationError as e:
logger.debug(
f"{UsageStatsExceptionPrefix.INVALID_MESSAGE.value} invalid message: "
+ str(e))
return False
def send_usage_message(
self,
event: str,
event_payload: Optional[dict] = None,
success: Optional[bool] = None,
) -> None:
"""send a usage statistics message."""
# noinspection PyBroadException
try:
message: dict = {
"event": event,
"event_payload": event_payload or {},
"success": success,
}
self.emit(message)
except Exception:
pass
def emit(self, message: dict) -> None:
"""
Emit a message.
"""
try:
if message["event"] == "data_context.__init__":
message["event_payload"] = self.build_init_payload()
message = self.build_envelope(message=message)
if not self.validate_message(
message, schema=anonymized_usage_statistics_record_schema):
return
self._message_queue.put(message)
# noinspection PyBroadException
except Exception as e:
# We *always* tolerate *any* error in usage statistics
log_message: str = (
f"{UsageStatsExceptionPrefix.EMIT_EXCEPTION.value}: {e} type: {type(e)}"
)
logger.debug(log_message)