def test_guid_generators():
key = builder.SchemaKey(database="test",
schema="Test",
platform="mysql",
instance="TestInstance")
guid_datahub = datahub_guid(key.dict(by_alias=True))
guid = key.guid()
assert guid == guid_datahub
def test_guid_generators():
key = builder.SchemaKey(database="test",
schema="Test",
platform="mysql",
instance="PROD")
guid_datahub = datahub_guid(key.__dict__)
guid = key.guid()
assert guid == guid_datahub
def get_dataset_partitions(self, batch_identifier, data_asset):
dataset_partitions = []
# for now, we support only v3-api and sqlalchemy execution engine
if isinstance(data_asset, Validator) and isinstance(
data_asset.execution_engine, SqlAlchemyExecutionEngine
):
ge_batch_spec = data_asset.active_batch_spec
partitionSpec = None
batchSpecProperties = {
"data_asset_name": str(
data_asset.active_batch_definition.data_asset_name
),
"datasource_name": str(
data_asset.active_batch_definition.datasource_name
),
}
if isinstance(ge_batch_spec, SqlAlchemyDatasourceBatchSpec):
# e.g. ConfiguredAssetSqlDataConnector with splitter_method or sampling_method
schema_name = ge_batch_spec.get("schema_name")
table_name = ge_batch_spec.get("table_name")
dataset_urn = make_dataset_urn_from_sqlalchemy_uri(
data_asset.execution_engine.engine.url,
schema_name,
table_name,
self.env,
self.get_platform_instance(
data_asset.active_batch_definition.datasource_name
),
)
batchSpec = BatchSpec(
nativeBatchId=batch_identifier,
customProperties=batchSpecProperties,
)
splitter_method = ge_batch_spec.get("splitter_method")
if (
splitter_method is not None
and splitter_method != "_split_on_whole_table"
):
batch_identifiers = ge_batch_spec.get("batch_identifiers", {})
partitionSpec = PartitionSpecClass(
partition=convert_to_string(batch_identifiers)
)
sampling_method = ge_batch_spec.get("sampling_method", "")
if sampling_method == "_sample_using_limit":
batchSpec.limit = ge_batch_spec["sampling_kwargs"]["n"]
dataset_partitions.append(
{
"dataset_urn": dataset_urn,
"partitionSpec": partitionSpec,
"batchSpec": batchSpec,
}
)
elif isinstance(ge_batch_spec, RuntimeQueryBatchSpec):
query = data_asset.batches[
batch_identifier
].batch_request.runtime_parameters["query"]
partitionSpec = PartitionSpecClass(
type=PartitionTypeClass.QUERY,
partition="Query_" + builder.datahub_guid(query),
)
batchSpec = BatchSpec(
nativeBatchId=batch_identifier,
query=query,
customProperties=batchSpecProperties,
)
tables = MetadataSQLSQLParser(query).get_tables()
if len(set(tables)) != 1:
warn(
"DataHubValidationAction does not support cross dataset assertions."
)
for table in tables:
dataset_urn = make_dataset_urn_from_sqlalchemy_uri(
data_asset.execution_engine.engine.url,
None,
table,
self.env,
self.get_platform_instance(
data_asset.active_batch_definition.datasource_name
),
)
dataset_partitions.append(
{
"dataset_urn": dataset_urn,
"partitionSpec": partitionSpec,
"batchSpec": batchSpec,
}
)
else:
warn(
f"DataHubValidationAction does not recognize this GE batch spec type- {type(ge_batch_spec)}."
)
else:
# TODO - v2-spec - SqlAlchemyDataset support
warn(
f"DataHubValidationAction does not recognize this GE data asset type - {type(data_asset)}. \
This is either using v2-api or execution engine other than sqlalchemy."
)
return dataset_partitions
def get_assertions_with_results(
self,
validation_result_suite,
expectation_suite_name,
run_id,
payload,
datasets,
):
dataPlatformInstance = DataPlatformInstance(
platform=builder.make_data_platform_urn(GE_PLATFORM_NAME)
)
docs_link = None
if payload:
# process the payload
for action_names in payload.keys():
if payload[action_names]["class"] == "UpdateDataDocsAction":
data_docs_pages = payload[action_names]
for docs_link_key, docs_link_val in data_docs_pages.items():
if "file://" not in docs_link_val and docs_link_key != "class":
docs_link = docs_link_val
assertions_with_results = []
for result in validation_result_suite.results:
expectation_config = result["expectation_config"]
expectation_type = expectation_config["expectation_type"]
success = True if result["success"] else False
kwargs = {
k: v for k, v in expectation_config["kwargs"].items() if k != "batch_id"
}
result = result["result"]
assertion_datasets = [d["dataset_urn"] for d in datasets]
if len(datasets) == 1 and "column" in kwargs:
assertion_fields = [
builder.make_schema_field_urn(
datasets[0]["dataset_urn"], kwargs["column"]
)
]
else:
assertion_fields = None # type:ignore
# Be careful what fields to consider for creating assertion urn.
# Any change in fields below would lead to a new assertion
# FIXME - Currently, when using evaluation parameters, new assertion is
# created when runtime resolved kwargs are different,
# possibly for each validation run
assertionUrn = builder.make_assertion_urn(
builder.datahub_guid(
{
"platform": GE_PLATFORM_NAME,
"nativeType": expectation_type,
"nativeParameters": kwargs,
"dataset": assertion_datasets[0],
"fields": assertion_fields,
}
)
)
assertionInfo: AssertionInfo = self.get_assertion_info(
expectation_type,
kwargs,
assertion_datasets[0],
assertion_fields,
expectation_suite_name,
)
# TODO: Understand why their run time is incorrect.
run_time = run_id.run_time.astimezone(timezone.utc)
assertionResults = []
evaluation_parameters = (
{
k: convert_to_string(v)
for k, v in validation_result_suite.evaluation_parameters.items()
}
if validation_result_suite.evaluation_parameters
else None
)
nativeResults = {
k: convert_to_string(v)
for k, v in result.items()
if (
k
in [
"observed_value",
"partial_unexpected_list",
"partial_unexpected_counts",
"details",
]
and v
)
}
actualAggValue = (
result.get("observed_value")
if isinstance(result.get("observed_value"), (int, float))
else None
)
ds = datasets[0]
# https://docs.greatexpectations.io/docs/reference/expectations/result_format/
assertionResult = AssertionRunEvent(
timestampMillis=int(round(time.time() * 1000)),
assertionUrn=assertionUrn,
asserteeUrn=ds["dataset_urn"],
runId=run_time.strftime("%Y-%m-%dT%H:%M:%SZ"),
result=AssertionResult(
type=AssertionResultType.SUCCESS
if success
else AssertionResultType.FAILURE,
rowCount=result.get("element_count"),
missingCount=result.get("missing_count"),
unexpectedCount=result.get("unexpected_count"),
actualAggValue=actualAggValue,
externalUrl=docs_link,
nativeResults=nativeResults,
),
batchSpec=ds["batchSpec"],
status=AssertionRunStatus.COMPLETE,
runtimeContext=evaluation_parameters,
)
if ds.get("partitionSpec") is not None:
assertionResult.partitionSpec = ds.get("partitionSpec")
assertionResults.append(assertionResult)
assertions_with_results.append(
{
"assertionUrn": assertionUrn,
"assertionInfo": assertionInfo,
"assertionPlatform": dataPlatformInstance,
"assertionResults": assertionResults,
}
)
return assertions_with_results