def calculate_integrity(self, df_ge: PandasDataset,
specs: SchemaParserResult) -> dict:
"""
Calculates the integrity from the defined types and the expectations.
"""
def get_unexpected(eg_result):
return eg_result[
'unexpected_count'] if 'unexpected_count' in eg_result else 0
def merge_dicts(d1, d2):
for key, value in d2.items():
for inner_value in d2[key]:
d1[key].append(inner_value)
return d1
all_elements = defaultdict(list)
invalid_elements = defaultdict(list)
for definition in specs.type_definitions:
result = df_ge.expect_column_to_exist(definition)
if not result.success:
if definition in specs.required_types: # does only count as error if required
invalid_elements[definition].append(df_ge.shape[0])
all_elements[definition].append(df_ge.shape[0])
continue
# check missing values
result = df_ge.expect_column_values_to_not_be_null(definition)
if definition in specs.required_types: # only count as error if required
invalid_elements[definition].append(
get_unexpected(result.result))
all_elements[definition].append(result.result['element_count'])
# check not correct types
type_specification = TypeSpecification.create(
specs.type_definitions.get(definition))
type_list = [t.__name__ for t in type_specification.get_types()]
# noinspection PyTypeChecker
result = df_ge.expect_column_values_to_be_in_type_list(
definition, type_list)
invalid_elements[definition].append(get_unexpected(result.result))
# handle attributes that are not specified
not_specified_fields = set(df_ge.columns) - set(specs.type_definitions)
if len(not_specified_fields) > 0:
for attribute in not_specified_fields:
result = df_ge.expect_column_values_to_be_null(attribute)
# integrity of not specified fields has been defined as 1 - so we add 0 to unexpected
invalid_elements[attribute].append(0)
all_elements[attribute].append(get_unexpected(result.result))
# check expectations
expectation_violations = self.validate_expectations(df_ge, specs)
merge_dicts(invalid_elements, expectation_violations)
# flatten attribute metrics
integrity_details = dict()
for k, v in invalid_elements.items():
integrity_details[k] = 1 - (np.sum(v) / sum(all_elements[k]))
return integrity_details
def _prepare_dataset(dataset: PandasDataset) -> PandasDataset:
dataset_copy = dataset.copy(deep=True)
for column in dataset.columns:
if dataset.expect_column_values_to_be_in_type_list(
column, type_list=sorted(list(ProfilerTypeMapping.DATETIME_TYPE_NAMES))
).success:
# GE cannot parse Timestamp or other pandas datetime time
dataset_copy[column] = dataset[column].dt.strftime("%Y-%m-%dT%H:%M:%S")
if dataset[column].dtype == np.float32:
# GE converts expectation arguments into native Python float
# This could cause error on comparison => so better to convert to double prematurely
dataset_copy[column] = dataset[column].astype(np.float64)
return dataset_copy