def metrics_from_states(
self, properties_and_states: Dict[Property,
State]) -> Dict[Property, Metric]:
property_metric_map: Dict[Property, Metric] = {}
for prop, state in properties_and_states.items():
if isinstance(prop, Quantile):
quantile_state = state #QuantileState(quantile_property.property_identifier(), serialized_kll, quantile)
if state.sketch_type == "floats":
kll_ser = kll_floats_sketch(DEFAULT_SKETCH_SIZE)
else:
kll_ser = kll_ints_sketch(DEFAULT_SKETCH_SIZE)
main_kll = kll_ser.deserialize(
bytes.fromhex(state.serializedKll))
quantile = main_kll.get_quantiles([prop.quantile])[0]
quantile_metric = metric_from_value(quantile, prop.name,
prop.instance, prop.entity)
property_metric_map[prop] = quantile_metric
elif isinstance(prop, ApproxDistinctness):
approx_distinct_state = state #ApproxDistinctState(approx_distinct_property.property_identifier(), serialized_hll, approx_distinct_count, num_rows)
approx_distinctness = min(
approx_distinct_state.approx_distinct_count /
approx_distinct_state.num_rows, 1.00)
approx_distinct_metric = metric_from_value(
approx_distinctness, prop.name, prop.instance, prop.entity)
property_metric_map[prop] = approx_distinct_metric
elif isinstance(prop, Schema):
schema_state = state #SchemaState(schema_property.property_identifier(),schema)
schema = schema_state.schema
schema_metric = metric_from_value(schema, prop.name,
prop.instance, prop.entity)
property_metric_map[prop] = schema_metric
else:
operator = SQLOperatorFactory.create_operator(prop)
metric = operator.get_metric(state)
property_metric_map[prop] = metric
return property_metric_map
def compute_metrics(self, properties: Set[Property],
repo: MetadataRepository):
quantile_properties = [
property for property in properties
if isinstance(property, Quantile)
]
quantile_metrics: Dict[Property, Metric] = {}
for quantile_property in quantile_properties:
data_col = self.data[quantile_property.column].to_numpy()
sketch_type = ""
if self.data[quantile_property.column].dtype == np.int64:
kll = kll_ints_sketch(DEFAULT_SKETCH_SIZE)
sketch_type = "ints"
elif self.data[quantile_property.column].dtype == np.float64:
kll = kll_floats_sketch(DEFAULT_SKETCH_SIZE)
sketch_type = "floats"
else:
raise NotImplementedError(
f"Data Type {self.data[quantile_property.column].dtype} is not supported for sketches!"
)
kll.update(data_col)
quantile = kll.get_quantiles([quantile_property.quantile])[0]
serialized_kll = kll.serialize().hex() #bytes.fromhex()
quantile_state = QuantileState(
quantile_property.property_identifier(), serialized_kll,
quantile, sketch_type)
repo.register_state(quantile_state)
quantile_metric = metric_from_value(quantile,
quantile_property.name,
quantile_property.instance,
quantile_property.entity)
quantile_metrics[quantile_property] = quantile_metric
approx_distinct_properties = [
property for property in properties
if isinstance(property, ApproxDistinctness)
]
approx_distinct_metrics: Dict[Property, Metric] = {}
for approx_distinct_property in approx_distinct_properties:
data_col = self.data[approx_distinct_property.column].to_numpy()
hll = hll_sketch(DEFAULT_HLL_K, DEFAULT_HLL_TYPE)
#for v in data_col: #slow
# hll.update(v)
hll.update(data_col) #works with local fork (np.array extension)
approx_distinct_count = hll.get_estimate()
num_rows = len(data_col)
serialized_hll = hll.serialize_updatable().hex() #bytes.fromhex()
approx_distinct_state = ApproxDistinctState(
approx_distinct_property.property_identifier(), serialized_hll,
approx_distinct_count, num_rows)
repo.register_state(approx_distinct_state)
approx_distinctness = min(approx_distinct_count / num_rows, 1.00)
approx_distinct_metric = metric_from_value(
approx_distinctness, approx_distinct_property.name,
approx_distinct_property.instance,
approx_distinct_property.entity)
approx_distinct_metrics[
approx_distinct_property] = approx_distinct_metric
other_properties = [
property for property in properties
if (not isinstance(property, Quantile)
and not isinstance(property, ApproxDistinctness))
]
metrics = self.engine.compute_metrics(other_properties, repo)
metrics.update(quantile_metrics)
metrics.update(approx_distinct_metrics)
return metrics
def get_metric(self, state: StandardDeviationState) -> DoubleMetric:
return metric_from_value(state.stddev, self.property.name,
self.property.instance, self.property.entity)
def get_metric(self, state: NumMatchesAndCount) -> DoubleMetric:
return metric_from_value(state.num_matches / state.count,
self.property.name, self.property.instance,
self.property.entity)
def get_metric(self, state: SumState) -> DoubleMetric:
return metric_from_value(state.sum_value, self.property.name,
self.property.instance, self.property.entity)
def get_metric(self, state: MeanState) -> DoubleMetric:
return metric_from_value(state.total / state.count, self.property.name,
self.property.instance, self.property.entity)
def extract_metric(self, result: DataFrame, num_rows: int) -> DoubleMetric:
distinct_n = int(result[self.distinct_count][0])
unique_n = int(result[self.unique_count][0])
return metric_from_value(unique_n / distinct_n, self.property.name,
self.property.instance, self.property.entity)
def extract_metric(self, result: DataFrame, num_rows: int) -> DoubleMetric:
uniqueness_count = int(result[self.unique_count][0])
return metric_from_value(uniqueness_count / num_rows,
self.property.name, self.property.instance,
self.property.entity)
def compute_metrics(self, properties: Set[Property], repo: MetadataRepository) -> Dict[Property, Metric]:
schema_property: Schema = None
scanning_operators: List[ScanShareableOperator] = []
grouping_operator_groups: Dict[Grouping, List[GroupingShareableOperator]] = {}
metrics: Dict[Property, Metric] = {}
for property in properties:
if isinstance(property, Schema):
schema_property = property
continue
try:
operator = SQLOperatorFactory.create_operator(property)
except UnsupportedPropertyException as ex:
metrics[property] = metric_from_failure(ex, property)
continue
if isinstance(operator, ScanShareableOperator):
scanning_operators.append(operator)
elif isinstance(operator, GroupingShareableOperator):
key = Grouping(operator.get_groupings(), operator.get_num_rows(), operator.filter)
operators = grouping_operator_groups.setdefault(key,[])
operators += [operator]
else:
raise UnknownOperatorTypeException(f"Operator '{operator.__class__.__name__}' "
f"is neither Scan nor Grouping operator.")
if schema_property is not None:
schema = self.get_schema()
schema_state = SchemaState(schema_property.property_identifier(),schema)
repo.register_state(schema_state)
schema_metric = metric_from_value(
schema, schema_property.name, schema_property.instance,schema_property.entity
)
metrics[schema_property] = schema_metric
aggregations = set()
# collect basic counts for grouping operators in first scan pass
for grouping, grouping_operators in grouping_operator_groups.items():
aggregations.add(grouping.get_num_rows_aggregation())
if self.no_sharing:
for operator in scanning_operators:
aggregations = operator.get_aggregations()
if len(aggregations) > 0:
scanning_result = self.execute_and_fetch(f"SELECT {', '.join(aggregations)} FROM {self.table}")
state = operator.extract_state(scanning_result)
state = repo.register_state(state)
metric = operator.get_metric(state)
metrics[operator.get_property()] = metric
else:
for operator in scanning_operators:
aggregations = aggregations.union(operator.get_aggregations())
if len(aggregations) > 0:
scanning_result = self.execute_and_fetch(f"SELECT {', '.join(aggregations)} FROM {self.table}")
for operator in scanning_operators:
state = operator.extract_state(scanning_result)
state = repo.register_state(state)
metric = operator.get_metric(state)
metrics[operator.get_property()] = metric
for grouping, grouping_operators in grouping_operator_groups.items():
grouping_columns = list(grouping.grouping_cols)
num_rows = grouping.extract_num_rows(scanning_result)
if grouping.filter is None:
query = f"SELECT {', '.join(list(grouping_columns) + ['COUNT(*) as count'])} FROM {self.table} GROUP BY {', '.join(grouping_columns)}"
else:
query = f"SELECT {', '.join(list(grouping_columns) + ['COUNT(*) as count'])} FROM {self.table} WHERE {grouping.filter} GROUP BY {', '.join(grouping_columns)}"
frequencies_table = repo.get_frequency_table_name(grouping.identifier())
if self.is_local_state_handler(repo):
self.execute_and_store(query, frequencies_table)
state = FrequenciesAndNumRows(grouping.identifier(), frequencies_table, grouping_columns, num_rows)
else:
self.execute_and_store(query, frequencies_table, temp=True)
grouping_df = self.execute_and_fetch(f"SELECT * FROM {frequencies_table}")
state = FrequenciesAndNumRows(grouping.identifier(), frequencies_table, grouping_columns, num_rows)
state.set_df(grouping_df)
aggregations = set()
for operator in grouping_operators:
aggregations = aggregations.union(operator.get_aggregations())
grouping_result = self.execute_and_fetch(f"SELECT {', '.join(aggregations)} FROM {frequencies_table}")
for operator in grouping_operators:
metric = operator.extract_metric(grouping_result, num_rows)
metrics[operator.get_property()] = metric
state.id = operator.get_property().property_identifier()
state = repo.register_state(state)
return metrics