def execute(self) -> MonitorMeasurement:
if not self.group_by_column_names:
start = datetime.now()
row = self.scan.warehouse.sql_fetchone(self.metric_sql)
query_milliseconds = int(((datetime.now() - start).total_seconds()) * 1000)
value = self.get_value(row)
return MonitorMeasurement(
metric=self.metric_type,
metric_id=self.metric_id,
sql=self.metric_sql,
column_name=self.column_name,
value=value,
query_milliseconds=query_milliseconds)
else:
start = datetime.now()
rows = self.scan.warehouse.sql_fetchall(self.metric_sql)
query_milliseconds = int(((datetime.now() - start).total_seconds()) * 1000)
group_values = []
for row in rows:
results = list(row)
group_columns_count = len(self.group_by_column_names)
group = results[:group_columns_count]
results = results[group_columns_count:]
value = self.get_value(results)
group_values.append(GroupValue(group=group, value=value))
return MonitorMeasurement(
metric=self.metric_type,
metric_id=self.metric_id,
sql=self.metric_sql,
column_name=self.column_name,
group_values=group_values,
query_milliseconds=query_milliseconds)
def test_column_group_measurement(self):
measurement = Measurement(
metric=Metric.MIN,
column_name='AGE',
group_values=[GroupValue(group={'country': 'US'}, value=3.4)])
self.assertEqual("min(AGE): \n group{'country': 'US'} = 3.4",
str(measurement))
JsonHelper.to_json(JsonHelper.to_jsonnable(measurement.to_dict()))
def from_dict(cls, dictionary: dict) -> 'MonitorMeasurement':
assert isinstance(dictionary, dict)
return MonitorMeasurement(
metric_id=dictionary.get('metricId'),
metric=dictionary.get('metricType'),
sql=dictionary.get('sql'),
column_name=dictionary.get('columnName'),
value=dictionary.get('value'),
group_values=GroupValue.from_json_list(dictionary.get('groupValues')),
query_milliseconds=dictionary.get('queryMilliseconds'))
def _run_sql_metric_with_groups_and_run_tests(self,
sql_metric: SqlMetricYml,
resolved_sql: str,
scan_column: Optional[ScanColumn]):
try:
measurements = []
test_results = []
group_fields_lower = set(group_field.lower() for group_field in sql_metric.group_fields)
rows, description = self.warehouse.sql_fetchall_description(resolved_sql)
self.queries_executed += 1
group_values_by_metric_name = {}
for row in rows:
group = {}
metric_values = {}
for i in range(len(row)):
metric_name = sql_metric.metric_names[i] if sql_metric.metric_names is not None else description[i][
0]
metric_value = row[i]
if metric_name.lower() in group_fields_lower:
group[metric_name] = metric_value
else:
metric_values[metric_name] = metric_value
if not group:
logging.error(f'None of the declared group_fields were found in '
f'result: {sql_metric.group_fields}. Skipping result.')
else:
for metric_name in metric_values:
metric_value = metric_values[metric_name]
if metric_name not in group_values_by_metric_name:
group_values_by_metric_name[metric_name] = []
group_values = group_values_by_metric_name[metric_name]
group_values.append(GroupValue(group=group, value=metric_value))
logging.debug(f'SQL metric {sql_metric.title} {metric_name} {group} -> {metric_value}')
sql_metric_tests = sql_metric.tests
test_variables = self._get_test_variables(scan_column)
test_variables.update(metric_values)
sql_metric_test_results = self._execute_tests(sql_metric_tests, test_variables, group)
test_results.extend(sql_metric_test_results)
column_name = scan_column.column_name if scan_column is not None else None
for metric_name in group_values_by_metric_name:
group_values = group_values_by_metric_name[metric_name]
measurement = Measurement(metric=metric_name, group_values=group_values, column_name=column_name)
self._log_and_append_query_measurement(measurements, measurement)
self._flush_measurements(measurements)
self._flush_test_results(test_results)
except Exception as e:
self.scan_result.add_error(ScanError(f'Exception during sql metric groups query {resolved_sql}', e))
def _run_sql_metric_with_groups_and_run_tests(
self,
sql_metric: SqlMetricYml,
resolved_sql: str,
column_name_lower: Optional[str] = None):
measurements = []
test_results = []
group_fields_lower = set(group_field.lower()
for group_field in sql_metric.group_fields)
rows, description = self.warehouse.sql_fetchall_description(
resolved_sql)
group_values_by_metric_name = {}
for row in rows:
group = {}
metric_values = {}
for i in range(len(row)):
metric_name = sql_metric.metric_names[
i] if sql_metric.metric_names is not None else description[
i][0]
metric_value = row[i]
if metric_name.lower() in group_fields_lower:
group[metric_name] = metric_value
else:
metric_values[metric_name] = metric_value
for metric_name in metric_values:
metric_value = metric_values[metric_name]
if metric_name not in group_values_by_metric_name:
group_values_by_metric_name[metric_name] = []
group_values = group_values_by_metric_name[metric_name]
group_values.append(GroupValue(group=group,
value=metric_value))
logging.debug(
f'SQL metric {sql_metric.name} {metric_name} {group} -> {metric_value}'
)
sql_metric_tests = sql_metric.tests
test_variables = self._get_test_variables(column_name_lower)
test_variables.update(metric_values)
sql_metric_test_results = self._execute_tests(
sql_metric_tests, test_variables, group)
test_results.extend(sql_metric_test_results)
for metric_name in group_values_by_metric_name:
group_values = group_values_by_metric_name[metric_name]
measurement = Measurement(metric=metric_name,
group_values=group_values)
self._log_and_append_query_measurement(measurements, measurement)
self._flush_measurements(measurements)
self._flush_test_results(test_results)