def parse_dbms_metrics(self, metrics):
# Some DBMSs measure different types of stats (e.g., global, local)
# at different scopes (e.g. indexes, # tables, database) so for now
# we just combine them
valid_metrics = self.parse_dbms_variables(metrics)
# Extract all valid metrics
metric_catalog = {
m.name: m
for m in MetricCatalog.objects.filter(dbms__id=self.dbms_id)
}
valid_metrics, diffs = self.extract_valid_variables(valid_metrics,
metric_catalog,
default_value='0')
# Combine values
for name, values in list(valid_metrics.items()):
metric = metric_catalog[name]
if len(values) == 1 or metric.metric_type in MetricType.nonnumeric(
):
valid_metrics[name] = values[0]
elif metric.metric_type in MetricType.numeric():
conv_fn = int if metric.vartype == VarType.INTEGER else float
values = [conv_fn(v) for v in values if v is not None]
if len(values) == 0:
valid_metrics[name] = 0
else:
valid_metrics[name] = str(sum(values))
else:
raise Exception('Invalid metric type: {}'.format(
metric.metric_type))
return valid_metrics, diffs
def calculate_change_in_metrics(self,
metrics_start,
metrics_end,
fix_metric_type=True,
allow_negative=True):
metric_catalog = {
m.name: m
for m in MetricCatalog.objects.filter(dbms__id=self.dbms_id)
}
adjusted_metrics = {}
for met_name, start_val in metrics_start.items():
end_val = metrics_end[met_name]
met_info = metric_catalog[met_name]
if met_info.vartype == VarType.INTEGER or \
met_info.vartype == VarType.REAL:
conversion_fn = self.convert_integer if \
met_info.vartype == VarType.INTEGER else \
self.convert_real
start_val = conversion_fn(start_val, met_info)
end_val = conversion_fn(end_val, met_info)
if met_info.metric_type == MetricType.COUNTER:
adj_val = end_val - start_val
else: # MetricType.STATISTICS or MetricType.INFO
adj_val = end_val
if fix_metric_type:
if adj_val < 0:
adj_val = end_val
LOG.warning(
"Changing metric %s from COUNTER to STATISTICS",
met_name)
met_info.metric_type = MetricType.STATISTICS
met_info.save()
if allow_negative and adj_val < 0:
LOG.warning(
'%s metric type %s value is negative (start=%s, end=%s, diff=%s)',
met_name, MetricType.name(met_info.metric_type),
start_val, end_val, end_val - start_val)
else:
assert adj_val >= 0, \
'{} wrong metric type: {} (start={}, end={}, diff={})'.format(
met_name, MetricType.name(met_info.metric_type), start_val,
end_val, end_val - start_val)
adjusted_metrics[met_name] = adj_val
else:
# This metric is either a bool, enum, string, or timestamp
# so take last recorded value from metrics_end
adjusted_metrics[met_name] = end_val
return adjusted_metrics
def calculate_change_in_metrics(self, metrics_start, metrics_end):
adjusted_metrics = {}
for met_name, start_val in list(metrics_start.items()):
end_val = metrics_end[met_name]
met_info = self.metric_catalog_[met_name]
if met_info.vartype == VarType.INTEGER or \
met_info.vartype == VarType.REAL:
conversion_fn = self.convert_integer if \
met_info.vartype == VarType.INTEGER else \
self.convert_real
start_val = conversion_fn(start_val, met_info)
end_val = conversion_fn(end_val, met_info)
if met_info.metric_type == MetricType.COUNTER:
adj_val = end_val - start_val
else: # MetricType.STATISTICS or MetricType.INFO
adj_val = end_val
assert adj_val >= 0, \
'{} wrong metric type: {} (start={}, end={}, diff={})'.format(
met_name, MetricType.name(met_info.metric_type), start_val,
end_val, end_val - start_val)
adjusted_metrics[met_name] = adj_val
else:
# This metric is either a bool, enum, string, or timestamp
# so take last recorded value from metrics_end
adjusted_metrics[met_name] = end_val
return adjusted_metrics
def setUp(self):
dbms_obj = DBMSCatalog.objects.filter(type=DBMSType.POSTGRES,
version="9.6").first()
self.test_dbms = parser._get(dbms_obj.pk) # pylint: disable=protected-access
self.knob_catalog = KnobCatalog.objects.filter(dbms=dbms_obj)
self.tunable_knob_catalog = self.knob_catalog.filter(tunable=True)
self.metric_catalog = MetricCatalog.objects.filter(dbms=dbms_obj)
self.numeric_metric_catalog = self.metric_catalog.filter(
metric_type__in=MetricType.numeric())
def convert_dbms_metrics(self, metrics, observation_time,
target_objective):
metric_data = {}
# Same as metric_data except COUNTER metrics are not divided by the time
base_metric_data = {}
numeric_metric_catalog = MetricCatalog.objects.filter(
dbms__id=self.dbms_id, metric_type__in=MetricType.numeric())
for metadata in numeric_metric_catalog:
name = metadata.name
value = metrics[name]
if metadata.vartype == VarType.INTEGER:
converted = float(self.convert_integer(value, metadata))
elif metadata.vartype == VarType.REAL:
converted = self.convert_real(value, metadata)
else:
raise ValueError(
("Found non-numeric metric '{}' in the numeric "
"metric catalog: value={}, type={}").format(
name, value, VarType.name(metadata.vartype)))
if metadata.metric_type == MetricType.COUNTER:
assert isinstance(converted, float)
base_metric_data[name] = converted
metric_data[name] = converted / observation_time
elif metadata.metric_type == MetricType.STATISTICS:
assert isinstance(converted, float)
base_metric_data[name] = converted
metric_data[name] = converted
else:
raise ValueError('Unknown metric type for {}: {}'.format(
name, metadata.metric_type))
target_list = target_objectives.get_all(self.dbms_id)
if target_objective not in target_list:
raise ValueError(
"Invalid target objective '{}'. Expected one of: {}.".format(
target_objective, ', '.join(target_list.keys())))
for target_name, target_instance in target_list.items():
metric_data[target_name] = target_instance.compute(
base_metric_data, observation_time)
return metric_data
def convert_dbms_metrics(self, metrics, observation_time,
target_objective):
base_metric_data = {}
metric_data = {}
numeric_metric_catalog = MetricCatalog.objects.filter(
dbms__id=self.dbms_id, metric_type__in=MetricType.numeric())
for name, value in list(metrics.items()):
prt_name = self.partial_name(name)
metadata = numeric_metric_catalog.filter(name=prt_name).first()
if metadata:
if metadata.vartype == VarType.INTEGER:
converted = float(self.convert_integer(value, metadata))
elif metadata.vartype == VarType.REAL:
converted = self.convert_real(value, metadata)
else:
raise ValueError(
("Found non-numeric metric '{}' in the numeric "
"metric catalog: value={}, type={}").format(
name, value, VarType.name(metadata.vartype)))
if metadata.metric_type == MetricType.COUNTER:
assert isinstance(converted, float)
base_metric_data[name] = converted
metric_data[name] = converted / observation_time
elif metadata.metric_type == MetricType.STATISTICS:
assert isinstance(converted, float)
base_metric_data[name] = converted
metric_data[name] = converted
else:
raise ValueError('Unknown metric type for {}: {}'.format(
name, metadata.metric_type))
target_objective_instance = target_objectives.get_instance(
self.dbms_id, target_objective)
metric_data[target_objective] = target_objective_instance.compute(
base_metric_data, observation_time)
return metric_data