def test_stats(self):
outdated = Stats.insert((id, point.dt, point.sum) for point in points)
self.assertEqual(outdated, [])
with assertQueries():
point = Stats.latest(id)
sample = id, point.dt, point.sum
self.assertEqual(Stats.insert([sample]), [sample])
self.assertEqual(point.timestamp % 10, 0)
self.assertGreater(point, points[-2])
self.assertLessEqual(point, points[-1])
count = len(points) + 1
for stat in Stats[:-1]:
timestamps = map(
operator.attrgetter('timestamp'),
Stats.select(
id, point.dt -
(stat.expiration_time - timedelta(seconds=stat.step)),
point.dt))
self.assertLess(len(timestamps), count)
count = len(timestamps)
steps = set(y - x for x, y in zip(timestamps, timestamps[1:]))
self.assertLessEqual(len(steps), 1)
timestamps = map(
operator.attrgetter('timestamp'),
Stats.select(id,
point.dt - timedelta(hours=1),
point.dt,
maxlen=100))
self.assertLessEqual(len(timestamps), 100)
self.assertFalse(any(timestamp % 60 for timestamp in timestamps))
self.assertTrue(any(timestamp % 300 for timestamp in timestamps))
for point in Stats.select(id,
point.dt - timedelta(hours=1),
point.dt,
rate=True):
self.assertEqual(point[1:], (1.0, 10))
selection = list(
Stats.select(id,
now - timedelta(seconds=30),
now + timedelta(seconds=30),
fixed=3))
self.assertEqual(len(selection), 3)
# This result of this can vary depending on how settings is configured, if we have only 1 day of 10 seconds then we get a different
# answer to if we have more than a day. So cope with both configurations. If Stats[0] is now then it using the 60 second roll up and
# so len = 6 (6 times 10) otherwise we get 3 (the 3 after 'now' because all dates are in the future)
if Stats[0].start(id) < now:
self.assertEqual(sum(point.len for point in selection), 3)
else:
# See HYD-3960 - This value does not always come out as 6 and so this code will fail, because there are 100
# points this case is never tested (see comments above - look for HYD-3660) but when it is run the value ends up as 4, 5 or 6
# I (Chris) don't know why it varies and when I've looked for patterns and not found any.
self.assertEqual(sum(point.len for point in selection), 6)
self.assertEqual(selection[0].len, 0)
point, = Stats.select(id, now, now + timedelta(seconds=5), fixed=1)
with assertQueries(*['DELETE'] * 5):
Stats.delete(id)
for model in Stats:
self.assertListEqual(list(model.select(id)), [])
def fetch(self,
fetch_metrics,
begin,
end,
max_points=float("inf"),
num_points=0):
"Return datetimes with dicts of field names and values."
result = collections.defaultdict(dict)
types = set()
begin = Stats[0].round(begin) # exclude points from a partial sample
end = Stats[0].round(end) # exclude points from a partial sample
for series in Series.filter(self.measured_object,
name__in=fetch_metrics):
types.add(series.type)
minimum = 0.0 if series.type == "Counter" else float("-inf")
for point in Stats.select(series.id,
begin,
end,
rate=series.type
in ("Counter", "Derive"),
maxlen=max_points,
fixed=num_points):
result[point.dt][series.name] = max(minimum, point.mean)
# if absolute and derived values are mixed, the earliest value will be incomplete
if result and types > set(["Gauge"]) and len(
result[min(result)]) < len(fetch_metrics):
del result[min(result)]
return dict(result)
def fetch_jobs(self,
metric,
begin,
end,
job,
max_points=float("inf"),
num_points=0):
"Return datetimes with dicts of field names and values."
result = collections.defaultdict(dict)
types = set()
begin = Stats[0].round(begin) # exclude points from a partial sample
end = Stats[0].round(end) # exclude points from a partial sample
series_ids = Series.filter(self.measured_object,
name__startswith="job_" +
metric).values("id")
series_ids = Stats[0].objects.filter(
id__in=series_ids, dt__gte=begin).values("id").distinct("id")
for series in Series.filter(self.measured_object, id__in=series_ids):
types.add(series.type)
for point in Stats.select(series.id,
begin,
end,
rate=True,
maxlen=max_points,
fixed=num_points):
result[point.dt][series.name.split("_", 3)[-1]] = point
assert types.issubset(Series.JOB_TYPES)
# translate job ids into metadata
metadata = dict((job_id, job_id) for points in result.values()
for job_id in points)
if job != "id":
for type in types: # there should generally be only one
metadata.update(scheduler.metadata(type, job, metadata))
for dt in result:
data = collections.defaultdict(lambda: Point.zero)
for job_id, point in result[dt].items():
data[metadata[job_id]] += point
result[dt] = dict((key, max(0.0, data[key].mean)) for key in data)
return dict(result)