def add_aggregate(self, step, chunk_mapper, aggregates, metadata=None):
"""Add an aggregate at the current level.
aggregates is a list of strings containing the names of the aggregates
to compute.
"""
# XXX should add error checking to aggregates?
if metadata is None:
metadata = {}
secs = calculate_interval(step)
metadata['AGGREGATES'] = aggregates
if not metadata.has_key('LAST_UPDATE'):
metadata['LAST_UPDATE'] = 0
if not metadata.has_key('VALID_RATIO'):
metadata['VALID_RATIO'] = 0.5
if not metadata.has_key('HEARTBEAT'):
metadata['HEARTBEAT'] = 3 * secs
try:
aggset = self.get_set("TSDBAggregates")
except:
aggset = self.add_set("TSDBAggregates")
self.agg_list = []
return aggset.add_var(str(secs), Aggregate, secs, chunk_mapper, metadata)
def build_constant_rate(self, name, rate=5, step="1h",
mapper=YYYYMMDDChunkMapper, aggs=["6h"],
calc_aggs=['average','delta','min','max'],
rtype=Counter32):
"""Build up constant rate data.
This method automatically builds an aggregate equal to the step size.
Step can be expressed as a string"""
nstep = calculate_interval(step)
self.var = self.db.add_var(name, rtype, nstep, mapper)
for i in range(24):
self.var.insert(rtype(i * nstep, ROW_VALID, i * rate * nstep))
self.var.add_aggregate(step, mapper, ['average','delta'],
metadata=dict(HEARTBEAT=12*60*60))
for agg in aggs:
self.var.add_aggregate(agg, mapper, calc_aggs)
assert self.var.get_aggregate(agg).metadata['LAST_UPDATE'] == 0
self.var.update_all_aggregates()
for agg in map(lambda x: self.var.get_aggregate(x), self.var.list_aggregates()):
if agg.metadata['STEP'] == nstep:
n = 1
else:
n = 2
assert agg.metadata['LAST_UPDATE'] == 24*3600 - n * agg.metadata['STEP']
def build_constant_rate(self,
name,
rate=5,
step="1h",
mapper=YYYYMMDDChunkMapper,
aggs=["6h"],
calc_aggs=['average', 'delta', 'min', 'max'],
rtype=Counter32):
"""Build up constant rate data.
This method automatically builds an aggregate equal to the step size.
Step can be expressed as a string"""
nstep = calculate_interval(step)
self.var = self.db.add_var(name, rtype, nstep, mapper)
for i in range(24):
self.var.insert(rtype(i * nstep, ROW_VALID, i * rate * nstep))
self.var.add_aggregate(step,
mapper, ['average', 'delta'],
metadata=dict(HEARTBEAT=12 * 60 * 60))
for agg in aggs:
self.var.add_aggregate(agg, mapper, calc_aggs)
assert self.var.get_aggregate(agg).metadata['LAST_UPDATE'] == 0
self.var.update_all_aggregates()
for agg in map(lambda x: self.var.get_aggregate(x),
self.var.list_aggregates()):
if agg.metadata['STEP'] == nstep:
n = 1
else:
n = 2
assert agg.metadata[
'LAST_UPDATE'] == 24 * 3600 - n * agg.metadata['STEP']
def test_calculate_interval():
for (x,y) in (("1s", 1), ("37s", 37), ("1m", 60), ("37m", 37*60),
("1h", 60*60), ("37h", 37*60*60), ("1d", 24*60*60),
("37d", 37*24*60*60), ("1w", 7*24*60*60),
("37w", 37*7*24*60*60), ("1", 1), ("37", 37)):
assert calculate_interval(x) == y
@nose.tools.raises(InvalidInterval)
def exception_test(arg):
calculate_interval(arg)
for arg in ("-99", "99p"):
print "IntervalError:", arg, InvalidInterval
exception_test(arg)
def get_aggregate(self, name):
"""Get an existing aggregate."""
name = str(calculate_interval(name))
if not self.aggs.has_key(name):
try:
set = self.get_set("TSDBAggregates")
except TSDBSetDoesNotExistError:
raise TSDBAggregateDoesNotExistError(name)
try:
self.aggs[name] = set.get_var(name)
except TSDBVarDoesNotExistError:
raise TSDBAggregateDoesNotExistError(name)
return self.aggs[name]
def test_calculate_interval():
for (x, y) in (("1s", 1), ("37s", 37), ("1m", 60), ("37m", 37 * 60),
("1h", 60 * 60), ("37h", 37 * 60 * 60), ("1d",
24 * 60 * 60),
("37d", 37 * 24 * 60 * 60), ("1w", 7 * 24 * 60 * 60),
("37w", 37 * 7 * 24 * 60 * 60), ("1", 1), ("37", 37)):
assert calculate_interval(x) == y
@nose.tools.raises(InvalidInterval)
def exception_test(arg):
calculate_interval(arg)
for arg in ("-99", "99p"):
print "IntervalError:", arg, InvalidInterval
exception_test(arg)
def list_aggregates(self):
"""Sorted list of existing aggregates."""
if not self.agg_list:
if not TSDBSet.is_tsdb_set(self.fs, os.path.join(self.path, "TSDBAggregates")):
return [] # XXX should this raise an exception instead?
def is_aggregate(x):
return TSDBVar.is_tsdb_var(self.fs,
os.path.join(self.path, "TSDBAggregates", x))
aggs = filter(is_aggregate,
self.fs.listdir(os.path.join(self.path, "TSDBAggregates")))
weighted = [ (calculate_interval(x), x) for x in aggs ]
weighted.sort()
self.agg_list = [ x[1] for x in weighted ]
return self.agg_list
def exception_test(arg):
calculate_interval(arg)
def exception_test(arg):
calculate_interval(arg)
