def operator(regex):
"""
Operator returned by ``new_operator()`` function.
:param regex: Metric filter used to aggregate perfdatas
:type regex: str
:returns: consolidated point as float
"""
result = float('nan')
points = manager.subset_perfdata_superposed(regex, perfdatas)
if points:
timeserie = TimeSerie(
period=period,
aggregation=opname
)
consolidated = timeserie.calculate(
points=points, timewindow=timewindow, usenan=usenan
)
if consolidated:
result = consolidated[0][1]
return result
def setUp(self):
self.conf = Configuration.load(TimeSerie.CONF_PATH, Ini)
self.timeserie = TimeSerie(self.conf)
points = [
(ts, 1) for ts in range(0, 24 * 3600, 3600)
]
self.timewindow = TimeWindow(start=points[0][0], stop=points[-1][0])
self.points = points
def test_intervals(self):
"""Test calculate on different intervals."""
now = time()
# let a period of 1 day
period = Period(day=1)
oneday = period.total_seconds()
rnow = period.round_timestamp(now)
# let a timewindow of 10+1/4 days
timewindow = TimeWindow(start=now - oneday, stop=now + 45/4 * oneday)
nan = float('nan')
points = [
# the first interval is empty
(rnow, nan), # the second interval contains nan at start
(rnow + oneday + 1, nan), # the third interval contains nan at start + 1
(rnow + 2 * oneday, 1), # the fourth interval contains 1 at start
(rnow + 3 * oneday + 1, 1), # the fourth interval contains 1 at start + 1
(rnow + 4 * oneday, nan), (rnow + 4 * oneday + 1, 1), # the fith interval contains 1 and nan
(rnow + 5 * oneday, 1), (rnow + 5 * oneday + 1, 1), # the sixth interval contains 1 and 1
(rnow + 6 * oneday, 1), (rnow + 6 * oneday, 1), # the sixth interval contains 1 and 1 at the same time
(rnow + 7 * oneday, nan), (rnow + 7 * oneday, nan), # the sixth interval contains nan and nan at the same time
]
timeserie = TimeSerie(
config=self.conf,
aggregation='sum',
period=period,
round_time=True
)
_points = timeserie.calculate(points, timewindow)
for i in [0, 1, 2, 5, 8, 9, 10, 11, 12]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertTrue(isnan(_points[i][1]))
for i in [3, 4]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 1)
for i in [6, 7]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 2)
self.assertEqual(len(_points), len(points) + 1)
def perfdata(
metric_id, timewindow=None, period=None, with_meta=True,
limit=0, skip=0, timeserie=None, meta=None, sliding_time=False
):
if timewindow is not None:
timewindow = TimeWindow(**timewindow)
if timeserie is not None:
if period is None:
period = timeserie.pop('period', None)
timeserie = TimeSerie(**timeserie)
if period is not None:
timeserie.period = Period(**period)
if not isinstance(metric_id, list):
metrics = [metric_id]
else:
metrics = metric_id
result = []
for metric_id in metrics:
# meta -> _meta
pts, _meta = manager.get(
metric_id=metric_id, with_meta=True,
timewindow=timewindow, limit=limit, skip=skip,
meta=meta, sliding_time=sliding_time
)
_meta['data_id'] = metric_id
if timeserie is not None:
pts = timeserie.calculate(pts, timewindow, meta=_meta)
if with_meta:
result.append({
'points': pts,
'meta': _meta
})
else:
result.append({
'points': pts
})
return (result, len(result))
def setUp(self):
self.timeserie = TimeSerie()
points = [
(ts, 1) for ts in range(0, 24 * 3600, 3600)
]
self.timewindow = TimeWindow(start=points[0][0], stop=points[-1][0])
self.points = points
def aggregation(
self, serieconf, timewindow, period=None, usenan=True, fixed=True
):
"""
Get aggregated perfdata from serie.
:param serieconf: Serie used for aggregation
:type serieconf: dict
:param timewindow: Time window used for perfdata fetching
:type timewindow: canopsis.timeserie.timewindow.TimeWindow
:returns: aggregated perfdata classified by metric id as dict
"""
tw, period, usenan, fixed = self.get_timewindow_period_usenan_fixed(
serieconf, timewindow, period, usenan, fixed
)
timeserie = TimeSerie(
period=period,
aggregation=serieconf.get(
'aggregation_method',
TimeSerie.VDEFAULT_AGGREGATION
),
round_time=fixed
)
metrics = self.get_metrics(serieconf['metric_filter'])
perfdatas = self.get_perfdata(metrics, timewindow=tw)
for key in perfdatas:
perfdatas[key]['aggregated'] = timeserie.calculate(
points=perfdatas[key]['points'],
timewindow=tw,
usenan=usenan
)
return perfdatas
def get(self, metric_id, timewindow=None, period=None, with_meta=True, limit=0, skip=0, timeserie=None):
if timewindow is not None:
timewindow = TimeWindow(**timewindow)
if timeserie is not None:
if period is None:
period = timeserie.pop("period", None)
timeserie = TimeSerie(**timeserie)
if period is not None:
timeserie.period = Period(**period)
if not isinstance(metric_id, list):
metrics = [metric_id]
else:
metrics = metric_id
result = []
for metric_id in metrics:
pts, meta = self.manager.get(
metric_id=metric_id, with_meta=True, timewindow=timewindow, limit=limit, skip=skip
)
meta = meta[0]
if timeserie is not None:
pts = timeserie.calculate(pts, timewindow, meta=meta)
if with_meta:
result.append({"points": pts, "meta": meta})
else:
result.append({"points": pts})
return (result, len(result))
def perfdata(metric_id, timewindow=None, period=None, with_meta=True, limit=0, skip=0, timeserie=None):
if timewindow is not None:
timewindow = TimeWindow(**timewindow)
if timeserie is not None:
if period is None:
period = timeserie.pop("period", None)
timeserie = TimeSerie(**timeserie)
if period is not None:
timeserie.period = Period(**period)
if not isinstance(metric_id, list):
metrics = [metric_id]
else:
metrics = metric_id
result = []
for metric_id in metrics:
# meta -> _meta
pts, _meta = manager.get(metric_id=metric_id, with_meta=True, timewindow=timewindow, limit=limit, skip=skip)
meta = _meta[0] if _meta is not None else {}
meta[manager[PerfData.META_STORAGE].DATA_ID] = metric_id
if timeserie is not None:
pts = timeserie.calculate(pts, timewindow, meta=meta)
if with_meta:
result.append({"points": pts, "meta": meta})
else:
result.append({"points": pts})
return (result, len(result))
class TimeSerieTest(TestCase):
def setUp(self):
self.timeserie = TimeSerie()
points = [
(ts, 1) for ts in range(0, 24 * 3600, 3600)
]
self.timewindow = TimeWindow(start=points[0][0], stop=points[-1][0])
self.points = points
def _test_agg_per_x(self, period, len_points):
self.timeserie.period = period
agg_points = self.timeserie.calculate(
points=self.points, timewindow=self.timewindow
)
self.assertEqual(len(agg_points), len_points)
def test_aggregation_per_day(self):
period = Period(day=1)
self._test_agg_per_x(period, 1)
def test_aggregation_per_6hours(self):
period = Period(hour=6)
self._test_agg_per_x(period, 4)
def _five_years_timewidow(self):
now = time()
rd = relativedelta(years=5)
now = datetime.now()
past = now - rd
past_ts = mktime(past.timetuple())
now_ts = mktime(now.timetuple())
result = TimeWindow(start=past_ts, stop=now_ts)
return result
def test_intervals(self):
"""Test calculate on different intervals."""
now = time()
# let a period of 1 day
period = Period(day=1)
oneday = period.total_seconds()
rnow = period.round_timestamp(now)
# let a timewindow of 10+1/4 days
timewindow = TimeWindow(start=now - oneday, stop=now + 45/4 * oneday)
nan = float('nan')
points = [
# the first interval is empty
(rnow, nan), # the second interval contains nan at start
(rnow + oneday + 1, nan), # the third interval contains nan at start + 1
(rnow + 2 * oneday, 1), # the fourth interval contains 1 at start
(rnow + 3 * oneday + 1, 1), # the fourth interval contains 1 at start + 1
(rnow + 4 * oneday, nan), (rnow + 4 * oneday + 1, 1), # the fith interval contains 1 and nan
(rnow + 5 * oneday, 1), (rnow + 5 * oneday + 1, 1), # the sixth interval contains 1 and 1
(rnow + 6 * oneday, 1), (rnow + 6 * oneday, 1), # the sixth interval contains 1 and 1 at the same time
(rnow + 7 * oneday, nan), (rnow + 7 * oneday, nan), # the sixth interval contains nan and nan at the same time
]
timeserie = TimeSerie(
aggregation='sum',
period=period,
round_time=True
)
_points = timeserie.calculate(points, timewindow)
for i in [0, 1, 2, 5, 8, 9, 10, 11, 12]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertTrue(isnan(_points[i][1]))
for i in [3, 4]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 1)
for i in [6, 7]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 2)
self.assertEqual(len(_points), len(points) + 1)
def test_scenario(self):
"""
Calculate aggregations over 5 years
"""
timewindow = self._five_years_timewidow()
# for all round_time values
for round_time in (True, False):
unit_length = 3600
# for all units
for index, unit in enumerate(Period.UNITS):
max_value_unit = Period.MAX_UNIT_VALUES[index]
if unit in (
Period.MICROSECOND,
Period.SECOND,
Period.MINUTE,
Period.WEEK,
Period.MONTH,
Period.YEAR
):
continue
value = randint(2, max_value_unit)
period = Period(**{unit: value})
kwargs = {'period': period}
period_length = unit_length * value
timeserie = TimeSerie(round_time=round_time, **kwargs)
timesteps = timeserie.timesteps(timewindow)
timesteps_gap = timesteps[1] - timesteps[0]
self.assertEqual(timesteps_gap, period_length)
for i in range(5):
points = [
(t, random()) for t in
range(
int(timewindow.start()),
int(timewindow.stop()),
Period(**{unit: 1}).total_seconds()
)
]
aggregated_points = timeserie.calculate(points, timewindow)
len_aggregated_points = len(aggregated_points)
self.assertIn(
len(timesteps) - 1,
(
len_aggregated_points,
len_aggregated_points + 1
)
)
unit_length *= max_value_unit
def test_scenario(self):
"""
Calculate aggregations over 5 years
"""
timewindow = self._five_years_timewidow()
# for all round_time values
for round_time in (True, False):
unit_length = 3600
# for all units
for index, unit in enumerate(Period.UNITS):
max_value_unit = Period.MAX_UNIT_VALUES[index]
if unit in (
Period.MICROSECOND,
Period.SECOND,
Period.MINUTE,
Period.WEEK,
Period.MONTH,
Period.YEAR
):
continue
value = randint(2, max_value_unit)
period = Period(**{unit: value})
kwargs = {'period': period}
period_length = unit_length * value
timeserie = TimeSerie(round_time=round_time, **kwargs)
timesteps = timeserie.timesteps(timewindow)
timesteps_gap = timesteps[1] - timesteps[0]
self.assertEqual(timesteps_gap, period_length)
for i in range(5):
points = [
(t, random()) for t in
range(
int(timewindow.start()),
int(timewindow.stop()),
Period(**{unit: 1}).total_seconds()
)
]
aggregated_points = timeserie.calculate(points, timewindow)
len_aggregated_points = len(aggregated_points)
self.assertIn(
len(timesteps) - 1,
(
len_aggregated_points,
len_aggregated_points + 1
)
)
unit_length *= max_value_unit
class TimeSerieTest(unittest.TestCase):
def setUp(self):
self.conf = Configuration.load(TimeSerie.CONF_PATH, Ini)
self.timeserie = TimeSerie(self.conf)
points = [
(ts, 1) for ts in range(0, 24 * 3600, 3600)
]
self.timewindow = TimeWindow(start=points[0][0], stop=points[-1][0])
self.points = points
def _test_agg_per_x(self, period, len_points):
self.timeserie.period = period
agg_points = self.timeserie.calculate(
points=self.points, timewindow=self.timewindow
)
self.assertEqual(len(agg_points), len_points)
def test_aggregation_per_day(self):
period = Period(day=1)
self._test_agg_per_x(period, 1)
def test_aggregation_per_6hours(self):
period = Period(hour=6)
self._test_agg_per_x(period, 4)
def _five_years_timewidow(self):
now = time()
rd = relativedelta(years=5)
now = datetime.now()
past = now - rd
past_ts = mktime(past.timetuple())
now_ts = mktime(now.timetuple())
result = TimeWindow(start=past_ts, stop=now_ts)
return result
def test_intervals(self):
"""Test calculate on different intervals."""
now = time()
# let a period of 1 day
period = Period(day=1)
oneday = period.total_seconds()
rnow = period.round_timestamp(now)
# let a timewindow of 10+1/4 days
timewindow = TimeWindow(start=now - oneday, stop=now + 45/4 * oneday)
nan = float('nan')
points = [
# the first interval is empty
(rnow, nan), # the second interval contains nan at start
(rnow + oneday + 1, nan), # the third interval contains nan at start + 1
(rnow + 2 * oneday, 1), # the fourth interval contains 1 at start
(rnow + 3 * oneday + 1, 1), # the fourth interval contains 1 at start + 1
(rnow + 4 * oneday, nan), (rnow + 4 * oneday + 1, 1), # the fith interval contains 1 and nan
(rnow + 5 * oneday, 1), (rnow + 5 * oneday + 1, 1), # the sixth interval contains 1 and 1
(rnow + 6 * oneday, 1), (rnow + 6 * oneday, 1), # the sixth interval contains 1 and 1 at the same time
(rnow + 7 * oneday, nan), (rnow + 7 * oneday, nan), # the sixth interval contains nan and nan at the same time
]
timeserie = TimeSerie(
config=self.conf,
aggregation='sum',
period=period,
round_time=True
)
_points = timeserie.calculate(points, timewindow)
for i in [0, 1, 2, 5, 8, 9, 10, 11, 12]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertTrue(isnan(_points[i][1]))
for i in [3, 4]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 1)
for i in [6, 7]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 2)
self.assertEqual(len(_points), len(points) + 1)
def test_scenario(self):
"""
Calculate aggregations over 5 years
"""
timewindow = self._five_years_timewidow()
# for all round_time values
for round_time in (True, False):
unit_length = 3600
# for all units
for index, unit in enumerate(Period.UNITS):
max_value_unit = Period.MAX_UNIT_VALUES[index]
if unit in (
Period.MICROSECOND,
Period.SECOND,
Period.MINUTE,
Period.WEEK,
Period.MONTH,
Period.YEAR
):
continue
value = randint(2, max_value_unit)
period = Period(**{unit: value})
kwargs = {'period': period}
period_length = unit_length * value
timeserie = TimeSerie(config=self.conf,
round_time=round_time,
**kwargs)
timesteps = timeserie.timesteps(timewindow)
timesteps_gap = timesteps[1] - timesteps[0]
self.assertEqual(timesteps_gap, period_length)
for i in range(5):
points = [
(t, random()) for t in
range(
int(timewindow.start()),
int(timewindow.stop()),
Period(**{unit: 1}).total_seconds()
)
]
aggregated_points = timeserie.calculate(points, timewindow)
len_aggregated_points = len(aggregated_points)
self.assertIn(
len(timesteps) - 1,
(
len_aggregated_points,
len_aggregated_points + 1
)
)
unit_length *= max_value_unit
def test_timesteps(self):
timewindow = TimeWindow(start=1508420521, stop=1508506921, timezone=-120)
steps = self.timeserie.timesteps(timewindow=timewindow)
self.assertEqual(len(steps), 3)
self.assertEqual(steps[0], 1508371200)
# TODO: do more tests or rewrite perfdatas
def test__get_period(self):
timewindow = TimeWindow(start=1508420521, stop=1508506921, timezone=-120)
period = self.timeserie._get_period(timewindow=timewindow)
self.assertEqual(period.total_seconds(), 86400)
def test_scenario(self):
"""
Calculate aggregations over 5 years
"""
timewindow = self._five_years_timewidow()
# for all round_time values
for round_time in (True, False):
unit_length = 3600
# for all units
for index, unit in enumerate(Period.UNITS):
max_value_unit = Period.MAX_UNIT_VALUES[index]
if unit in (
Period.MICROSECOND,
Period.SECOND,
Period.MINUTE,
Period.WEEK,
Period.MONTH,
Period.YEAR
):
continue
value = randint(2, max_value_unit)
period = Period(**{unit: value})
kwargs = {'period': period}
period_length = unit_length * value
timeserie = TimeSerie(config=self.conf,
round_time=round_time,
**kwargs)
timesteps = timeserie.timesteps(timewindow)
timesteps_gap = timesteps[1] - timesteps[0]
self.assertEqual(timesteps_gap, period_length)
for i in range(5):
points = [
(t, random()) for t in
range(
int(timewindow.start()),
int(timewindow.stop()),
Period(**{unit: 1}).total_seconds()
)
]
aggregated_points = timeserie.calculate(points, timewindow)
len_aggregated_points = len(aggregated_points)
self.assertIn(
len(timesteps) - 1,
(
len_aggregated_points,
len_aggregated_points + 1
)
)
unit_length *= max_value_unit