def test_round_timestamp(self):
t = time()
for unit in Period.UNITS:
period = Period(**{unit: 1})
st = period.round_timestamp(t)
self.assertEqual(t, st)
def test_intervals(self):
"""Test calculate on different intervals."""
now = time()
# let a period of 1 day
period = Period(day=1)
oneday = period.total_seconds()
rnow = period.round_timestamp(now)
# let a timewindow of 10+1/4 days
timewindow = TimeWindow(start=now - oneday, stop=now + 45/4 * oneday)
nan = float('nan')
points = [
# the first interval is empty
(rnow, nan), # the second interval contains nan at start
(rnow + oneday + 1, nan), # the third interval contains nan at start + 1
(rnow + 2 * oneday, 1), # the fourth interval contains 1 at start
(rnow + 3 * oneday + 1, 1), # the fourth interval contains 1 at start + 1
(rnow + 4 * oneday, nan), (rnow + 4 * oneday + 1, 1), # the fith interval contains 1 and nan
(rnow + 5 * oneday, 1), (rnow + 5 * oneday + 1, 1), # the sixth interval contains 1 and 1
(rnow + 6 * oneday, 1), (rnow + 6 * oneday, 1), # the sixth interval contains 1 and 1 at the same time
(rnow + 7 * oneday, nan), (rnow + 7 * oneday, nan), # the sixth interval contains nan and nan at the same time
]
timeserie = TimeSerie(
config=self.conf,
aggregation='sum',
period=period,
round_time=True
)
_points = timeserie.calculate(points, timewindow)
for i in [0, 1, 2, 5, 8, 9, 10, 11, 12]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertTrue(isnan(_points[i][1]))
for i in [3, 4]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 1)
for i in [6, 7]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 2)
self.assertEqual(len(_points), len(points) + 1)
def period(self, value):
"""Change of period."""
if isinstance(value, basestring):
value = Period.from_str(value)
self._period = value
def test_mul(self):
p = Period(s=5, mn=10)
p1 = p * 5
self.assertEqual(p1.unit_values, {'s': 5 * 5, 'mn': 10 * 5})
def test_new(self):
args = int(36)
period = Period.new(args)
self.assertEqual(period.total_seconds(), args)
args = float(args)
period = Period.new(args)
self.assertEqual(period.total_seconds(), args)
period = Period.new({'second': args})
self.assertEqual(period.total_seconds(), args)
def test_round_datetime(self, ts=None):
now = datetime.now() if ts is None else datetime.utcfromtimestamp(ts)
for unitindex, unit in enumerate(Period.UNITS):
if unit in (Period.MONTH, Period.YEAR):
continue
for i in range(0, 5):
period = Period(**{unit: i})
round_dt = period.round_datetime(now)
round_value = getattr(round_dt, unit, None)
unitval = getattr(now, unit, None)
if unitval is not None:
if unit is Period.DAY:
roundday = unitval - (unitval % max(1, i))
if roundday <= 0:
month = now.month - 1
if month == 0:
month = 12
_, roundday = monthrange(now.year, month)
self.assertEqual(round_value, roundday)
else:
maxunitval = Period.MAX_UNIT_VALUES[unitindex]
rval = (unitval - (unitval % max(1, i))) % maxunitval
self.assertEqual(rval, round_value)
for _unit in Period.UNITS[:max(0, unitindex - 1)]:
_round_value = getattr(round_dt, _unit, None)
if _unit in (Period.MONTH, Period.DAY):
_round_value -= 1
self.assertFalse(_round_value)
def test_new(self):
args = int(36)
period = Period.new(args)
self.assertEqual(period.total_seconds(), args)
args = float(args)
period = Period.new(args)
self.assertEqual(period.total_seconds(), args)
period = Period.new({'second': args})
self.assertEqual(period.total_seconds(), args)
def test_round_datetime(self, ts=None):
now = datetime.now() if ts is None else datetime.utcfromtimestamp(ts)
for unitindex, unit in enumerate(Period.UNITS):
if unit in (Period.MONTH, Period.YEAR):
continue
for i in range(0, 5):
period = Period(**{unit: i})
round_dt = period.round_datetime(now)
round_value = getattr(round_dt, unit, None)
unitval = getattr(now, unit, None)
if unitval is not None:
if unit is Period.DAY:
roundday = unitval - (unitval % max(1, i))
if roundday <= 0:
month = now.month - 1
if month == 0:
month = 12
_, roundday = monthrange(now.year, month)
self.assertEqual(round_value, roundday)
else:
maxunitval = Period.MAX_UNIT_VALUES[unitindex]
rval = (unitval - (unitval % max(1, i))) % maxunitval
self.assertEqual(rval, round_value)
for _unit in Period.UNITS[:max(0, unitindex - 1)]:
_round_value = getattr(round_dt, _unit, None)
if _unit in (Period.MONTH, Period.DAY):
_round_value -= 1
self.assertFalse(_round_value)
def _get_period(self, timewindow):
"""Get a period related to input max_points or a period."""
result = self.period
if result is None:
seconds = ((timewindow.stop() - timewindow.start()) /
self.max_points)
result = Period(second=seconds)
return result
def test_round_datetime(self):
# get current datetime
dt = datetime.now()
for unit in Period.UNITS:
for i in range(0, 5):
period = Period(**{unit: i})
round_dt = period.round_datetime(dt)
value = getattr(dt, unit, None)
if value is not None:
value_to_set = value + 1 if unit != Period.YEAR else 2000
period.unit_values[unit] = value_to_set
round_dt = period.round_datetime(dt)
round_value = getattr(round_dt, unit)
if round_value is not None:
if unit is Period.YEAR:
self.assertEqual(round_value, 2000)
elif unit is Period.DAY:
month = dt.month - 1
if month == 0:
month = 12
_, monthday = monthrange(dt.year, month)
self.assertEqual(round_value, monthday)
elif unit is Period.MONTH:
self.assertEqual(round_value, 12)
else:
self.assertEqual(round_value, 0)
if Period.MICROSECOND is not unit:
normalized_dt = period.round_datetime(dt, normalize=True)
for _unit in Period.UNITS[0:Period.UNITS.index(unit) - 1]:
if _unit is not Period.WEEK:
normalized_dt_unit = getattr(normalized_dt, _unit)
if _unit is Period.MONTH or _unit is Period.DAY:
self.assertEqual(normalized_dt_unit, 1)
else:
self.assertEqual(normalized_dt_unit, 0)
def _new_period():
unit_values = {}
for unit in Period.UNITS:
unit_values[unit] = int(random() * 10)
result = Period(**unit_values)
return result
def test_units_and_values(self):
seconds = 1
value = randint(0, 10**9 - 1)
total_seconds = Period(microsecond=value).total_seconds()
self.assertEqual(total_seconds, value * 10**-9)
for index, unit in enumerate(Period.UNITS[1:-1]):
value = randint(1, 10**10)
total_seconds = Period(**{unit: value}).total_seconds()
self.assertEqual(total_seconds, seconds * value,
'{0}:{1}:{2}'.format(value, total_seconds, unit))
seconds *= Period.MAX_UNIT_VALUES[index + 1]
def test_total_seconds_mix(self):
"""
Test total seconds with all units
"""
kwargs = {
Period.MICROSECOND: 1,
Period.SECOND: 1,
Period.MINUTE: 1,
Period.HOUR: 1,
Period.DAY: 1,
Period.WEEK: 1,
Period.MONTH: 1,
Period.YEAR: 1
}
period = Period(**kwargs)
self.assertEqual(
period.total_seconds(), 10**-9 + 1 + 60 + 3600 + 86400 +
86400 * 7 + 86400 * 7 * 4 + 86400 * 7 * 4 * 12)
def test_total_seconds_mix(self):
"""
Test total seconds with all units
"""
kwargs = {
Period.MICROSECOND: 1,
Period.SECOND: 1,
Period.MINUTE: 1,
Period.HOUR: 1,
Period.DAY: 1,
Period.WEEK: 1,
Period.MONTH: 1,
Period.YEAR: 1
}
period = Period(**kwargs)
self.assertEqual(
period.total_seconds(),
10**-9 + 1 + 60 + 3600 + 86400 + 86400*7 + 86400*7*4 + 86400*7*4*12
)
def get_timewindow_period_usenan_fixed(
serieconf, timewindow, period=None, usenan=None, fixed=None
):
"""Get the right timewindow, period and usenan."""
if fixed is None:
fixed = serieconf.get('round_time_interval', TimeSerie.VROUND_TIME)
if period is None:
interval = serieconf.get('aggregation_interval', TimeSerie.VPERIOD)
period = Period.new(interval)
if fixed:
timewindow = timewindow.get_round_timewindow(period=period)
if usenan is None:
usenan = serieconf.get('usenan', True)
result = timewindow, period, usenan, fixed
return result
def prepare_event(
self,
display_name,
sla_measures,
output,
sla_state,
alerts_percent,
alerts_duration,
avail_duration,
timewindow_dict,
now
):
perf_data_array = []
# Compute metrics to publish
for state in self.states:
perf_data_array.append({
'metric': 'cps_pct_by_{}'.format(state),
'value': round(sla_measures[state] * 100.0, 2),
'max': 100
})
availability = (1.0 - alerts_percent) * 100.0
perf_data_array.append({
'metric': 'cps_avail',
'value': round(availability, 2),
'max': 100,
SLIDING_TIME: True
})
perf_data_array.append({
'metric': 'cps_avail_duration',
'value': avail_duration,
SLIDING_TIME: True
})
perf_data_array.append({
'metric': 'cps_alerts_duration',
'value': alerts_duration,
SLIDING_TIME: True
})
period_options = {
timewindow_dict['durationType']: timewindow_dict['value']
}
self.logger.debug(u'period options {}, now {}'.format(
period_options,
now
))
period = Period(**period_options)
periodic_timestamp = period.round_timestamp(now, next_period=True)
self.logger.debug(u'periodic timestamp {}'.format(periodic_timestamp))
event = forger(
connector='sla',
connector_name='engine',
event_type='sla',
source_type='resource',
component=display_name,
resource='sla',
state=sla_state,
output=output,
perf_data_array=perf_data_array,
display_name=display_name,
timestamp=periodic_timestamp
)
self.logger.info(u'publishing sla {}, states {}'.format(
display_name,
sla_measures
))
self.logger.debug(u'event : {}'.format(pp.pformat(event)))
return event
def test_aggregation_per_6hours(self):
period = Period(hour=6)
self._test_agg_per_x(period, 4)
# You should have received a copy of the GNU Affero General Public License
# along with Canopsis. If not, see <http://www.gnu.org/licenses/>.
# ---------------------------------
"""Timeserie module."""
# provide only TimeSerie
__all__ = ['TimeSerie']
from math import isnan
from canopsis.confng.helpers import cfg_to_bool
from canopsis.timeserie.timewindow import Period, TimeWindow
from canopsis.timeserie.aggregation import get_aggregation, DELTA
DEFAULT_AGGREGATION = 'MEAN'
DEFAULT_PERIOD = Period(day=1)
DEFAULT_FILL = False
DEFAULT_ROUND_TIME = True
DEFAULT_MAX_POINTS = 500
class TimeSerie():
"""
Time serie management. Contain a period and operation of aggregation,
and a round time and a fill boolean properties.
- period: interval of steps of aggregated points.
- aggregation: aggregation operation name.
- round_time: round_time of input timewindow during calculations.
- fill: change None values by 0.
"""
def test_aggregation_per_day(self):
period = Period(day=1)
self._test_agg_per_x(period, 1)
def test_scenario(self):
"""
Calculate aggregations over 5 years
"""
timewindow = self._five_years_timewidow()
# for all round_time values
for round_time in (True, False):
unit_length = 3600
# for all units
for index, unit in enumerate(Period.UNITS):
max_value_unit = Period.MAX_UNIT_VALUES[index]
if unit in (
Period.MICROSECOND,
Period.SECOND,
Period.MINUTE,
Period.WEEK,
Period.MONTH,
Period.YEAR
):
continue
value = randint(2, max_value_unit)
period = Period(**{unit: value})
kwargs = {'period': period}
period_length = unit_length * value
timeserie = TimeSerie(config=self.conf,
round_time=round_time,
**kwargs)
timesteps = timeserie.timesteps(timewindow)
timesteps_gap = timesteps[1] - timesteps[0]
self.assertEqual(timesteps_gap, period_length)
for i in range(5):
points = [
(t, random()) for t in
range(
int(timewindow.start()),
int(timewindow.stop()),
Period(**{unit: 1}).total_seconds()
)
]
aggregated_points = timeserie.calculate(points, timewindow)
len_aggregated_points = len(aggregated_points)
self.assertIn(
len(timesteps) - 1,
(
len_aggregated_points,
len_aggregated_points + 1
)
)
unit_length *= max_value_unit
# along with Canopsis. If not, see <http://www.gnu.org/licenses/>.
# ---------------------------------
from canopsis.mongo.core import MongoStorage
from canopsis.storage.timed import TimedStorage
from canopsis.timeserie.timewindow import Period
from md5 import new as md5
from operator import itemgetter
from datetime import datetime
from time import mktime
DEFAULT_PERIOD = Period(week=1)
class MongoTimedStorage(MongoStorage, TimedStorage):
"""MongoStorage dedicated to manage periodic data."""
class Index:
DATA_ID = 'i'
TIMESTAMP = 't'
VALUES = 'v'
LAST_UPDATE = 'l'
TAGS = MongoStorage.TAGS
QUERY = [(DATA_ID, 1), (TIMESTAMP, 1), (TAGS, 1)]
def count(self, data_id, timewindow=None, *args, **kwargs):
def prepare_event(
self,
display_name,
sla_measures,
output,
sla_state,
alerts_percent,
alerts_duration,
avail_duration,
timewindow_dict,
now
):
perf_data_array = []
# Compute metrics to publish
for state in self.states:
perf_data_array.append({
'metric': 'cps_pct_by_{}'.format(state),
'value': round(sla_measures[state] * 100.0, 2),
'max': 100
})
availability = (1.0 - alerts_percent) * 100.0
perf_data_array.append({
'metric': 'cps_avail',
'value': round(availability, 2),
'max': 100,
SLIDING_TIME: True
})
perf_data_array.append({
'metric': 'cps_avail_duration',
'value': avail_duration,
SLIDING_TIME: True
})
perf_data_array.append({
'metric': 'cps_alerts_duration',
'value': alerts_duration,
SLIDING_TIME: True
})
period_options = {
timewindow_dict['durationType']: timewindow_dict['value']
}
self.logger.debug(u'period options {}, now {}'.format(
period_options,
now
))
period = Period(**period_options)
periodic_timestamp = period.round_timestamp(now, next_period=True)
self.logger.debug(u'periodic timestamp {}'.format(periodic_timestamp))
event = forger(
connector='sla',
connector_name='engine',
event_type='sla',
source_type='resource',
component=display_name,
resource='sla',
state=sla_state,
output=output,
perf_data_array=perf_data_array,
display_name=display_name,
timestamp=periodic_timestamp
)
self.logger.info(u'publishing sla {}, states {}'.format(
display_name,
sla_measures
))
self.logger.debug(u'event : {}'.format(pp.pformat(event)))
return event
def period(self, value):
if isinstance(value, basestring):
value = Period.from_str(value)
self._period = value