예제 #1
0
    def test_round_timestamp(self):

        t = time()

        for unit in Period.UNITS:
            period = Period(**{unit: 1})
            st = period.round_timestamp(t)
            self.assertEqual(t, st)
예제 #2
0
    def test_intervals(self):
        """Test calculate on different intervals."""

        now = time()

        # let a period of 1 day
        period = Period(day=1)
        oneday = period.total_seconds()

        rnow = period.round_timestamp(now)

        # let a timewindow of 10+1/4 days
        timewindow = TimeWindow(start=now - oneday, stop=now + 45/4 * oneday)

        nan = float('nan')

        points = [
            # the first interval is empty
            (rnow, nan),  # the second interval contains nan at start
            (rnow + oneday + 1, nan),  # the third interval contains nan at start + 1
            (rnow + 2 * oneday, 1),  # the fourth interval contains 1 at start
            (rnow + 3 * oneday + 1, 1),  # the fourth interval contains 1 at start + 1
            (rnow + 4 * oneday, nan), (rnow + 4 * oneday + 1, 1),  # the fith interval contains 1 and nan
            (rnow + 5 * oneday, 1), (rnow + 5 * oneday + 1, 1),  # the sixth interval contains 1 and 1
            (rnow + 6 * oneday, 1), (rnow + 6 * oneday, 1),  # the sixth interval contains 1 and 1 at the same time
            (rnow + 7 * oneday, nan), (rnow + 7 * oneday, nan),  # the sixth interval contains nan and nan at the same time
        ]

        timeserie = TimeSerie(
            config=self.conf,
            aggregation='sum',
            period=period,
            round_time=True
        )

        _points = timeserie.calculate(points, timewindow)

        for i in [0, 1, 2, 5, 8, 9, 10, 11, 12]:
            self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
            self.assertTrue(isnan(_points[i][1]))

        for i in [3, 4]:
            self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
            self.assertEqual(_points[i][1], 1)

        for i in [6, 7]:
            self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
            self.assertEqual(_points[i][1], 2)

        self.assertEqual(len(_points), len(points) + 1)
예제 #3
0
파일: core.py 프로젝트: crudbug/canopsis
    def period(self, value):
        """Change of period."""

        if isinstance(value, basestring):
            value = Period.from_str(value)

        self._period = value
예제 #4
0
    def test_mul(self):

        p = Period(s=5, mn=10)

        p1 = p * 5

        self.assertEqual(p1.unit_values, {'s': 5 * 5, 'mn': 10 * 5})
예제 #5
0
    def test_new(self):

        args = int(36)

        period = Period.new(args)

        self.assertEqual(period.total_seconds(), args)

        args = float(args)

        period = Period.new(args)

        self.assertEqual(period.total_seconds(), args)

        period = Period.new({'second': args})

        self.assertEqual(period.total_seconds(), args)
예제 #6
0
    def test_round_datetime(self, ts=None):

        now = datetime.now() if ts is None else datetime.utcfromtimestamp(ts)

        for unitindex, unit in enumerate(Period.UNITS):

            if unit in (Period.MONTH, Period.YEAR):
                continue

            for i in range(0, 5):

                period = Period(**{unit: i})
                round_dt = period.round_datetime(now)
                round_value = getattr(round_dt, unit, None)

                unitval = getattr(now, unit, None)

                if unitval is not None:

                    if unit is Period.DAY:

                        roundday = unitval - (unitval % max(1, i))
                        if roundday <= 0:
                            month = now.month - 1

                            if month == 0:
                                month = 12

                            _, roundday = monthrange(now.year, month)

                        self.assertEqual(round_value, roundday)

                    else:
                        maxunitval = Period.MAX_UNIT_VALUES[unitindex]

                        rval = (unitval - (unitval % max(1, i))) % maxunitval

                        self.assertEqual(rval, round_value)

                    for _unit in Period.UNITS[:max(0, unitindex - 1)]:
                        _round_value = getattr(round_dt, _unit, None)

                        if _unit in (Period.MONTH, Period.DAY):
                            _round_value -= 1

                        self.assertFalse(_round_value)
예제 #7
0
    def test_new(self):

        args = int(36)

        period = Period.new(args)

        self.assertEqual(period.total_seconds(), args)

        args = float(args)

        period = Period.new(args)

        self.assertEqual(period.total_seconds(), args)

        period = Period.new({'second': args})

        self.assertEqual(period.total_seconds(), args)
예제 #8
0
    def test_round_datetime(self, ts=None):

        now = datetime.now() if ts is None else datetime.utcfromtimestamp(ts)

        for unitindex, unit in enumerate(Period.UNITS):

            if unit in (Period.MONTH, Period.YEAR):
                continue

            for i in range(0, 5):

                period = Period(**{unit: i})
                round_dt = period.round_datetime(now)
                round_value = getattr(round_dt, unit, None)

                unitval = getattr(now, unit, None)

                if unitval is not None:

                    if unit is Period.DAY:

                        roundday = unitval - (unitval % max(1, i))
                        if roundday <= 0:
                            month = now.month - 1

                            if month == 0:
                                month = 12

                            _, roundday = monthrange(now.year, month)

                        self.assertEqual(round_value, roundday)

                    else:
                        maxunitval = Period.MAX_UNIT_VALUES[unitindex]

                        rval = (unitval - (unitval % max(1, i))) % maxunitval

                        self.assertEqual(rval, round_value)

                    for _unit in Period.UNITS[:max(0, unitindex - 1)]:
                        _round_value = getattr(round_dt, _unit, None)

                        if _unit in (Period.MONTH, Period.DAY):
                            _round_value -= 1

                        self.assertFalse(_round_value)
예제 #9
0
    def _get_period(self, timewindow):
        """Get a period related to input max_points or a period."""
        result = self.period

        if result is None:
            seconds = ((timewindow.stop() - timewindow.start()) /
                       self.max_points)
            result = Period(second=seconds)

        return result
예제 #10
0
    def test_round_datetime(self):

        # get current datetime
        dt = datetime.now()

        for unit in Period.UNITS:

            for i in range(0, 5):

                period = Period(**{unit: i})

                round_dt = period.round_datetime(dt)

                value = getattr(dt, unit, None)
                if value is not None:
                    value_to_set = value + 1 if unit != Period.YEAR else 2000
                    period.unit_values[unit] = value_to_set
                    round_dt = period.round_datetime(dt)
                    round_value = getattr(round_dt, unit)

                    if round_value is not None:
                        if unit is Period.YEAR:
                            self.assertEqual(round_value, 2000)
                        elif unit is Period.DAY:
                            month = dt.month - 1
                            if month == 0:
                                month = 12
                            _, monthday = monthrange(dt.year, month)
                            self.assertEqual(round_value, monthday)
                        elif unit is Period.MONTH:
                            self.assertEqual(round_value, 12)
                        else:
                            self.assertEqual(round_value, 0)

                if Period.MICROSECOND is not unit:
                    normalized_dt = period.round_datetime(dt, normalize=True)
                    for _unit in Period.UNITS[0:Period.UNITS.index(unit) - 1]:
                        if _unit is not Period.WEEK:
                            normalized_dt_unit = getattr(normalized_dt, _unit)
                            if _unit is Period.MONTH or _unit is Period.DAY:
                                self.assertEqual(normalized_dt_unit, 1)
                            else:
                                self.assertEqual(normalized_dt_unit, 0)
예제 #11
0
    def _new_period():

        unit_values = {}

        for unit in Period.UNITS:
            unit_values[unit] = int(random() * 10)

        result = Period(**unit_values)

        return result
예제 #12
0
    def test_units_and_values(self):

        seconds = 1

        value = randint(0, 10**9 - 1)

        total_seconds = Period(microsecond=value).total_seconds()

        self.assertEqual(total_seconds, value * 10**-9)

        for index, unit in enumerate(Period.UNITS[1:-1]):

            value = randint(1, 10**10)

            total_seconds = Period(**{unit: value}).total_seconds()

            self.assertEqual(total_seconds, seconds * value,
                             '{0}:{1}:{2}'.format(value, total_seconds, unit))

            seconds *= Period.MAX_UNIT_VALUES[index + 1]
예제 #13
0
    def test_total_seconds_mix(self):
        """
        Test total seconds with all units
        """

        kwargs = {
            Period.MICROSECOND: 1,
            Period.SECOND: 1,
            Period.MINUTE: 1,
            Period.HOUR: 1,
            Period.DAY: 1,
            Period.WEEK: 1,
            Period.MONTH: 1,
            Period.YEAR: 1
        }

        period = Period(**kwargs)

        self.assertEqual(
            period.total_seconds(), 10**-9 + 1 + 60 + 3600 + 86400 +
            86400 * 7 + 86400 * 7 * 4 + 86400 * 7 * 4 * 12)
예제 #14
0
    def test_total_seconds_mix(self):
        """
        Test total seconds with all units
        """

        kwargs = {
            Period.MICROSECOND: 1,
            Period.SECOND: 1,
            Period.MINUTE: 1,
            Period.HOUR: 1,
            Period.DAY: 1,
            Period.WEEK: 1,
            Period.MONTH: 1,
            Period.YEAR: 1
        }

        period = Period(**kwargs)

        self.assertEqual(
            period.total_seconds(),
            10**-9 + 1 + 60 + 3600 + 86400 + 86400*7 + 86400*7*4 + 86400*7*4*12
        )
예제 #15
0
파일: manager.py 프로젝트: crudbug/canopsis
    def get_timewindow_period_usenan_fixed(
            serieconf, timewindow, period=None, usenan=None, fixed=None
    ):
        """Get the right timewindow, period and usenan."""

        if fixed is None:
            fixed = serieconf.get('round_time_interval', TimeSerie.VROUND_TIME)

        if period is None:
            interval = serieconf.get('aggregation_interval', TimeSerie.VPERIOD)
            period = Period.new(interval)

            if fixed:
                timewindow = timewindow.get_round_timewindow(period=period)

        if usenan is None:
            usenan = serieconf.get('usenan', True)

        result = timewindow, period, usenan, fixed

        return result
예제 #16
0
    def prepare_event(
        self,
        display_name,
        sla_measures,
        output,
        sla_state,
        alerts_percent,
        alerts_duration,
        avail_duration,
        timewindow_dict,
        now
    ):
        perf_data_array = []

        # Compute metrics to publish
        for state in self.states:

            perf_data_array.append({
                'metric': 'cps_pct_by_{}'.format(state),
                'value': round(sla_measures[state] * 100.0, 2),
                'max': 100
            })

        availability = (1.0 - alerts_percent) * 100.0
        perf_data_array.append({
            'metric': 'cps_avail',
            'value': round(availability, 2),
            'max': 100,
            SLIDING_TIME: True
        })
        perf_data_array.append({
            'metric': 'cps_avail_duration',
            'value': avail_duration,
            SLIDING_TIME: True
        })
        perf_data_array.append({
            'metric': 'cps_alerts_duration',
            'value': alerts_duration,
            SLIDING_TIME: True
        })

        period_options = {
            timewindow_dict['durationType']: timewindow_dict['value']
        }
        self.logger.debug(u'period options {}, now {}'.format(
            period_options,
            now
        ))

        period = Period(**period_options)

        periodic_timestamp = period.round_timestamp(now, next_period=True)

        self.logger.debug(u'periodic timestamp {}'.format(periodic_timestamp))

        event = forger(
            connector='sla',
            connector_name='engine',
            event_type='sla',
            source_type='resource',
            component=display_name,
            resource='sla',
            state=sla_state,
            output=output,
            perf_data_array=perf_data_array,
            display_name=display_name,
            timestamp=periodic_timestamp
        )

        self.logger.info(u'publishing sla {}, states {}'.format(
            display_name,
            sla_measures
        ))

        self.logger.debug(u'event : {}'.format(pp.pformat(event)))

        return event
예제 #17
0
    def test_aggregation_per_6hours(self):

        period = Period(hour=6)

        self._test_agg_per_x(period, 4)
예제 #18
0
# You should have received a copy of the GNU Affero General Public License
# along with Canopsis.  If not, see <http://www.gnu.org/licenses/>.
# ---------------------------------
"""Timeserie module."""

# provide only TimeSerie
__all__ = ['TimeSerie']

from math import isnan

from canopsis.confng.helpers import cfg_to_bool
from canopsis.timeserie.timewindow import Period, TimeWindow
from canopsis.timeserie.aggregation import get_aggregation, DELTA

DEFAULT_AGGREGATION = 'MEAN'
DEFAULT_PERIOD = Period(day=1)
DEFAULT_FILL = False
DEFAULT_ROUND_TIME = True
DEFAULT_MAX_POINTS = 500


class TimeSerie():
    """
    Time serie management. Contain a period and operation of aggregation,
    and a round time and a fill boolean properties.

    - period: interval of steps of aggregated points.
    - aggregation: aggregation operation name.
    - round_time: round_time of input timewindow during calculations.
    - fill: change None values by 0.
    """
예제 #19
0
    def test_aggregation_per_day(self):

        period = Period(day=1)

        self._test_agg_per_x(period, 1)
예제 #20
0
    def test_scenario(self):
        """
        Calculate aggregations over 5 years
        """

        timewindow = self._five_years_timewidow()

        # for all round_time values
        for round_time in (True, False):

            unit_length = 3600

            # for all units
            for index, unit in enumerate(Period.UNITS):

                max_value_unit = Period.MAX_UNIT_VALUES[index]

                if unit in (
                    Period.MICROSECOND,
                    Period.SECOND,
                    Period.MINUTE,
                    Period.WEEK,
                    Period.MONTH,
                    Period.YEAR
                ):
                    continue

                value = randint(2, max_value_unit)

                period = Period(**{unit: value})
                kwargs = {'period': period}
                period_length = unit_length * value

                timeserie = TimeSerie(config=self.conf,
                                      round_time=round_time,
                                      **kwargs)

                timesteps = timeserie.timesteps(timewindow)

                timesteps_gap = timesteps[1] - timesteps[0]

                self.assertEqual(timesteps_gap, period_length)

                for i in range(5):
                    points = [
                        (t, random()) for t in
                        range(
                            int(timewindow.start()),
                            int(timewindow.stop()),
                            Period(**{unit: 1}).total_seconds()
                        )
                    ]

                    aggregated_points = timeserie.calculate(points, timewindow)
                    len_aggregated_points = len(aggregated_points)
                    self.assertIn(
                        len(timesteps) - 1,
                        (
                            len_aggregated_points,
                            len_aggregated_points + 1
                        )
                    )

                unit_length *= max_value_unit
예제 #21
0
# along with Canopsis.  If not, see <http://www.gnu.org/licenses/>.
# ---------------------------------

from canopsis.mongo.core import MongoStorage
from canopsis.storage.timed import TimedStorage
from canopsis.timeserie.timewindow import Period

from md5 import new as md5

from operator import itemgetter

from datetime import datetime

from time import mktime

DEFAULT_PERIOD = Period(week=1)


class MongoTimedStorage(MongoStorage, TimedStorage):
    """MongoStorage dedicated to manage periodic data."""
    class Index:

        DATA_ID = 'i'
        TIMESTAMP = 't'
        VALUES = 'v'
        LAST_UPDATE = 'l'
        TAGS = MongoStorage.TAGS

        QUERY = [(DATA_ID, 1), (TIMESTAMP, 1), (TAGS, 1)]

    def count(self, data_id, timewindow=None, *args, **kwargs):
예제 #22
0
    def prepare_event(
        self,
        display_name,
        sla_measures,
        output,
        sla_state,
        alerts_percent,
        alerts_duration,
        avail_duration,
        timewindow_dict,
        now
    ):
        perf_data_array = []

        # Compute metrics to publish
        for state in self.states:

            perf_data_array.append({
                'metric': 'cps_pct_by_{}'.format(state),
                'value': round(sla_measures[state] * 100.0, 2),
                'max': 100
            })

        availability = (1.0 - alerts_percent) * 100.0
        perf_data_array.append({
            'metric': 'cps_avail',
            'value': round(availability, 2),
            'max': 100,
            SLIDING_TIME: True
        })
        perf_data_array.append({
            'metric': 'cps_avail_duration',
            'value': avail_duration,
            SLIDING_TIME: True
        })
        perf_data_array.append({
            'metric': 'cps_alerts_duration',
            'value': alerts_duration,
            SLIDING_TIME: True
        })

        period_options = {
            timewindow_dict['durationType']: timewindow_dict['value']
        }
        self.logger.debug(u'period options {}, now {}'.format(
            period_options,
            now
        ))

        period = Period(**period_options)

        periodic_timestamp = period.round_timestamp(now, next_period=True)

        self.logger.debug(u'periodic timestamp {}'.format(periodic_timestamp))

        event = forger(
            connector='sla',
            connector_name='engine',
            event_type='sla',
            source_type='resource',
            component=display_name,
            resource='sla',
            state=sla_state,
            output=output,
            perf_data_array=perf_data_array,
            display_name=display_name,
            timestamp=periodic_timestamp
        )

        self.logger.info(u'publishing sla {}, states {}'.format(
            display_name,
            sla_measures
        ))

        self.logger.debug(u'event : {}'.format(pp.pformat(event)))

        return event
예제 #23
0
 def period(self, value):
     if isinstance(value, basestring):
         value = Period.from_str(value)
     self._period = value