def test_no_startstop(self):
start, stop = 5, 10
intervals = Interval((start, stop))
timewindow = TimeWindow(intervals=intervals)
self.assertEqual(timewindow.start(), start)
self.assertEqual(timewindow.stop(), stop)
def test_no_startstop(self):
start, stop = 5, 10
intervals = Interval((start, stop))
timewindow = TimeWindow(intervals=intervals)
self.assertEqual(timewindow.start(), start)
self.assertEqual(timewindow.stop(), stop)
def test_start_stop(self):
start = random() * 10000
stop = start + random() * 10000
timewindow = TimeWindow(start=start, stop=stop)
TimeWindow()
self.assertEqual(timewindow.start(), int(start))
self.assertEqual(timewindow.stop(), int(stop))
def test_start_stop(self):
start = random() * 10000
stop = start + random() * 10000
timewindow = TimeWindow(start=start, stop=stop)
TimeWindow()
self.assertEqual(timewindow.start(), int(start))
self.assertEqual(timewindow.stop(), int(stop))
def test_CRUD(self):
data_id = 'test_store_id'
# start in droping data
self.storage.drop()
# ensure count is 0
count = self.storage.count(data_id=data_id)
self.assertEquals(count, 0)
# let's play with different data_names
meta = {'min': None, 'max': 0}
timewindow = TimeWindow()
before_timewindow = [timewindow.start() - 1000]
in_timewindow = [
timewindow.start(),
timewindow.start() + 5,
timewindow.stop() - 5,
timewindow.stop()
]
after_timewindow = [timewindow.stop() + 1000]
# set timestamps without sort
timestamps = after_timewindow + before_timewindow + in_timewindow
for timestamp in timestamps:
# add a document at starting time window
self.storage.put(data_id=data_id, value=meta, timestamp=timestamp)
# check for count equals 5
count = self.storage.count(data_id=data_id)
self.assertEquals(count, 2)
# check for_data before now
data = self.storage.get(data_ids=[data_id])
self.assertEquals(len(data[data_id]), 2)
# check for data inside timewindow and just before
data = self.storage.get(data_ids=[data_id], timewindow=timewindow)
self.assertEquals(len(data), 1)
# remove data inside timewindow
self.storage.remove(data_ids=[data_id], timewindow=timewindow)
# check for data outside timewindow
count = self.storage.count(data_id=data_id)
self.assertEquals(
count, len(before_timewindow) + len(after_timewindow))
# remove all data
self.storage.remove(data_ids=[data_id])
# check for count equals 0
count = self.storage.count(data_id=data_id)
self.assertEquals(count, 0)
def get(
self, metric_id, meta=None, with_meta=True, timewindow=None,
limit=0, skip=0, sort=None, timeserie=None, sliding_time=False
):
"""Get a set of data related to input data_id on the timewindow and
input period.
If with_meta, result is a couple of (points, list of tags by timestamp)
"""
data_id, tags = self._data_id_tags(metric_id, meta)
if sliding_time: # apply sliding_time
if timewindow is None:
timewindow = TimeWindow()
_timewindow = timewindow
timewindow = TimeWindow(
start=timewindow.start(),
stop=timewindow.stop() + SLIDING_TIME_UPPER_BOUND
)
result = self[PerfData.PERFDATA_STORAGE].get(
data_id=data_id, timewindow=timewindow, limit=limit,
skip=skip, timeserie=timeserie, tags=tags, with_tags=with_meta,
sort=sort
)
if sliding_time:
if with_meta:
points = result[0]
else:
points = result
points = [
(min(ts, _timewindow.stop()), val)
for (ts, val) in points
]
if with_meta:
result = points, result[1]
else:
result = points
return result
class TimeWindowTest(TestCase):
def setUp(self):
self.timewindow = TimeWindow()
def test_copy(self):
copy = self.timewindow.copy()
self.assertEqual(copy, self.timewindow)
def test_total_seconds(self):
self.assertEqual(
self.timewindow.total_seconds(),
TimeWindow.DEFAULT_DURATION)
def test_start_stop(self):
start = random() * 10000
stop = start + random() * 10000
timewindow = TimeWindow(start=start, stop=stop)
TimeWindow()
self.assertEqual(timewindow.start(), int(start))
self.assertEqual(timewindow.stop(), int(stop))
def test_get_datetime(self):
now = time()
dt = TimeWindow.get_datetime(now)
ts_now = mktime(dt.timetuple())
ri = randint(1, 500000)
dt = TimeWindow.get_datetime(now + ri)
self.assertEqual(ts_now + ri, mktime(dt.timetuple()))
dt = TimeWindow.get_datetime(now, timezone)
ts = mktime(dt.timetuple())
self.assertEqual(ts, ts_now + timezone)
def test_no_startstop(self):
start, stop = 5, 10
intervals = Interval((start, stop))
timewindow = TimeWindow(intervals=intervals)
self.assertEqual(timewindow.start(), start)
self.assertEqual(timewindow.stop(), stop)
class TimeWindowTest(TestCase):
def setUp(self):
self.timewindow = TimeWindow()
def test_copy(self):
copy = self.timewindow.copy()
self.assertEqual(copy, self.timewindow)
def test_total_seconds(self):
self.assertEqual(self.timewindow.total_seconds(),
TimeWindow.DEFAULT_DURATION)
def test_start_stop(self):
start = random() * 10000
stop = start + random() * 10000
timewindow = TimeWindow(start=start, stop=stop)
TimeWindow()
self.assertEqual(timewindow.start(), int(start))
self.assertEqual(timewindow.stop(), int(stop))
def test_get_datetime(self):
now = time()
dt = TimeWindow.get_datetime(now)
ts_now = mktime(dt.timetuple())
ri = randint(1, 500000)
dt = TimeWindow.get_datetime(now + ri)
self.assertEqual(ts_now + ri, mktime(dt.timetuple()))
dt = TimeWindow.get_datetime(now, timezone)
ts = mktime(dt.timetuple())
self.assertEqual(ts, ts_now + timezone)
def test_no_startstop(self):
start, stop = 5, 10
intervals = Interval((start, stop))
timewindow = TimeWindow(intervals=intervals)
self.assertEqual(timewindow.start(), start)
self.assertEqual(timewindow.stop(), stop)
def test_timesteps(self):
timewindow = TimeWindow(start=1508420521, stop=1508506921, timezone=-120)
steps = self.timeserie.timesteps(timewindow=timewindow)
self.assertEqual(len(steps), 3)
self.assertEqual(steps[0], 1508371200)
def timesteps(self, timewindow):
"""Get a list of same longer intervals inside timewindow.
The upper bound is timewindow.stop_datetime()
"""
result = []
# get the right period to apply on timewindow
period = self._get_period(timewindow=timewindow)
# set start and stop datetime
start_datetime = timewindow.start_datetime()
stop_datetime = timewindow.stop_datetime()
if self.round_time: # normalize if round time is True
start_datetime = period.round_datetime(datetime=start_datetime)
current_datetime = start_datetime
delta = period.get_delta()
while current_datetime < stop_datetime:
timestamp = TimeWindow.get_timestamp(current_datetime)
result.append(timestamp)
current_datetime += delta
result.append(timewindow.stop())
return result
def test_get_datetime(self):
now = time()
dt = TimeWindow.get_datetime(now)
ts_now = mktime(dt.timetuple())
ri = randint(1, 500000)
dt = TimeWindow.get_datetime(now + ri)
self.assertEqual(ts_now + ri, mktime(dt.timetuple()))
dt = TimeWindow.get_datetime(now, timezone)
ts = mktime(dt.timetuple())
self.assertEqual(ts, ts_now + timezone)
def test_get_datetime(self):
now = time()
dt = TimeWindow.get_datetime(now)
ts_now = mktime(dt.timetuple())
ri = randint(1, 500000)
dt = TimeWindow.get_datetime(now + ri)
self.assertEqual(ts_now + ri, mktime(dt.timetuple()))
dt = TimeWindow.get_datetime(now, timezone)
ts = mktime(dt.timetuple())
self.assertEqual(ts, ts_now + timezone)
def timesteps(self, timewindow):
"""Get a list of same longer intervals inside timewindow.
The upper bound is timewindow.stop_datetime()
"""
result = []
# get the right period to apply on timewindow
period = self._get_period(timewindow=timewindow)
# set start and stop datetime
start_datetime = timewindow.start_datetime()
stop_datetime = timewindow.stop_datetime()
if self.round_time: # normalize if round time is True
start_datetime = period.round_datetime(datetime=start_datetime)
current_datetime = start_datetime
delta = period.get_delta()
while current_datetime < stop_datetime:
timestamp = TimeWindow.get_timestamp(current_datetime)
result.append(timestamp)
current_datetime += delta
result.append(timewindow.stop())
return result
def setUp(self):
self.conf = Configuration.load(TimeSerie.CONF_PATH, Ini)
self.timeserie = TimeSerie(self.conf)
points = [
(ts, 1) for ts in range(0, 24 * 3600, 3600)
]
self.timewindow = TimeWindow(start=points[0][0], stop=points[-1][0])
self.points = points
def count_alarms_by_period(
self,
start,
stop,
subperiod=None,
limit=100,
query=None,
):
"""
Count alarms that have been opened during (stop - start) period.
:param start: Beginning timestamp of period
:type start: int
:param stop: End timestamp of period
:type stop: int
:param subperiod: Cut (stop - start) in ``subperiod`` subperiods.
:type subperiod: dict
:param limit: Counts cannot exceed this value
:type limit: int
:param query: Custom mongodb filter for alarms
:type query: dict
:return: List in which each item contains an interval and the
related count
:rtype: list
"""
if subperiod is None:
subperiod = {'day': 1}
if query is None:
query = {}
intervals = Interval.get_intervals_by_period(start, stop, subperiod)
results = []
for date in intervals:
count = self.alarm_storage.count(
data_ids=None,
timewindow=TimeWindow(start=date['begin'], stop=date['end']),
window_start_bind=True,
_filter=query,
)
results.append(
{
'date': date,
'count': limit if count > limit else count,
}
)
return results
def _five_years_timewidow(self):
now = time()
rd = relativedelta(years=5)
now = datetime.now()
past = now - rd
past_ts = mktime(past.timetuple())
now_ts = mktime(now.timetuple())
result = TimeWindow(start=past_ts, stop=now_ts)
return result
def _test_CRUD(self, storage):
# let's play with different data_names
data_ids = ['m0', 'm1']
timewindow = TimeWindow()
points = [
(timewindow.start(), None), # lower bound
(timewindow.stop(), 0), # upper bound
(timewindow.start() - 1, 1), # outside timewindow (minus 1)
(timewindow.start() + 1, 2), # inside timewindow (plus 1)
(timewindow.stop() + 1, 3) # outside timewindow (plus 1)
]
sorted_points = sorted(points, key=lambda point: point[0])
inserted_points = {}
# starts to put points for every aggregations and periods
for data_id in data_ids:
inserted_points[data_id] = points
# add documents
storage.put(data_id=data_id, points=points)
points_count_in_timewindow = len(
[point for point in points if point[0] in timewindow])
# check for reading methods
for data_id in data_ids:
# iterate on data_ids
count = storage.count(data_id=data_id)
self.assertEquals(count, len(points))
count = storage.count(data_id=data_id, timewindow=timewindow)
self.assertEquals(count, points_count_in_timewindow)
data = storage.get(data_id=data_id)
self.assertEquals(len(data), len(points))
self.assertEquals(data, sorted_points)
data = storage.get(data_id=data_id, timewindow=timewindow)
self.assertEquals(len(data), points_count_in_timewindow)
self.assertEquals(
data,
[point for point in sorted_points if point[0] in timewindow])
storage.remove(data_id=data_id, timewindow=timewindow)
# check for count equals 1
count = storage.count(data_id=data_id, timewindow=timewindow)
self.assertEquals(count, 0)
count = storage.count(data_id=data_id)
self.assertEquals(count, len(points) - points_count_in_timewindow)
storage.remove(data_id=data_id)
# check for count equals 0
count = storage.count(data_id=data_id)
self.assertEquals(count, 0)
def test_intervals(self):
"""Test calculate on different intervals."""
now = time()
# let a period of 1 day
period = Period(day=1)
oneday = period.total_seconds()
rnow = period.round_timestamp(now)
# let a timewindow of 10+1/4 days
timewindow = TimeWindow(start=now - oneday, stop=now + 45/4 * oneday)
nan = float('nan')
points = [
# the first interval is empty
(rnow, nan), # the second interval contains nan at start
(rnow + oneday + 1, nan), # the third interval contains nan at start + 1
(rnow + 2 * oneday, 1), # the fourth interval contains 1 at start
(rnow + 3 * oneday + 1, 1), # the fourth interval contains 1 at start + 1
(rnow + 4 * oneday, nan), (rnow + 4 * oneday + 1, 1), # the fith interval contains 1 and nan
(rnow + 5 * oneday, 1), (rnow + 5 * oneday + 1, 1), # the sixth interval contains 1 and 1
(rnow + 6 * oneday, 1), (rnow + 6 * oneday, 1), # the sixth interval contains 1 and 1 at the same time
(rnow + 7 * oneday, nan), (rnow + 7 * oneday, nan), # the sixth interval contains nan and nan at the same time
]
timeserie = TimeSerie(
config=self.conf,
aggregation='sum',
period=period,
round_time=True
)
_points = timeserie.calculate(points, timewindow)
for i in [0, 1, 2, 5, 8, 9, 10, 11, 12]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertTrue(isnan(_points[i][1]))
for i in [3, 4]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 1)
for i in [6, 7]:
self.assertEqual(_points[i][0], rnow + (i - 1) * oneday)
self.assertEqual(_points[i][1], 2)
self.assertEqual(len(_points), len(points) + 1)
def test__get_period(self):
timewindow = TimeWindow(start=1508420521, stop=1508506921, timezone=-120)
period = self.timeserie._get_period(timewindow=timewindow)
self.assertEqual(period.total_seconds(), 86400)
def setUp(self):
self.timewindow = TimeWindow()
def _test_CRUD(self, storage):
# let's play with different data_names
data_ids = ['m0', 'm1']
timewindow = TimeWindow()
points = [
(timewindow.start(), None), # lower bound
(timewindow.stop(), 0), # upper bound
(timewindow.start() - 1, 1), # outside timewindow (minus 1)
(timewindow.start() + 1, 2), # inside timewindow (plus 1)
(timewindow.stop() + 1, 3) # outside timewindow (plus 1)
]
sorted_points = sorted(points, key=lambda point: point[0])
inserted_points = {}
# starts to put points for every aggregations and periods
for data_id in data_ids:
inserted_points[data_id] = points
# add documents
storage.put(data_id=data_id, points=points)
points_count_in_timewindow = len(
[point for point in points if point[0] in timewindow])
# check for reading methods
for data_id in data_ids:
# iterate on data_ids
count = storage.count(data_id=data_id)
self.assertEquals(count, len(points))
count = storage.count(data_id=data_id, timewindow=timewindow)
self.assertEquals(count, points_count_in_timewindow)
data = storage.get(data_id=data_id)
self.assertEquals(len(data), len(points))
self.assertEquals(data, sorted_points)
data = storage.get(data_id=data_id, timewindow=timewindow)
self.assertEquals(len(data), points_count_in_timewindow)
self.assertEquals(
data,
[
point for point in sorted_points
if point[0] in timewindow
]
)
storage.remove(data_id=data_id, timewindow=timewindow)
# check for count equals 1
count = storage.count(data_id=data_id, timewindow=timewindow)
self.assertEquals(count, 0)
count = storage.count(data_id=data_id)
self.assertEquals(
count, len(points) - points_count_in_timewindow)
storage.remove(data_id=data_id)
# check for count equals 0
count = storage.count(data_id=data_id)
self.assertEquals(count, 0)
def setUp(self):
self.timewindow = TimeWindow()
def test_put_get_data(self):
timewindow = TimeWindow()
metric_id = 'test_manager'
self.perfdata.remove(metric_id=metric_id, with_meta=True)
count = self.perfdata.count(metric_id=metric_id)
self.assertEqual(count, 0)
tv0 = (int(timewindow.start()), None)
tv1 = (int(timewindow.start() + 1), 0)
tv2 = (int(timewindow.stop()), 2)
tv3 = (int(timewindow.stop() + 1000000), 3)
# set values with timestamp without order
points = [tv0, tv2, tv1, tv3]
meta = {'plop': None}
self.perfdata.put(
metric_id=metric_id,
points=points,
meta=meta
)
data, _meta = self.perfdata.get(
metric_id=metric_id,
timewindow=timewindow,
with_meta=True
)
self.assertEqual(meta, _meta[0][PerfData.META_VALUE])
self.assertEqual([tv0, tv1, tv2], data)
# remove 1 data at stop point
_timewindow = get_offset_timewindow(timewindow.stop())
self.perfdata.remove(
metric_id=metric_id,
timewindow=_timewindow
)
data, _meta = self.perfdata.get(
metric_id=metric_id,
timewindow=timewindow,
with_meta=True
)
self.assertEqual(meta, _meta[0][PerfData.META_VALUE])
self.assertEqual(data, [tv0, tv1])
# get data on timewindow
data, _meta = self.perfdata.get(
metric_id=metric_id,
timewindow=timewindow,
with_meta=True
)
self.assertEqual(meta, _meta[0][PerfData.META_VALUE])
# get all data
data, _meta = self.perfdata.get(
metric_id=metric_id,
with_meta=True
)
self.assertEqual(meta, _meta[0][PerfData.META_VALUE])
self.assertEqual(len(data), 3)
# remove all data
self.perfdata.remove(
metric_id=metric_id,
with_meta=True
)
data, _meta = self.perfdata.get(
metric_id=metric_id,
with_meta=True
)
self.assertIsNone(_meta)
self.assertEqual(len(data), 0)
def test_CRUD(self):
data_id = "test_store_id"
# start in droping data
self.storage.drop()
# ensure count is 0
count = self.storage.count(data_ids=data_id)
self.assertEquals(count, 0)
# let's play with different data_names
meta = {"min": None, "max": 0}
timewindow = TimeWindow()
before_timewindow = [timewindow.start() - 1000]
in_timewindow = [timewindow.start(), timewindow.start() + 5, timewindow.stop() - 5, timewindow.stop()]
after_timewindow = [timewindow.stop() + 1000]
# set timestamps without sort
timestamps = after_timewindow + before_timewindow + in_timewindow
for timestamp in timestamps:
# add a document at starting time window
self.storage.put(data_id=data_id, value=meta, timestamp=timestamp)
# check for count equals 2
count = self.storage.count(data_ids=data_id)
self.assertEquals(count, 2)
# check for_data before now
data = self.storage.get(data_ids=[data_id])
self.assertEquals(len(data[data_id]), 2)
# check for data before now with single id
data = self.storage.get(data_ids=data_id)
self.assertEquals(len(data), 2)
# check values are meta
self.assertEqual(data[0][MongoTimedStorage.VALUE]["max"], meta["max"])
# check filtering with max == 1
data = self.storage.get(data_ids=data_id, _filter={"max": 1})
self.assertEquals(len(data), 0)
# check filtering with max == 0
data = self.storage.get(data_ids=data_id, _filter={"max": 0})
self.assertEquals(len(data), 2)
# check find
data = self.storage.find(_filter={"max": 1})
self.assertEquals(len(data), 0)
data = self.storage.find(_filter={"max": 0})[data_id]
self.assertEquals(len(data), 2)
data = self.storage.find()[data_id]
self.assertEquals(len(data), 2)
# add twice same documents with different values
meta["max"] += 1
for dat in data:
# add a document at starting time window
self.storage.put(data_id=data_id, value=meta, timestamp=dat[MongoTimedStorage.TIMESTAMP])
# check for_data before now
data = self.storage.get(data_ids=[data_id])
self.assertEquals(len(data[data_id]), 2)
# check for_data before now
data = self.storage.get(data_ids=[data_id])
self.assertEquals(len(data[data_id]), 2)
# check for_data before now with single index
data = self.storage.get(data_ids=data_id)
self.assertEquals(len(data), 2)
# check values are new meta
self.assertEqual(data[0][MongoTimedStorage.VALUE]["max"], meta["max"])
# check filtering with max == 0
data = self.storage.get(data_ids=data_id, _filter={"max": 0})
self.assertEquals(len(data), 0)
# check filtering with max == 1
data = self.storage.get(data_ids=data_id, _filter={"max": 1})
self.assertEquals(len(data), 2)
# check find
data = self.storage.find(_filter={"max": 0})
self.assertEquals(len(data), 0)
data = self.storage.find(_filter={"max": 1})[data_id]
self.assertEquals(len(data), 2)
data = self.storage.find()[data_id]
self.assertEquals(len(data), 2)
# check for data inside timewindow and just before
data = self.storage.get(data_ids=[data_id], timewindow=timewindow)
self.assertEquals(len(data), 1)
# check for data inside timewindow and just before
data = self.storage.find(timewindow=timewindow)
self.assertEquals(len(data), 1)
# remove data inside timewindow
self.storage.remove(data_ids=[data_id], timewindow=timewindow)
# check for data outside timewindow
count = self.storage.count(data_ids=data_id)
self.assertEquals(count, len(before_timewindow) + len(after_timewindow))
# remove all data
self.storage.remove(data_ids=data_id)
# check for count equals 0
count = self.storage.count(data_ids=data_id)
self.assertEquals(count, 0)
def test_CRUD(self):
# start in droping data
self.storage.drop()
# let's play with different data_names
data_ids = ["m0", "m1"]
periods = [Period(**{Period.MINUTE: 60}), Period(**{Period.HOUR: 24})]
timewindow = TimeWindow()
points = [
(timewindow.start(), None), # lower bound
(timewindow.stop(), 0), # upper bound
(timewindow.start() - 1, 1), # outside timewindow (minus 1)
(timewindow.start() + 1, 2), # inside timewindow (plus 1)
(timewindow.stop() + 1, 3), # outside timewindow (plus 1)
]
sorted_points = sorted(points, key=lambda point: point[0])
inserted_points = dict()
# starts to put points for every aggregations and periods
for data_id in data_ids:
inserted_points[data_id] = dict()
for period in periods:
inserted_points[data_id][period] = points
# add documents
self.storage.put(data_id=data_id, period=period, points=points)
points_count_in_timewindow = len([point for point in points if point[0] in timewindow])
# check for reading methods
for data_id in data_ids:
# iterate on data_ids
for period in periods:
count = self.storage.count(data_id=data_id, period=period)
self.assertEquals(count, len(points))
count = self.storage.count(data_id=data_id, period=period, timewindow=timewindow)
self.assertEquals(count, points_count_in_timewindow)
data = self.storage.get(data_id=data_id, period=period)
self.assertEquals(len(data), len(points))
self.assertEquals(data, sorted_points)
data = self.storage.get(data_id=data_id, period=period, timewindow=timewindow)
self.assertEquals(len(data), points_count_in_timewindow)
self.assertEquals(data, [point for point in sorted_points if point[0] in timewindow])
self.storage.remove(data_id=data_id, period=period, timewindow=timewindow)
# check for count equals 1
count = self.storage.count(data_id=data_id, period=period, timewindow=timewindow)
self.assertEquals(count, 0)
count = self.storage.count(data_id=data_id, period=period)
self.assertEquals(count, len(points) - points_count_in_timewindow)
self.storage.remove(data_id=data_id, period=period)
# check for count equals 0
count = self.storage.count(data_id=data_id, period=period)
self.assertEquals(count, 0)
