def _test_CRUD(self, storage):
# let's play with different data_names
data_ids = ['m0', 'm1']
timewindow = TimeWindow()
points = [
(timewindow.start(), None), # lower bound
(timewindow.stop(), 0), # upper bound
(timewindow.start() - 1, 1), # outside timewindow (minus 1)
(timewindow.start() + 1, 2), # inside timewindow (plus 1)
(timewindow.stop() + 1, 3) # outside timewindow (plus 1)
]
sorted_points = sorted(points, key=lambda point: point[0])
inserted_points = {}
# starts to put points for every aggregations and periods
for data_id in data_ids:
inserted_points[data_id] = points
# add documents
storage.put(data_id=data_id, points=points)
points_count_in_timewindow = len(
[point for point in points if point[0] in timewindow])
# check for reading methods
for data_id in data_ids:
# iterate on data_ids
count = storage.count(data_id=data_id)
self.assertEquals(count, len(points))
count = storage.count(data_id=data_id, timewindow=timewindow)
self.assertEquals(count, points_count_in_timewindow)
data = storage.get(data_id=data_id)
self.assertEquals(len(data), len(points))
self.assertEquals(data, sorted_points)
data = storage.get(data_id=data_id, timewindow=timewindow)
self.assertEquals(len(data), points_count_in_timewindow)
self.assertEquals(
data,
[point for point in sorted_points if point[0] in timewindow])
storage.remove(data_id=data_id, timewindow=timewindow)
# check for count equals 1
count = storage.count(data_id=data_id, timewindow=timewindow)
self.assertEquals(count, 0)
count = storage.count(data_id=data_id)
self.assertEquals(count, len(points) - points_count_in_timewindow)
storage.remove(data_id=data_id)
# check for count equals 0
count = storage.count(data_id=data_id)
self.assertEquals(count, 0)
def test_CRUD(self):
data_id = 'test_store_id'
# start in droping data
self.storage.drop()
# ensure count is 0
count = self.storage.count(data_id=data_id)
self.assertEquals(count, 0)
# let's play with different data_names
meta = {'min': None, 'max': 0}
timewindow = TimeWindow()
before_timewindow = [timewindow.start() - 1000]
in_timewindow = [
timewindow.start(),
timewindow.start() + 5,
timewindow.stop() - 5,
timewindow.stop()
]
after_timewindow = [timewindow.stop() + 1000]
# set timestamps without sort
timestamps = after_timewindow + before_timewindow + in_timewindow
for timestamp in timestamps:
# add a document at starting time window
self.storage.put(data_id=data_id, value=meta, timestamp=timestamp)
# check for count equals 5
count = self.storage.count(data_id=data_id)
self.assertEquals(count, 2)
# check for_data before now
data = self.storage.get(data_ids=[data_id])
self.assertEquals(len(data[data_id]), 2)
# check for data inside timewindow and just before
data = self.storage.get(data_ids=[data_id], timewindow=timewindow)
self.assertEquals(len(data), 1)
# remove data inside timewindow
self.storage.remove(data_ids=[data_id], timewindow=timewindow)
# check for data outside timewindow
count = self.storage.count(data_id=data_id)
self.assertEquals(
count, len(before_timewindow) + len(after_timewindow))
# remove all data
self.storage.remove(data_ids=[data_id])
# check for count equals 0
count = self.storage.count(data_id=data_id)
self.assertEquals(count, 0)
def test_no_startstop(self):
start, stop = 5, 10
intervals = Interval((start, stop))
timewindow = TimeWindow(intervals=intervals)
self.assertEqual(timewindow.start(), start)
self.assertEqual(timewindow.stop(), stop)
def test_no_startstop(self):
start, stop = 5, 10
intervals = Interval((start, stop))
timewindow = TimeWindow(intervals=intervals)
self.assertEqual(timewindow.start(), start)
self.assertEqual(timewindow.stop(), stop)
def test_start_stop(self):
start = random() * 10000
stop = start + random() * 10000
timewindow = TimeWindow(start=start, stop=stop)
TimeWindow()
self.assertEqual(timewindow.start(), int(start))
self.assertEqual(timewindow.stop(), int(stop))
def test_start_stop(self):
start = random() * 10000
stop = start + random() * 10000
timewindow = TimeWindow(start=start, stop=stop)
TimeWindow()
self.assertEqual(timewindow.start(), int(start))
self.assertEqual(timewindow.stop(), int(stop))
def get(
self, metric_id, meta=None, with_meta=True, timewindow=None,
limit=0, skip=0, sort=None, timeserie=None, sliding_time=False
):
"""Get a set of data related to input data_id on the timewindow and
input period.
If with_meta, result is a couple of (points, list of tags by timestamp)
"""
data_id, tags = self._data_id_tags(metric_id, meta)
if sliding_time: # apply sliding_time
if timewindow is None:
timewindow = TimeWindow()
_timewindow = timewindow
timewindow = TimeWindow(
start=timewindow.start(),
stop=timewindow.stop() + SLIDING_TIME_UPPER_BOUND
)
result = self[PerfData.PERFDATA_STORAGE].get(
data_id=data_id, timewindow=timewindow, limit=limit,
skip=skip, timeserie=timeserie, tags=tags, with_tags=with_meta,
sort=sort
)
if sliding_time:
if with_meta:
points = result[0]
else:
points = result
points = [
(min(ts, _timewindow.stop()), val)
for (ts, val) in points
]
if with_meta:
result = points, result[1]
else:
result = points
return result
def test_CRUD(self):
data_id = "test_store_id"
# start in droping data
self.storage.drop()
# ensure count is 0
count = self.storage.count(data_ids=data_id)
self.assertEquals(count, 0)
# let's play with different data_names
meta = {"min": None, "max": 0}
timewindow = TimeWindow()
before_timewindow = [timewindow.start() - 1000]
in_timewindow = [timewindow.start(), timewindow.start() + 5, timewindow.stop() - 5, timewindow.stop()]
after_timewindow = [timewindow.stop() + 1000]
# set timestamps without sort
timestamps = after_timewindow + before_timewindow + in_timewindow
for timestamp in timestamps:
# add a document at starting time window
self.storage.put(data_id=data_id, value=meta, timestamp=timestamp)
# check for count equals 2
count = self.storage.count(data_ids=data_id)
self.assertEquals(count, 2)
# check for_data before now
data = self.storage.get(data_ids=[data_id])
self.assertEquals(len(data[data_id]), 2)
# check for data before now with single id
data = self.storage.get(data_ids=data_id)
self.assertEquals(len(data), 2)
# check values are meta
self.assertEqual(data[0][MongoTimedStorage.VALUE]["max"], meta["max"])
# check filtering with max == 1
data = self.storage.get(data_ids=data_id, _filter={"max": 1})
self.assertEquals(len(data), 0)
# check filtering with max == 0
data = self.storage.get(data_ids=data_id, _filter={"max": 0})
self.assertEquals(len(data), 2)
# check find
data = self.storage.find(_filter={"max": 1})
self.assertEquals(len(data), 0)
data = self.storage.find(_filter={"max": 0})[data_id]
self.assertEquals(len(data), 2)
data = self.storage.find()[data_id]
self.assertEquals(len(data), 2)
# add twice same documents with different values
meta["max"] += 1
for dat in data:
# add a document at starting time window
self.storage.put(data_id=data_id, value=meta, timestamp=dat[MongoTimedStorage.TIMESTAMP])
# check for_data before now
data = self.storage.get(data_ids=[data_id])
self.assertEquals(len(data[data_id]), 2)
# check for_data before now
data = self.storage.get(data_ids=[data_id])
self.assertEquals(len(data[data_id]), 2)
# check for_data before now with single index
data = self.storage.get(data_ids=data_id)
self.assertEquals(len(data), 2)
# check values are new meta
self.assertEqual(data[0][MongoTimedStorage.VALUE]["max"], meta["max"])
# check filtering with max == 0
data = self.storage.get(data_ids=data_id, _filter={"max": 0})
self.assertEquals(len(data), 0)
# check filtering with max == 1
data = self.storage.get(data_ids=data_id, _filter={"max": 1})
self.assertEquals(len(data), 2)
# check find
data = self.storage.find(_filter={"max": 0})
self.assertEquals(len(data), 0)
data = self.storage.find(_filter={"max": 1})[data_id]
self.assertEquals(len(data), 2)
data = self.storage.find()[data_id]
self.assertEquals(len(data), 2)
# check for data inside timewindow and just before
data = self.storage.get(data_ids=[data_id], timewindow=timewindow)
self.assertEquals(len(data), 1)
# check for data inside timewindow and just before
data = self.storage.find(timewindow=timewindow)
self.assertEquals(len(data), 1)
# remove data inside timewindow
self.storage.remove(data_ids=[data_id], timewindow=timewindow)
# check for data outside timewindow
count = self.storage.count(data_ids=data_id)
self.assertEquals(count, len(before_timewindow) + len(after_timewindow))
# remove all data
self.storage.remove(data_ids=data_id)
# check for count equals 0
count = self.storage.count(data_ids=data_id)
self.assertEquals(count, 0)
def test_put_get_data(self):
timewindow = TimeWindow()
metric_id = 'test_manager'
self.perfdata.remove(metric_id=metric_id, with_meta=True)
count = self.perfdata.count(metric_id=metric_id)
self.assertEqual(count, 0)
tv0 = (int(timewindow.start()), None)
tv1 = (int(timewindow.start() + 1), 0)
tv2 = (int(timewindow.stop()), 2)
tv3 = (int(timewindow.stop() + 1000000), 3)
# set values with timestamp without order
points = [tv0, tv2, tv1, tv3]
meta = {'plop': None}
self.perfdata.put(
metric_id=metric_id,
points=points,
meta=meta
)
data, _meta = self.perfdata.get(
metric_id=metric_id,
timewindow=timewindow,
with_meta=True
)
self.assertEqual(meta, _meta[0][PerfData.META_VALUE])
self.assertEqual([tv0, tv1, tv2], data)
# remove 1 data at stop point
_timewindow = get_offset_timewindow(timewindow.stop())
self.perfdata.remove(
metric_id=metric_id,
timewindow=_timewindow
)
data, _meta = self.perfdata.get(
metric_id=metric_id,
timewindow=timewindow,
with_meta=True
)
self.assertEqual(meta, _meta[0][PerfData.META_VALUE])
self.assertEqual(data, [tv0, tv1])
# get data on timewindow
data, _meta = self.perfdata.get(
metric_id=metric_id,
timewindow=timewindow,
with_meta=True
)
self.assertEqual(meta, _meta[0][PerfData.META_VALUE])
# get all data
data, _meta = self.perfdata.get(
metric_id=metric_id,
with_meta=True
)
self.assertEqual(meta, _meta[0][PerfData.META_VALUE])
self.assertEqual(len(data), 3)
# remove all data
self.perfdata.remove(
metric_id=metric_id,
with_meta=True
)
data, _meta = self.perfdata.get(
metric_id=metric_id,
with_meta=True
)
self.assertIsNone(_meta)
self.assertEqual(len(data), 0)
def test_CRUD(self):
# start in droping data
self.storage.drop()
# let's play with different data_names
data_ids = ["m0", "m1"]
periods = [Period(**{Period.MINUTE: 60}), Period(**{Period.HOUR: 24})]
timewindow = TimeWindow()
points = [
(timewindow.start(), None), # lower bound
(timewindow.stop(), 0), # upper bound
(timewindow.start() - 1, 1), # outside timewindow (minus 1)
(timewindow.start() + 1, 2), # inside timewindow (plus 1)
(timewindow.stop() + 1, 3), # outside timewindow (plus 1)
]
sorted_points = sorted(points, key=lambda point: point[0])
inserted_points = dict()
# starts to put points for every aggregations and periods
for data_id in data_ids:
inserted_points[data_id] = dict()
for period in periods:
inserted_points[data_id][period] = points
# add documents
self.storage.put(data_id=data_id, period=period, points=points)
points_count_in_timewindow = len([point for point in points if point[0] in timewindow])
# check for reading methods
for data_id in data_ids:
# iterate on data_ids
for period in periods:
count = self.storage.count(data_id=data_id, period=period)
self.assertEquals(count, len(points))
count = self.storage.count(data_id=data_id, period=period, timewindow=timewindow)
self.assertEquals(count, points_count_in_timewindow)
data = self.storage.get(data_id=data_id, period=period)
self.assertEquals(len(data), len(points))
self.assertEquals(data, sorted_points)
data = self.storage.get(data_id=data_id, period=period, timewindow=timewindow)
self.assertEquals(len(data), points_count_in_timewindow)
self.assertEquals(data, [point for point in sorted_points if point[0] in timewindow])
self.storage.remove(data_id=data_id, period=period, timewindow=timewindow)
# check for count equals 1
count = self.storage.count(data_id=data_id, period=period, timewindow=timewindow)
self.assertEquals(count, 0)
count = self.storage.count(data_id=data_id, period=period)
self.assertEquals(count, len(points) - points_count_in_timewindow)
self.storage.remove(data_id=data_id, period=period)
# check for count equals 0
count = self.storage.count(data_id=data_id, period=period)
self.assertEquals(count, 0)
def _test_CRUD(self, storage):
# let's play with different data_names
data_ids = ['m0', 'm1']
timewindow = TimeWindow()
points = [
(timewindow.start(), None), # lower bound
(timewindow.stop(), 0), # upper bound
(timewindow.start() - 1, 1), # outside timewindow (minus 1)
(timewindow.start() + 1, 2), # inside timewindow (plus 1)
(timewindow.stop() + 1, 3) # outside timewindow (plus 1)
]
sorted_points = sorted(points, key=lambda point: point[0])
inserted_points = {}
# starts to put points for every aggregations and periods
for data_id in data_ids:
inserted_points[data_id] = points
# add documents
storage.put(data_id=data_id, points=points)
points_count_in_timewindow = len(
[point for point in points if point[0] in timewindow])
# check for reading methods
for data_id in data_ids:
# iterate on data_ids
count = storage.count(data_id=data_id)
self.assertEquals(count, len(points))
count = storage.count(data_id=data_id, timewindow=timewindow)
self.assertEquals(count, points_count_in_timewindow)
data = storage.get(data_id=data_id)
self.assertEquals(len(data), len(points))
self.assertEquals(data, sorted_points)
data = storage.get(data_id=data_id, timewindow=timewindow)
self.assertEquals(len(data), points_count_in_timewindow)
self.assertEquals(
data,
[
point for point in sorted_points
if point[0] in timewindow
]
)
storage.remove(data_id=data_id, timewindow=timewindow)
# check for count equals 1
count = storage.count(data_id=data_id, timewindow=timewindow)
self.assertEquals(count, 0)
count = storage.count(data_id=data_id)
self.assertEquals(
count, len(points) - points_count_in_timewindow)
storage.remove(data_id=data_id)
# check for count equals 0
count = storage.count(data_id=data_id)
self.assertEquals(count, 0)