def test_get_cross_metric_measures_empty_metrics_no_overlap(self):
self.assertRaises(
cross_metric.MetricUnaggregatable,
cross_metric.get_cross_metric_measures, self.storage, [
storage.Metric(uuid.uuid4(), self.archive_policies['low']),
storage.Metric(uuid.uuid4(), self.archive_policies['low'])
])
def test_get_cross_metric_measures_unknown_metric(self):
self.assertEqual([],
self.storage.get_cross_metric_measures(
[storage.Metric(uuid.uuid4(),
self.archive_policies['low']),
storage.Metric(uuid.uuid4(),
self.archive_policies['low'])]))
def test_rewrite_measures_corruption_bad_data(self):
# Create an archive policy that spans on several splits. Each split
# being 3600 points, let's go for 36k points so we have 10 splits.
apname = str(uuid.uuid4())
ap = archive_policy.ArchivePolicy(apname, 0, [(36000, 60)])
self.index.create_archive_policy(ap)
self.metric = storage.Metric(uuid.uuid4(), ap)
self.index.create_metric(self.metric.id, str(uuid.uuid4()), apname)
# First store some points scattered across different splits
self.incoming.add_measures(self.metric, [
storage.Measure(utils.dt_to_unix_ns(2016, 1, 1, 12, 0, 1), 69),
storage.Measure(utils.dt_to_unix_ns(2016, 1, 2, 13, 7, 31), 42),
storage.Measure(utils.dt_to_unix_ns(2016, 1, 4, 14, 9, 31), 4),
storage.Measure(utils.dt_to_unix_ns(2016, 1, 6, 15, 12, 45), 44),
])
self.trigger_processing()
splits = {'1451520000.0', '1451736000.0', '1451952000.0'}
self.assertEqual(
splits,
self.storage._list_split_keys_for_metric(self.metric, "mean",
60.0))
if self.storage.WRITE_FULL:
assertCompressedIfWriteFull = self.assertTrue
else:
assertCompressedIfWriteFull = self.assertFalse
data = self.storage._get_measures(self.metric, '1451520000.0', "mean",
60.0)
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, '1451736000.0', "mean",
60.0)
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, '1451952000.0', "mean",
60.0)
assertCompressedIfWriteFull(
carbonara.AggregatedTimeSerie.is_compressed(data))
self.assertEqual([
(utils.datetime_utc(2016, 1, 1, 12), 60.0, 69),
(utils.datetime_utc(2016, 1, 2, 13, 7), 60.0, 42),
(utils.datetime_utc(2016, 1, 4, 14, 9), 60.0, 4),
(utils.datetime_utc(2016, 1, 6, 15, 12), 60.0, 44),
], self.storage.get_measures(self.metric, granularity=60.0))
# Test what happens if we write garbage
self.storage._store_metric_measures(self.metric, '1451952000.0',
"mean", 60.0, b"oh really?")
# Now store brand new points that should force a rewrite of one of the
# split (keep in mind the back window size in one hour here). We move
# the BoundTimeSerie processing timeserie far away from its current
# range.
self.incoming.add_measures(self.metric, [
storage.Measure(utils.dt_to_unix_ns(2016, 1, 10, 16, 18, 45), 45),
storage.Measure(utils.dt_to_unix_ns(2016, 1, 10, 17, 12, 45), 46),
])
self.trigger_processing()
def _create_metric(self, archive_policy_name="low"):
"""Create a metric and return it"""
m = storage.Metric(uuid.uuid4(),
self.archive_policies[archive_policy_name])
m_sql = self.index.create_metric(m.id, str(uuid.uuid4()),
archive_policy_name)
return m, m_sql
def test_resize_policy(self):
name = str(uuid.uuid4())
ap = archive_policy.ArchivePolicy(name, 0, [(3, 5)])
self.index.create_archive_policy(ap)
m = storage.Metric(uuid.uuid4(), ap)
self.index.create_metric(m.id, str(uuid.uuid4()), str(uuid.uuid4()),
name)
self.storage.add_measures(m, [
storage.Measure(datetime.datetime(2014, 1, 1, 12, 0, 0), 1),
storage.Measure(datetime.datetime(2014, 1, 1, 12, 0, 5), 1),
storage.Measure(datetime.datetime(2014, 1, 1, 12, 0, 10), 1),
])
self.storage.process_background_tasks(self.index, sync=True)
self.assertEqual([
(utils.datetime_utc(2014, 1, 1, 12, 0, 0), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 5), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 10), 5.0, 1.0),
], self.storage.get_measures(m))
# expand to more points
self.index.update_archive_policy(
name, [archive_policy.ArchivePolicyItem(granularity=5, points=6)])
self.storage.add_measures(m, [
storage.Measure(datetime.datetime(2014, 1, 1, 12, 0, 15), 1),
])
self.storage.process_background_tasks(self.index, sync=True)
self.assertEqual([
(utils.datetime_utc(2014, 1, 1, 12, 0, 0), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 5), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 10), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 15), 5.0, 1.0),
], self.storage.get_measures(m))
# shrink timespan
self.index.update_archive_policy(
name, [archive_policy.ArchivePolicyItem(granularity=5, points=2)])
# unchanged after update if no samples
self.storage.process_background_tasks(self.index, sync=True)
self.assertEqual([
(utils.datetime_utc(2014, 1, 1, 12, 0, 0), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 5), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 10), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 15), 5.0, 1.0),
], self.storage.get_measures(m))
# drop points
self.storage.add_measures(m, [
storage.Measure(datetime.datetime(2014, 1, 1, 12, 0, 20), 1),
])
self.storage.process_background_tasks(self.index, sync=True)
self.assertEqual([
(utils.datetime_utc(2014, 1, 1, 12, 0, 15), 5.0, 1.0),
(utils.datetime_utc(2014, 1, 1, 12, 0, 20), 5.0, 1.0),
], self.storage.get_measures(m))
def _test_create_metric_and_data(self, data, spacing):
metric = storage.Metric(uuid.uuid4(), self.archive_policies['medium'])
start_time = datetime.datetime(2014, 1, 1, 12)
incr = datetime.timedelta(seconds=spacing)
measures = [
storage.Measure(start_time + incr * n, val)
for n, val in enumerate(data)
]
self.index.create_metric(metric.id, str(uuid.uuid4()),
str(uuid.uuid4()), 'medium')
self.storage.add_measures(metric, measures)
self.storage.process_background_tasks(self.index, sync=True)
return metric
def _test_create_metric_and_data(self, data, spacing):
metric = storage.Metric(uuid.uuid4(), self.archive_policies['medium'])
start_time = utils.datetime_utc(2014, 1, 1, 12)
incr = datetime.timedelta(seconds=spacing)
measures = [
storage.Measure(utils.dt_in_unix_ns(start_time + incr * n), val)
for n, val in enumerate(data)
]
self.index.create_metric(metric.id, str(uuid.uuid4()), 'medium')
self.storage.incoming.add_measures(metric, measures)
metrics = tests_utils.list_all_incoming_metrics(self.storage.incoming)
self.storage.process_background_tasks(self.index, metrics, sync=True)
return metric
def test_get_cross_metric_measures_unknown_granularity(self):
metric2 = storage.Metric(uuid.uuid4(), self.archive_policies['low'])
self.incoming.add_measures(self.metric, [
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44),
])
self.incoming.add_measures(metric2, [
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44),
])
self.assertRaises(storage.GranularityDoesNotExist,
self.storage.get_cross_metric_measures,
[self.metric, metric2],
granularity=12345.456)
def test_get_cross_metric_measures_unknown_aggregation(self):
metric2 = storage.Metric(uuid.uuid4(), self.archive_policies['low'])
self.incoming.add_measures(self.metric, [
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44),
])
self.incoming.add_measures(metric2, [
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44),
])
self.assertRaises(storage.AggregationDoesNotExist,
self.storage.get_cross_metric_measures,
[self.metric, metric2],
aggregation='last')
def test_get_cross_metric_measures_unknown_granularity(self):
metric2 = storage.Metric(uuid.uuid4(), self.archive_policies['low'])
self.incoming.add_measures(self.metric, [
storage.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
storage.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
storage.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
storage.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
])
self.incoming.add_measures(metric2, [
storage.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
storage.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
storage.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
storage.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
])
self.assertRaises(storage.GranularityDoesNotExist,
cross_metric.get_cross_metric_measures,
self.storage, [self.metric, metric2],
granularity=numpy.timedelta64(12345456, 'ms'))
def test_add_and_get_cross_metric_measures_different_archives(self):
metric2 = storage.Metric(uuid.uuid4(),
self.archive_policies['no_granularity_match'])
self.incoming.add_measures(self.metric, [
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44),
])
self.incoming.add_measures(metric2, [
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4),
storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44),
])
self.assertRaises(storage.MetricUnaggregatable,
self.storage.get_cross_metric_measures,
[self.metric, metric2])
def setUp(self):
super(TestCarbonaraMigration, self).setUp()
if not isinstance(self.storage, _carbonara.CarbonaraBasedStorage):
self.skipTest("This driver is not based on Carbonara")
self.metric = storage.Metric(uuid.uuid4(),
self.archive_policies['low'])
archive = carbonara.TimeSerieArchive.from_definitions([
(v.granularity, v.points)
for v in self.metric.archive_policy.definition
])
archive_max = carbonara.TimeSerieArchive.from_definitions(
[(v.granularity, v.points)
for v in self.metric.archive_policy.definition],
aggregation_method='max',
)
for a in (archive, archive_max):
a.update(
carbonara.TimeSerie.from_data([
datetime.datetime(2014, 1, 1, 12, 0, 0),
datetime.datetime(2014, 1, 1, 12, 0, 4),
datetime.datetime(2014, 1, 1, 12, 0, 9)
], [4, 5, 6]))
self.storage._create_metric(self.metric)
# serialise in old format
with mock.patch('gnocchi.carbonara.AggregatedTimeSerie.to_dict',
autospec=True) as f:
f.side_effect = _to_dict_v1_3
self.storage._store_metric_archive(
self.metric, archive.agg_timeseries[0].aggregation_method,
archive.serialize())
self.storage._store_metric_archive(
self.metric, archive_max.agg_timeseries[0].aggregation_method,
archive_max.serialize())
def test_rewrite_measures_oldest_mutable_timestamp_eq_next_key(self):
"""See LP#1655422"""
# Create an archive policy that spans on several splits. Each split
# being 3600 points, let's go for 36k points so we have 10 splits.
apname = str(uuid.uuid4())
ap = archive_policy.ArchivePolicy(apname, 0, [(36000, 60)])
self.index.create_archive_policy(ap)
self.metric = storage.Metric(uuid.uuid4(), ap)
self.index.create_metric(self.metric.id, str(uuid.uuid4()), apname)
# First store some points scattered across different splits
self.incoming.add_measures(self.metric, [
storage.Measure(utils.dt_to_unix_ns(2016, 1, 1, 12, 0, 1), 69),
storage.Measure(utils.dt_to_unix_ns(2016, 1, 2, 13, 7, 31), 42),
storage.Measure(utils.dt_to_unix_ns(2016, 1, 4, 14, 9, 31), 4),
storage.Measure(utils.dt_to_unix_ns(2016, 1, 6, 15, 12, 45), 44),
])
self.trigger_processing()
splits = {'1451520000.0', '1451736000.0', '1451952000.0'}
self.assertEqual(
splits,
self.storage._list_split_keys_for_metric(self.metric, "mean",
60.0))
if self.storage.WRITE_FULL:
assertCompressedIfWriteFull = self.assertTrue
else:
assertCompressedIfWriteFull = self.assertFalse
data = self.storage._get_measures(self.metric, '1451520000.0', "mean",
60.0)
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, '1451736000.0', "mean",
60.0)
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, '1451952000.0', "mean",
60.0)
assertCompressedIfWriteFull(
carbonara.AggregatedTimeSerie.is_compressed(data))
self.assertEqual([
(utils.datetime_utc(2016, 1, 1, 12), 60.0, 69),
(utils.datetime_utc(2016, 1, 2, 13, 7), 60.0, 42),
(utils.datetime_utc(2016, 1, 4, 14, 9), 60.0, 4),
(utils.datetime_utc(2016, 1, 6, 15, 12), 60.0, 44),
], self.storage.get_measures(self.metric, granularity=60.0))
# Now store brand new points that should force a rewrite of one of the
# split (keep in mind the back window size in one hour here). We move
# the BoundTimeSerie processing timeserie far away from its current
# range.
# Here we test a special case where the oldest_mutable_timestamp will
# be 2016-01-10TOO:OO:OO = 1452384000.0, our new split key.
self.incoming.add_measures(self.metric, [
storage.Measure(utils.dt_to_unix_ns(2016, 1, 10, 0, 12), 45),
])
self.trigger_processing()
self.assertEqual(
{'1452384000.0', '1451736000.0', '1451520000.0', '1451952000.0'},
self.storage._list_split_keys_for_metric(self.metric, "mean",
60.0))
data = self.storage._get_measures(self.metric, '1451520000.0', "mean",
60.0)
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, '1451736000.0', "mean",
60.0)
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, '1451952000.0', "mean",
60.0)
# Now this one is compressed because it has been rewritten!
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, '1452384000.0', "mean",
60.0)
assertCompressedIfWriteFull(
carbonara.AggregatedTimeSerie.is_compressed(data))
self.assertEqual([
(utils.datetime_utc(2016, 1, 1, 12), 60.0, 69),
(utils.datetime_utc(2016, 1, 2, 13, 7), 60.0, 42),
(utils.datetime_utc(2016, 1, 4, 14, 9), 60.0, 4),
(utils.datetime_utc(2016, 1, 6, 15, 12), 60.0, 44),
(utils.datetime_utc(2016, 1, 10, 0, 12), 60.0, 45),
], self.storage.get_measures(self.metric, granularity=60.0))
def setUp(self):
super(TestIncomingDriver, self).setUp()
# A lot of tests wants a metric, create one
self.metric = storage.Metric(uuid.uuid4(),
self.archive_policies["low"])
def test_rewrite_measures_corruption_missing_file(self):
# Create an archive policy that spans on several splits. Each split
# being 3600 points, let's go for 36k points so we have 10 splits.
apname = str(uuid.uuid4())
ap = archive_policy.ArchivePolicy(apname, 0, [(36000, 60)])
self.index.create_archive_policy(ap)
self.metric = storage.Metric(uuid.uuid4(), ap)
self.index.create_metric(self.metric.id, str(uuid.uuid4()), apname)
# First store some points scattered across different splits
self.incoming.add_measures(self.metric, [
storage.Measure(datetime64(2016, 1, 1, 12, 0, 1), 69),
storage.Measure(datetime64(2016, 1, 2, 13, 7, 31), 42),
storage.Measure(datetime64(2016, 1, 4, 14, 9, 31), 4),
storage.Measure(datetime64(2016, 1, 6, 15, 12, 45), 44),
])
self.trigger_processing()
self.assertEqual(
{
carbonara.SplitKey(numpy.datetime64('2015-12-31T00:00:00'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64('2016-01-02T12:00:00'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64('2016-01-05T00:00:00'),
numpy.timedelta64(1, 'm')),
},
self.storage._list_split_keys_for_metric(self.metric, "mean",
numpy.timedelta64(1,
'm')))
if self.storage.WRITE_FULL:
assertCompressedIfWriteFull = self.assertTrue
else:
assertCompressedIfWriteFull = self.assertFalse
data = self.storage._get_measures(
self.metric,
carbonara.SplitKey(
numpy.datetime64(1451520000, 's'),
numpy.timedelta64(1, 'm'),
), "mean")
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(
self.metric,
carbonara.SplitKey(numpy.datetime64(1451736000, 's'),
numpy.timedelta64(1, 'm')), "mean")
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(
self.metric,
carbonara.SplitKey(
numpy.datetime64(1451952000, 's'),
numpy.timedelta64(1, 'm'),
), "mean")
assertCompressedIfWriteFull(
carbonara.AggregatedTimeSerie.is_compressed(data))
self.assertEqual([
(datetime64(2016, 1, 1, 12), numpy.timedelta64(1, 'm'), 69),
(datetime64(2016, 1, 2, 13, 7), numpy.timedelta64(1, 'm'), 42),
(datetime64(2016, 1, 4, 14, 9), numpy.timedelta64(1, 'm'), 4),
(datetime64(2016, 1, 6, 15, 12), numpy.timedelta64(1, 'm'), 44),
],
self.storage.get_measures(
self.metric,
granularity=numpy.timedelta64(60, 's')))
# Test what happens if we delete the latest split and then need to
# compress it!
self.storage._delete_metric_measures(
self.metric,
carbonara.SplitKey(
numpy.datetime64(1451952000, 's'),
numpy.timedelta64(1, 'm'),
), 'mean')
# Now store brand new points that should force a rewrite of one of the
# split (keep in mind the back window size in one hour here). We move
# the BoundTimeSerie processing timeserie far away from its current
# range.
self.incoming.add_measures(self.metric, [
storage.Measure(datetime64(2016, 1, 10, 16, 18, 45), 45),
storage.Measure(datetime64(2016, 1, 10, 17, 12, 45), 46),
])
self.trigger_processing()
