def test_get_cross_metric_measures_empty_metrics_no_overlap(self):
self.assertRaises(
cross_metric.MetricUnaggregatable,
cross_metric.get_cross_metric_measures, self.storage, [
indexer.Metric(uuid.uuid4(), self.archive_policies['low']),
indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
])
def _create_metric(self, archive_policy_name="low"):
"""Create a metric and return it"""
m = indexer.Metric(uuid.uuid4(),
self.archive_policies[archive_policy_name])
m_sql = self.index.create_metric(m.id, str(uuid.uuid4()),
archive_policy_name)
return m, m_sql
def _test_create_metric_and_data(self, data, spacing):
metric = indexer.Metric(
uuid.uuid4(), self.archive_policies['medium'])
start_time = utils.datetime_utc(2014, 1, 1, 12)
incr = datetime.timedelta(seconds=spacing)
measures = [incoming.Measure(
utils.dt_in_unix_ns(start_time + incr * n), val)
for n, val in enumerate(data)]
self.index.create_metric(metric.id, str(uuid.uuid4()), 'medium')
self.incoming.add_measures(metric, measures)
metrics = tests_utils.list_all_incoming_metrics(self.incoming)
self.storage.process_new_measures(
self.index, self.incoming, metrics, sync=True)
return metric
def test_get_cross_metric_measures_unknown_granularity(self):
metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
self.incoming.add_measures(self.metric, [
incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
])
self.incoming.add_measures(metric2, [
incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
])
self.assertRaises(storage.GranularityDoesNotExist,
cross_metric.get_cross_metric_measures,
self.storage, [self.metric, metric2],
granularity=numpy.timedelta64(12345456, 'ms'))
def test_get_cross_metric_measures_unknown_aggregation(self):
metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
self.incoming.add_measures(self.metric, [
incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
])
self.incoming.add_measures(metric2, [
incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
])
self.assertRaises(storage.AggregationDoesNotExist,
cross_metric.get_cross_metric_measures,
self.storage, [self.metric, metric2],
aggregation='last')
def test_add_and_get_cross_metric_measures_different_archives(self):
metric2 = indexer.Metric(uuid.uuid4(),
self.archive_policies['no_granularity_match'])
self.incoming.add_measures(self.metric, [
incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
])
self.incoming.add_measures(metric2, [
incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
])
self.assertRaises(cross_metric.MetricUnaggregatable,
cross_metric.get_cross_metric_measures, self.storage,
[self.metric, metric2])
def setUp(self):
super(TestIncomingDriver, self).setUp()
# A lot of tests wants a metric, create one
self.metric = indexer.Metric(uuid.uuid4(),
self.archive_policies["low"])
def test_rewrite_measures_corruption_bad_data(self):
# Create an archive policy that spans on several splits. Each split
# being 3600 points, let's go for 36k points so we have 10 splits.
apname = str(uuid.uuid4())
ap = archive_policy.ArchivePolicy(apname, 0, [(36000, 60)])
self.index.create_archive_policy(ap)
self.metric = indexer.Metric(uuid.uuid4(), ap)
self.index.create_metric(self.metric.id, str(uuid.uuid4()), apname)
# First store some points scattered across different splits
self.incoming.add_measures(self.metric.id, [
incoming.Measure(datetime64(2016, 1, 1, 12, 0, 1), 69),
incoming.Measure(datetime64(2016, 1, 2, 13, 7, 31), 42),
incoming.Measure(datetime64(2016, 1, 4, 14, 9, 31), 4),
incoming.Measure(datetime64(2016, 1, 6, 15, 12, 45), 44),
])
self.trigger_processing()
self.assertEqual(
{
carbonara.SplitKey(numpy.datetime64(1451520000, 's'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64(1451736000, 's'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64(1451952000, 's'),
numpy.timedelta64(1, 'm')),
},
self.storage._list_split_keys_for_metric(self.metric, "mean",
numpy.timedelta64(1,
'm')))
if self.storage.WRITE_FULL:
assertCompressedIfWriteFull = self.assertTrue
else:
assertCompressedIfWriteFull = self.assertFalse
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(
numpy.datetime64(1451520000, 's'),
numpy.timedelta64(60, 's'),
)
], "mean")[0]
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(
numpy.datetime64(1451736000, 's'),
numpy.timedelta64(1, 'm'),
)
], "mean")[0]
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(
numpy.datetime64(1451952000, 's'),
numpy.timedelta64(1, 'm'),
)
], "mean")[0]
assertCompressedIfWriteFull(
carbonara.AggregatedTimeSerie.is_compressed(data))
self.assertEqual([
(datetime64(2016, 1, 1, 12), numpy.timedelta64(1, 'm'), 69),
(datetime64(2016, 1, 2, 13, 7), numpy.timedelta64(1, 'm'), 42),
(datetime64(2016, 1, 4, 14, 9), numpy.timedelta64(1, 'm'), 4),
(datetime64(2016, 1, 6, 15, 12), numpy.timedelta64(1, 'm'), 44),
],
self.storage.get_measures(
self.metric,
granularities=[numpy.timedelta64(1, 'm')]))
# Test what happens if we write garbage
self.storage._store_metric_measures(
self.metric,
carbonara.SplitKey(
numpy.datetime64(1451952000, 's'),
numpy.timedelta64(1, 'm'),
), "mean", b"oh really?")
# Now store brand new points that should force a rewrite of one of the
# split (keep in mind the back window size in one hour here). We move
# the BoundTimeSerie processing timeserie far away from its current
# range.
self.incoming.add_measures(self.metric.id, [
incoming.Measure(datetime64(2016, 1, 10, 16, 18, 45), 45),
incoming.Measure(datetime64(2016, 1, 10, 17, 12, 45), 46),
])
self.trigger_processing()
def test_rewrite_measures_oldest_mutable_timestamp_eq_next_key(self):
"""See LP#1655422"""
# Create an archive policy that spans on several splits. Each split
# being 3600 points, let's go for 36k points so we have 10 splits.
apname = str(uuid.uuid4())
ap = archive_policy.ArchivePolicy(apname, 0, [(36000, 60)])
self.index.create_archive_policy(ap)
self.metric = indexer.Metric(uuid.uuid4(), ap)
self.index.create_metric(self.metric.id, str(uuid.uuid4()), apname)
# First store some points scattered across different splits
self.incoming.add_measures(self.metric.id, [
incoming.Measure(datetime64(2016, 1, 1, 12, 0, 1), 69),
incoming.Measure(datetime64(2016, 1, 2, 13, 7, 31), 42),
incoming.Measure(datetime64(2016, 1, 4, 14, 9, 31), 4),
incoming.Measure(datetime64(2016, 1, 6, 15, 12, 45), 44),
])
self.trigger_processing()
self.assertEqual(
{
carbonara.SplitKey(numpy.datetime64(1451520000, 's'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64(1451736000, 's'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64(1451952000, 's'),
numpy.timedelta64(1, 'm')),
},
self.storage._list_split_keys_for_metric(self.metric, "mean",
numpy.timedelta64(1,
'm')))
if self.storage.WRITE_FULL:
assertCompressedIfWriteFull = self.assertTrue
else:
assertCompressedIfWriteFull = self.assertFalse
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(
numpy.datetime64(1451520000, 's'),
numpy.timedelta64(1, 'm'),
)
], "mean")[0]
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(
numpy.datetime64(1451736000, 's'),
numpy.timedelta64(1, 'm'),
)
], "mean")[0]
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(numpy.datetime64(1451952000, 's'),
numpy.timedelta64(1, 'm'))
], "mean")[0]
assertCompressedIfWriteFull(
carbonara.AggregatedTimeSerie.is_compressed(data))
self.assertEqual([
(datetime64(2016, 1, 1, 12), numpy.timedelta64(1, 'm'), 69),
(datetime64(2016, 1, 2, 13, 7), numpy.timedelta64(1, 'm'), 42),
(datetime64(2016, 1, 4, 14, 9), numpy.timedelta64(1, 'm'), 4),
(datetime64(2016, 1, 6, 15, 12), numpy.timedelta64(1, 'm'), 44),
],
self.storage.get_measures(
self.metric,
granularities=[numpy.timedelta64(60, 's')]))
# Now store brand new points that should force a rewrite of one of the
# split (keep in mind the back window size in one hour here). We move
# the BoundTimeSerie processing timeserie far away from its current
# range.
# Here we test a special case where the oldest_mutable_timestamp will
# be 2016-01-10TOO:OO:OO = 1452384000.0, our new split key.
self.incoming.add_measures(self.metric.id, [
incoming.Measure(datetime64(2016, 1, 10, 0, 12), 45),
])
self.trigger_processing()
self.assertEqual(
{
carbonara.SplitKey(numpy.datetime64('2016-01-10T00:00:00'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64('2016-01-02T12:00:00'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64('2015-12-31T00:00:00'),
numpy.timedelta64(1, 'm')),
carbonara.SplitKey(numpy.datetime64('2016-01-05T00:00:00'),
numpy.timedelta64(1, 'm')),
},
self.storage._list_split_keys_for_metric(self.metric, "mean",
numpy.timedelta64(1,
'm')))
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(
numpy.datetime64(1451520000, 's'),
numpy.timedelta64(1, 'm'),
)
], "mean")[0]
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(
numpy.datetime64(1451736000, 's'),
numpy.timedelta64(1, 'm'),
)
], "mean")[0]
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(numpy.datetime64(1451952000, 's'),
numpy.timedelta64(60, 's'))
], "mean")[0]
# Now this one is compressed because it has been rewritten!
self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data))
data = self.storage._get_measures(self.metric, [
carbonara.SplitKey(
numpy.datetime64(1452384000, 's'),
numpy.timedelta64(1, 'm'),
)
], "mean")[0]
assertCompressedIfWriteFull(
carbonara.AggregatedTimeSerie.is_compressed(data))
self.assertEqual([
(datetime64(2016, 1, 1, 12), numpy.timedelta64(1, 'm'), 69),
(datetime64(2016, 1, 2, 13, 7), numpy.timedelta64(1, 'm'), 42),
(datetime64(2016, 1, 4, 14, 9), numpy.timedelta64(1, 'm'), 4),
(datetime64(2016, 1, 6, 15, 12), numpy.timedelta64(1, 'm'), 44),
(datetime64(2016, 1, 10, 0, 12), numpy.timedelta64(1, 'm'), 45),
],
self.storage.get_measures(
self.metric,
granularities=[numpy.timedelta64(60, 's')]))
