def test_get_measure_unknown_aggregation(self): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.assertRaises(storage.AggregationDoesNotExist, self.storage.get_measures, self.metric, aggregation='last')
def test_add_and_get_cross_metric_measures_with_holes(self): metric2, __ = self._create_metric() self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 5, 31), 8), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 42), ]) self.incoming.add_measures(metric2, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 5), 9), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 2), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 6), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 13, 10), 2), ]) self.trigger_processing([str(self.metric.id), str(metric2.id)]) values = self.storage.get_cross_metric_measures([self.metric, metric2]) self.assertEqual([ (utils.datetime_utc(2014, 1, 1, 0, 0, 0), 86400.0, 18.875), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 3600.0, 18.875), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 300.0, 39.0), (utils.datetime_utc(2014, 1, 1, 12, 5, 0), 300.0, 11.0), (utils.datetime_utc(2014, 1, 1, 12, 10, 0), 300.0, 22.0) ], values)
def test_add_measures_big(self): m, __ = self._create_metric('high') self.incoming.add_measures(m, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, i, j), 100) for i in six.moves.range(0, 60) for j in six.moves.range(0, 60)]) self.trigger_processing([str(m.id)]) self.assertEqual(3661, len(self.storage.get_measures(m)))
def test_delete_expunge_metric(self): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), ]) self.trigger_processing() self.index.delete_metric(self.metric.id) self.storage.expunge_metrics(self.incoming, self.index, sync=True) self.assertRaises(indexer.NoSuchMetric, self.index.delete_metric, self.metric.id)
def test_updated_measures(self): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), ]) self.trigger_processing() self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 55.5), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 55.5), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69), (utils.datetime_utc(2014, 1, 1, 12, 5), 300.0, 42.0), ], self.storage.get_measures(self.metric)) self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.trigger_processing() self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), (utils.datetime_utc(2014, 1, 1, 12, 5), 300.0, 23.0), (utils.datetime_utc(2014, 1, 1, 12, 10), 300.0, 44.0), ], self.storage.get_measures(self.metric)) self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 69), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 69.0), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), (utils.datetime_utc(2014, 1, 1, 12, 5), 300.0, 42.0), (utils.datetime_utc(2014, 1, 1, 12, 10), 300.0, 44.0), ], self.storage.get_measures(self.metric, aggregation='max')) self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 4), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 4), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), (utils.datetime_utc(2014, 1, 1, 12, 5), 300.0, 4.0), (utils.datetime_utc(2014, 1, 1, 12, 10), 300.0, 44.0), ], self.storage.get_measures(self.metric, aggregation='min'))
def test_measures_reporting(self): m2, __ = self._create_metric('medium') for i in six.moves.range(60): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, i), 69), ]) self.incoming.add_measures(m2, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, i), 69), ]) report = self.incoming.measures_report(True) self.assertIsInstance(report, dict) self.assertEqual(2, report['summary']['metrics']) self.assertEqual(120, report['summary']['measures']) self.assertIn('details', report) self.assertIsInstance(report['details'], dict) report = self.incoming.measures_report(False) self.assertIsInstance(report, dict) self.assertEqual(2, report['summary']['metrics']) self.assertEqual(120, report['summary']['measures'])
def test_add_measures_update_subset(self): m, m_sql = self._create_metric('medium') measures = [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 6, i, j, 0), 100) for i in six.moves.range(2) for j in six.moves.range(0, 60, 2)] self.incoming.add_measures(m, measures) self.trigger_processing([str(m.id)]) # add measure to end, in same aggregate time as last point. new_point = utils.dt_to_unix_ns(2014, 1, 6, 1, 58, 1) self.incoming.add_measures( m, [storage.Measure(new_point, 100)]) with mock.patch.object(self.incoming, 'add_measures') as c: self.trigger_processing([str(m.id)]) for __, args, __ in c.mock_calls: self.assertEqual( list(args[3])[0][0], carbonara.round_timestamp( new_point, args[1].granularity * 10e8))
def test_delete_nonempty_metric_unprocessed(self): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), ]) self.index.delete_metric(self.metric.id) self.trigger_processing() __, __, details = self.incoming._build_report(True) self.assertIn(str(self.metric.id), details) self.storage.expunge_metrics(self.incoming, self.index, sync=True) __, __, details = self.incoming._build_report(True) self.assertNotIn(str(self.metric.id), details)
def test_delete_nonempty_metric(self): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), ]) self.trigger_processing() self.storage.delete_metric(self.incoming, self.metric, sync=True) self.trigger_processing() self.assertEqual([], self.storage.get_measures(self.metric)) self.assertRaises(storage.MetricDoesNotExist, self.storage._get_unaggregated_timeserie, self.metric)
def test_list_metric_with_measures_to_process(self): metrics = tests_utils.list_all_incoming_metrics(self.incoming) self.assertEqual(set(), metrics) self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), ]) metrics = tests_utils.list_all_incoming_metrics(self.incoming) self.assertEqual(set([str(self.metric.id)]), metrics) self.trigger_processing() metrics = tests_utils.list_all_incoming_metrics(self.incoming) self.assertEqual(set([]), metrics)
def test_search_value(self): metric2, __ = self._create_metric() self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns( 2014, 1, 1, 12, 0, 1, ), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 5, 31), 8), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 42), ]) self.incoming.add_measures(metric2, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 5), 9), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 2), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 6), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 13, 10), 2), ]) self.trigger_processing([str(self.metric.id), str(metric2.id)]) self.assertEqual( { metric2: [], self.metric: [(utils.datetime_utc(2014, 1, 1), 86400, 33), (utils.datetime_utc(2014, 1, 1, 12), 3600, 33), (utils.datetime_utc(2014, 1, 1, 12), 300, 69), (utils.datetime_utc(2014, 1, 1, 12, 10), 300, 42) ] }, self.storage.search_value([metric2, self.metric], {u"≥": 30})) self.assertEqual( { metric2: [], self.metric: [] }, self.storage.search_value([metric2, self.metric], {u"∧": [{ u"eq": 100 }, { u"≠": 50 }]}))
def test_rewrite_measures_corruption_bad_data(self): # Create an archive policy that spans on several splits. Each split # being 3600 points, let's go for 36k points so we have 10 splits. apname = str(uuid.uuid4()) ap = archive_policy.ArchivePolicy(apname, 0, [(36000, 60)]) self.index.create_archive_policy(ap) self.metric = storage.Metric(uuid.uuid4(), ap) self.index.create_metric(self.metric.id, str(uuid.uuid4()), apname) # First store some points scattered across different splits self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2016, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2016, 1, 2, 13, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2016, 1, 4, 14, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2016, 1, 6, 15, 12, 45), 44), ]) self.trigger_processing() splits = {'1451520000.0', '1451736000.0', '1451952000.0'} self.assertEqual( splits, self.storage._list_split_keys_for_metric(self.metric, "mean", 60.0)) if self.storage.WRITE_FULL: assertCompressedIfWriteFull = self.assertTrue else: assertCompressedIfWriteFull = self.assertFalse data = self.storage._get_measures(self.metric, '1451520000.0', "mean", 60.0) self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data)) data = self.storage._get_measures(self.metric, '1451736000.0', "mean", 60.0) self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data)) data = self.storage._get_measures(self.metric, '1451952000.0', "mean", 60.0) assertCompressedIfWriteFull( carbonara.AggregatedTimeSerie.is_compressed(data)) self.assertEqual([ (utils.datetime_utc(2016, 1, 1, 12), 60.0, 69), (utils.datetime_utc(2016, 1, 2, 13, 7), 60.0, 42), (utils.datetime_utc(2016, 1, 4, 14, 9), 60.0, 4), (utils.datetime_utc(2016, 1, 6, 15, 12), 60.0, 44), ], self.storage.get_measures(self.metric, granularity=60.0)) # Test what happens if we write garbage self.storage._store_metric_measures(self.metric, '1451952000.0', "mean", 60.0, b"oh really?") # Now store brand new points that should force a rewrite of one of the # split (keep in mind the back window size in one hour here). We move # the BoundTimeSerie processing timeserie far away from its current # range. self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2016, 1, 10, 16, 18, 45), 45), storage.Measure(utils.dt_to_unix_ns(2016, 1, 10, 17, 12, 45), 46), ]) self.trigger_processing()
def test_delete_old_measures(self): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.trigger_processing() self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), (utils.datetime_utc(2014, 1, 1, 12, 5), 300.0, 23.0), (utils.datetime_utc(2014, 1, 1, 12, 10), 300.0, 44.0), ], self.storage.get_measures(self.metric)) # One year later… self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2015, 1, 1, 12, 0, 1), 69), ]) self.trigger_processing() self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2015, 1, 1), 86400.0, 69), (utils.datetime_utc(2015, 1, 1, 12), 3600.0, 69), (utils.datetime_utc(2015, 1, 1, 12), 300.0, 69), ], self.storage.get_measures(self.metric)) self.assertEqual({"1244160000.0"}, self.storage._list_split_keys_for_metric( self.metric, "mean", 86400.0)) self.assertEqual({"1412640000.0"}, self.storage._list_split_keys_for_metric( self.metric, "mean", 3600.0)) self.assertEqual({"1419120000.0"}, self.storage._list_split_keys_for_metric( self.metric, "mean", 300.0))
def test_resize_policy(self): name = str(uuid.uuid4()) ap = archive_policy.ArchivePolicy(name, 0, [(3, 5)]) self.index.create_archive_policy(ap) m = self.index.create_metric(uuid.uuid4(), str(uuid.uuid4()), name) m = self.index.list_metrics(ids=[m.id])[0] self.incoming.add_measures(m, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 0), 1), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 5), 1), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 10), 1), ]) self.trigger_processing([str(m.id)]) self.assertEqual([ (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 5.0, 1.0), (utils.datetime_utc(2014, 1, 1, 12, 0, 5), 5.0, 1.0), (utils.datetime_utc(2014, 1, 1, 12, 0, 10), 5.0, 1.0), ], self.storage.get_measures(m)) # expand to more points self.index.update_archive_policy( name, [archive_policy.ArchivePolicyItem(granularity=5, points=6)]) m = self.index.list_metrics(ids=[m.id])[0] self.incoming.add_measures(m, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 15), 1), ]) self.trigger_processing([str(m.id)]) self.assertEqual([ (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 5.0, 1.0), (utils.datetime_utc(2014, 1, 1, 12, 0, 5), 5.0, 1.0), (utils.datetime_utc(2014, 1, 1, 12, 0, 10), 5.0, 1.0), (utils.datetime_utc(2014, 1, 1, 12, 0, 15), 5.0, 1.0), ], self.storage.get_measures(m)) # shrink timespan self.index.update_archive_policy( name, [archive_policy.ArchivePolicyItem(granularity=5, points=2)]) m = self.index.list_metrics(ids=[m.id])[0] self.assertEqual([ (utils.datetime_utc(2014, 1, 1, 12, 0, 10), 5.0, 1.0), (utils.datetime_utc(2014, 1, 1, 12, 0, 15), 5.0, 1.0), ], self.storage.get_measures(m))
def test_corrupted_data(self): if not isinstance(self.storage, _carbonara.CarbonaraBasedStorage): self.skipTest("This driver is not based on Carbonara") self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), ]) self.trigger_processing() self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 13, 0, 1), 1), ]) with mock.patch('gnocchi.carbonara.AggregatedTimeSerie.unserialize', side_effect=carbonara.InvalidData()): with mock.patch('gnocchi.carbonara.BoundTimeSerie.unserialize', side_effect=carbonara.InvalidData()): self.trigger_processing() m = self.storage.get_measures(self.metric) self.assertIn((utils.datetime_utc(2014, 1, 1), 86400.0, 1), m) self.assertIn((utils.datetime_utc(2014, 1, 1, 13), 3600.0, 1), m) self.assertIn((utils.datetime_utc(2014, 1, 1, 13), 300.0, 1), m)
def test_add_measures_update_subset_split(self): m, m_sql = self._create_metric('medium') measures = [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 6, i, j, 0), 100) for i in six.moves.range(2) for j in six.moves.range(0, 60, 2) ] self.incoming.add_measures(m, measures) self.trigger_processing([str(m.id)]) # add measure to end, in same aggregate time as last point. self.incoming.add_measures( m, [storage.Measure(utils.dt_to_unix_ns(2014, 1, 6, 1, 58, 1), 100)]) with mock.patch.object(self.storage, '_store_metric_measures') as c: # should only resample last aggregate self.trigger_processing([str(m.id)]) count = 0 for call in c.mock_calls: # policy is 60 points and split is 48. should only update 2nd half args = call[1] if args[0] == m_sql and args[2] == 'mean' and args[3] == 60.0: count += 1 self.assertEqual(1, count)
def test_get_cross_metric_measures_unknown_granularity(self): metric2 = storage.Metric(uuid.uuid4(), self.archive_policies['low']) self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.incoming.add_measures(metric2, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.assertRaises(storage.GranularityDoesNotExist, self.storage.get_cross_metric_measures, [self.metric, metric2], granularity=12345.456)
def test_get_cross_metric_measures_unknown_aggregation(self): metric2 = storage.Metric(uuid.uuid4(), self.archive_policies['low']) self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.incoming.add_measures(metric2, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.assertRaises(storage.AggregationDoesNotExist, self.storage.get_cross_metric_measures, [self.metric, metric2], aggregation='last')
def test_add_and_get_cross_metric_measures_different_archives(self): metric2 = storage.Metric(uuid.uuid4(), self.archive_policies['no_granularity_match']) self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.incoming.add_measures(metric2, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.assertRaises(storage.MetricUnaggregatable, self.storage.get_cross_metric_measures, [self.metric, metric2])
def test_aborted_initial_processing(self): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 5), ]) with mock.patch.object(self.storage, '_store_unaggregated_timeserie', side_effect=Exception): try: self.trigger_processing() except Exception: pass with mock.patch('gnocchi.storage._carbonara.LOG') as LOG: self.trigger_processing() self.assertFalse(LOG.error.called) m = self.storage.get_measures(self.metric) self.assertIn((utils.datetime_utc(2014, 1, 1), 86400.0, 5.0), m) self.assertIn((utils.datetime_utc(2014, 1, 1, 12), 3600.0, 5.0), m) self.assertIn((utils.datetime_utc(2014, 1, 1, 12), 300.0, 5.0), m)
def test_add_and_get_cross_metric_measures(self): metric2, __ = self._create_metric() self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.incoming.add_measures(metric2, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 5), 9), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 41), 2), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 10, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 13, 10), 4), ]) self.trigger_processing([str(self.metric.id), str(metric2.id)]) values = self.storage.get_cross_metric_measures([self.metric, metric2]) self.assertEqual( [(utils.datetime_utc(2014, 1, 1, 0, 0, 0), 86400.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 3600.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 300.0, 39.0), (utils.datetime_utc(2014, 1, 1, 12, 5, 0), 300.0, 12.5), (utils.datetime_utc(2014, 1, 1, 12, 10, 0), 300.0, 24.0)], values) values = self.storage.get_cross_metric_measures([self.metric, metric2], reaggregation='max') self.assertEqual( [(utils.datetime_utc(2014, 1, 1, 0, 0, 0), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 300.0, 69), (utils.datetime_utc(2014, 1, 1, 12, 5, 0), 300.0, 23), (utils.datetime_utc(2014, 1, 1, 12, 10, 0), 300.0, 44)], values) values = self.storage.get_cross_metric_measures( [self.metric, metric2], from_timestamp=datetime.datetime(2014, 1, 1, 12, 10, 0)) self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12, 10, 0), 300.0, 24.0), ], values) values = self.storage.get_cross_metric_measures( [self.metric, metric2], to_timestamp=datetime.datetime(2014, 1, 1, 12, 5, 0)) self.assertEqual([ (utils.datetime_utc(2014, 1, 1, 0, 0, 0), 86400.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 3600.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 300.0, 39.0), ], values) values = self.storage.get_cross_metric_measures( [self.metric, metric2], from_timestamp=datetime.datetime(2014, 1, 1, 12, 10, 10), to_timestamp=datetime.datetime(2014, 1, 1, 12, 10, 10)) self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12, 10), 300.0, 24.0), ], values) values = self.storage.get_cross_metric_measures( [self.metric, metric2], from_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 0), to_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 1)) self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 3600.0, 22.25), (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 300.0, 39.0), ], values) values = self.storage.get_cross_metric_measures( [self.metric, metric2], from_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 0), to_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 1), granularity=300.0) self.assertEqual([ (utils.datetime_utc(2014, 1, 1, 12, 0, 0), 300.0, 39.0), ], values)
def test_add_and_get_measures(self): self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2014, 1, 1, 12, 12, 45), 44), ]) self.trigger_processing() self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), (utils.datetime_utc(2014, 1, 1, 12, 5), 300.0, 23.0), (utils.datetime_utc(2014, 1, 1, 12, 10), 300.0, 44.0), ], self.storage.get_measures(self.metric)) self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12, 10), 300.0, 44.0), ], self.storage.get_measures( self.metric, from_timestamp=datetime.datetime( 2014, 1, 1, 12, 10, 0))) self.assertEqual([ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), (utils.datetime_utc(2014, 1, 1, 12, 5), 300.0, 23.0), ], self.storage.get_measures( self.metric, to_timestamp=datetime.datetime( 2014, 1, 1, 12, 6, 0))) self.assertEqual( [ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12, 10), 300.0, 44.0), ], self.storage.get_measures( self.metric, to_timestamp=datetime.datetime(2014, 1, 1, 12, 10, 10), from_timestamp=datetime.datetime(2014, 1, 1, 12, 10, 10))) self.assertEqual( [ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), ], self.storage.get_measures( self.metric, from_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 0), to_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 2))) self.assertEqual( [ (utils.datetime_utc(2014, 1, 1), 86400.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), ], self.storage.get_measures( self.metric, from_timestamp=iso8601.parse_date("2014-1-1 13:00:00+01:00"), to_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 2))) self.assertEqual( [ (utils.datetime_utc(2014, 1, 1, 12), 3600.0, 39.75), ], self.storage.get_measures( self.metric, from_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 0), to_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 2), granularity=3600.0)) self.assertEqual( [ (utils.datetime_utc(2014, 1, 1, 12), 300.0, 69.0), ], self.storage.get_measures( self.metric, from_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 0), to_timestamp=datetime.datetime(2014, 1, 1, 12, 0, 2), granularity=300.0)) self.assertRaises(storage.GranularityDoesNotExist, self.storage.get_measures, self.metric, granularity=42)
def test_rewrite_measures_oldest_mutable_timestamp_eq_next_key(self): """See LP#1655422""" # Create an archive policy that spans on several splits. Each split # being 3600 points, let's go for 36k points so we have 10 splits. apname = str(uuid.uuid4()) ap = archive_policy.ArchivePolicy(apname, 0, [(36000, 60)]) self.index.create_archive_policy(ap) self.metric = storage.Metric(uuid.uuid4(), ap) self.index.create_metric(self.metric.id, str(uuid.uuid4()), apname) # First store some points scattered across different splits self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2016, 1, 1, 12, 0, 1), 69), storage.Measure(utils.dt_to_unix_ns(2016, 1, 2, 13, 7, 31), 42), storage.Measure(utils.dt_to_unix_ns(2016, 1, 4, 14, 9, 31), 4), storage.Measure(utils.dt_to_unix_ns(2016, 1, 6, 15, 12, 45), 44), ]) self.trigger_processing() splits = {'1451520000.0', '1451736000.0', '1451952000.0'} self.assertEqual( splits, self.storage._list_split_keys_for_metric(self.metric, "mean", 60.0)) if self.storage.WRITE_FULL: assertCompressedIfWriteFull = self.assertTrue else: assertCompressedIfWriteFull = self.assertFalse data = self.storage._get_measures(self.metric, '1451520000.0', "mean", 60.0) self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data)) data = self.storage._get_measures(self.metric, '1451736000.0', "mean", 60.0) self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data)) data = self.storage._get_measures(self.metric, '1451952000.0', "mean", 60.0) assertCompressedIfWriteFull( carbonara.AggregatedTimeSerie.is_compressed(data)) self.assertEqual([ (utils.datetime_utc(2016, 1, 1, 12), 60.0, 69), (utils.datetime_utc(2016, 1, 2, 13, 7), 60.0, 42), (utils.datetime_utc(2016, 1, 4, 14, 9), 60.0, 4), (utils.datetime_utc(2016, 1, 6, 15, 12), 60.0, 44), ], self.storage.get_measures(self.metric, granularity=60.0)) # Now store brand new points that should force a rewrite of one of the # split (keep in mind the back window size in one hour here). We move # the BoundTimeSerie processing timeserie far away from its current # range. # Here we test a special case where the oldest_mutable_timestamp will # be 2016-01-10TOO:OO:OO = 1452384000.0, our new split key. self.incoming.add_measures(self.metric, [ storage.Measure(utils.dt_to_unix_ns(2016, 1, 10, 0, 12), 45), ]) self.trigger_processing() self.assertEqual( {'1452384000.0', '1451736000.0', '1451520000.0', '1451952000.0'}, self.storage._list_split_keys_for_metric(self.metric, "mean", 60.0)) data = self.storage._get_measures(self.metric, '1451520000.0', "mean", 60.0) self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data)) data = self.storage._get_measures(self.metric, '1451736000.0', "mean", 60.0) self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data)) data = self.storage._get_measures(self.metric, '1451952000.0', "mean", 60.0) # Now this one is compressed because it has been rewritten! self.assertTrue(carbonara.AggregatedTimeSerie.is_compressed(data)) data = self.storage._get_measures(self.metric, '1452384000.0', "mean", 60.0) assertCompressedIfWriteFull( carbonara.AggregatedTimeSerie.is_compressed(data)) self.assertEqual([ (utils.datetime_utc(2016, 1, 1, 12), 60.0, 69), (utils.datetime_utc(2016, 1, 2, 13, 7), 60.0, 42), (utils.datetime_utc(2016, 1, 4, 14, 9), 60.0, 4), (utils.datetime_utc(2016, 1, 6, 15, 12), 60.0, 44), (utils.datetime_utc(2016, 1, 10, 0, 12), 60.0, 45), ], self.storage.get_measures(self.metric, granularity=60.0))