def test_fill_should_handle_gaps(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] complete = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 ] holes = [ 1, 2, 3, 4, 5, 6, None, None, None, None, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 ] end = int(time.time()) + schema[0][0] start = end - (schema[0][1] * schema[0][0]) times = range(start, end, schema[0][0]) complete_data = zip(times, complete) holes_data = [t for t in zip(times, holes) if t[1] is not None] self._createdb(self.db, schema, complete_data) self._createdb(testdb, schema, holes_data) fill_archives(self.db, testdb, time.time()) original_data = whisper.fetch(self.db, 0) filled_data = whisper.fetch(testdb, 0) self.assertEqual(original_data, filled_data)
def test_fill_should_handle_gaps(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] complete = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] holes = [1, 2, 3, 4, 5, 6, None, None, None, None, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] end = int(time.time()) + schema[0][0] start = end - (schema[0][1] * schema[0][0]) times = range(start, end, schema[0][0]) complete_data = zip(times, complete) holes_data = [t for t in zip(times, holes) if t[1] is not None] self._createdb(self.db, schema, complete_data) self._createdb(testdb, schema, holes_data) fill_archives(self.db, testdb, time.time()) original_data = whisper.fetch(self.db, 0) filled_data = whisper.fetch(testdb, 0) self.assertEqual(original_data, filled_data)
def test_fill_should_not_override_destination(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] data = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 ] end = int(time.time()) + schema[0][0] start = end - (schema[0][1] * schema[0][0]) times = range(start, end, schema[0][0]) override_data = zip(times, data) emptyData = [data] startTime = time.time() self._createdb(self.db, schema) self._createdb(testdb, schema, override_data) fill_archives(self.db, testdb, startTime) original_data = whisper.fetch(self.db, 0) filled_data = whisper.fetch(testdb, 0) self.assertEqual(data, filled_data[1])
def test_fill_should_not_override_destination(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] end = int(time.time()) + schema[0][0] start = end - (schema[0][1] * schema[0][0]) times = range(start, end, schema[0][0]) override_data = zip(times, data) emptyData = [data] startTime = time.time() self._createdb(self.db, schema) self._createdb(testdb, schema, override_data) fill_archives(self.db, testdb, startTime) original_data = whisper.fetch(self.db, 0) filled_data = whisper.fetch(testdb, 0) self.assertEqual(data, filled_data[1])
def test_fill_should_handle_gaps(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] complete = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] original = [91, # make sure only null values are filled 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, None, # empty value doesn't overwrite 20, ] holes = [1, None, None, # multiple concecutive missing data points 4, None, # single missing data point 6, None, None, None, None, 11, 12, 13, 14, 15, 16, 17, 18, 19, None, # trailing missing data point ] end = int(time.time()) + schema[0][0] start = end - (schema[0][1] * schema[0][0]) times = range(start, end, schema[0][0]) original_data = [t for t in zip(times, original) if t[1] is not None] holes_data = [t for t in zip(times, holes) if t[1] is not None] self._createdb(self.db, schema, original_data) self._createdb(testdb, schema, holes_data) fill_archives(self.db, testdb, time.time()) filled_data = whisper.fetch(testdb, 0) self.assertEqual(complete, filled_data[1])
def test_fill_empty(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] emptyData = [] startTime = time.time() self._createdb(self.db, schema) self._createdb(testdb, schema, emptyData) fill_archives(self.db, testdb, startTime) original_data = whisper.fetch(self.db, 0) filled_data = whisper.fetch(testdb, 0) self.assertEqual(original_data, filled_data)
def whisper_fill(): parser = argparse.ArgumentParser( description='Backfill datapoints from one whisper file into another', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('source', metavar='SRC', help='Whisper source file') parser.add_argument('dest', metavar='DST', help='Whisper destination file') args = parser.parse_args() src = args.source dst = args.dest if not os.path.isfile(src): raise SystemExit('Source file not found.') if not os.path.isfile(dst): raise SystemExit('Destination file not found.') startFrom = time() fill_archives(src, dst, startFrom)
def test_fill_endat(self, unused_mock_time): dst_db = "dst-%s" % self.db self._removedb() try: os.unlink(dst_db) except (IOError, OSError): pass complete = range(1, 21) seconds_per_point = 1 seconds_per_point_l2 = seconds_per_point * 4 points_number = len(complete) schema = [ (seconds_per_point, points_number), (seconds_per_point_l2, points_number), ] empty_data = [] end = int(time.time()) + seconds_per_point start = end - (points_number * seconds_per_point) times = range(start, end, seconds_per_point) complete_data = zip(times, complete) self._createdb(self.db, schema, complete_data) self._createdb(dst_db, schema, empty_data) quarter = points_number // 4 half = points_number // 2 three_quarter = points_number * 3 // 4 # fills a fourth of data, from 2/4th to 3/4th fill_archives(self.db, dst_db, time.time() - quarter, time.time() - half) quarter_filled_data = whisper.fetch(dst_db, start - seconds_per_point)[1] expected = [None] * half + complete[half:three_quarter] + [None ] * quarter self.assertEqual(expected, quarter_filled_data) # Fetching data older than start forces the use of the second level of aggregation # We get a first empty cell and then quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1] average_l1 = _average(quarter_filled_data) average_l2 = _average(quarter_filled_data_l2) self.assertEqual(average_l1, average_l2) # fills a half of data, from 2/4th to 4/4th fill_archives(self.db, dst_db, time.time(), time.time() - half) half_filled_data = whisper.fetch(dst_db, start - seconds_per_point)[1] expected = [None] * half + complete[half:] self.assertEqual(expected, half_filled_data) # Explicitly passes the default value of endAt=now (excluded) fill_archives(self.db, dst_db, time.time(), endAt=0) filled_data = whisper.fetch(dst_db, start - seconds_per_point)[1] self.assertEqual(complete[:-1], filled_data[:-1]) self.assertIsNone(filled_data[-1])
def heal_metric(source, dest): try: with open(dest): try: fill_archives(source, dest, time()) except CorruptWhisperFile as e: logging.warn("Overwriting corrupt file %s!" % dest) try: os.makedirs(os.path.dirname(dest)) except os.error: pass try: shutil.copyfile(source, dest) except IOError as e: logging.warn("Failed to copy %s! %s" % (dest, e)) except IOError: try: os.makedirs(os.path.dirname(dest)) except os.error: pass try: shutil.copyfile(source, dest) except IOError as e: logging.warn("Failed to copy %s! %s" % (dest, e))
def test_fill_endat(self, unused_mock_time): dst_db = "dst-%s" % self.db self._removedb() try: os.unlink(dst_db) except (IOError, OSError): pass complete = range(1, 21) seconds_per_point = 1 seconds_per_point_l2 = seconds_per_point * 4 points_number = len(complete) schema = [(seconds_per_point, points_number), (seconds_per_point_l2, points_number), ] empty_data = [] end = int(time.time()) + seconds_per_point start = end - (points_number * seconds_per_point) times = range(start, end, seconds_per_point) complete_data = zip(times, complete) self._createdb(self.db, schema, complete_data) self._createdb(dst_db, schema, empty_data) quarter = points_number // 4 half = points_number // 2 three_quarter = points_number * 3 // 4 # fills a fourth of data, from 2/4th to 3/4th fill_archives(self.db, dst_db, time.time()-quarter, time.time()-half) quarter_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1] expected = [None]*half + complete[half:three_quarter] + [None]*quarter self.assertEqual(expected, quarter_filled_data) # Fetching data older than start forces the use of the second level of aggregation # We get a first empty cell and then quarter_filled_data_l2 = whisper.fetch(dst_db, 0)[1] average_l1 = _average(quarter_filled_data) average_l2 = _average(quarter_filled_data_l2) self.assertEqual(average_l1, average_l2) # fills a half of data, from 2/4th to 4/4th fill_archives(self.db, dst_db, time.time(), time.time()-half) half_filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1] expected = [None]*half + complete[half:] self.assertEqual(expected, half_filled_data) # Explicitly passes the default value of endAt=now (excluded) fill_archives(self.db, dst_db, time.time(), endAt=0) filled_data = whisper.fetch(dst_db, start-seconds_per_point)[1] self.assertEqual(complete[:-1], filled_data[:-1]) self.assertEqual(filled_data[-1], None)