def test_heal_mixed_data(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] have = [ 1, 2, 3, None, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, None, 17, 18, 19, None ] remote = [ 1, 2, 3, 4, 5, 6, None, None, None, None, 11, 12, 13, 14, 15, 16, 17, 18, None, 20 ] end = int(time.time()) + schema[0][0] start = end - (schema[0][1] * schema[0][0]) times = list(range(start, end, schema[0][0])) have_data = [t for t in zip(times, have) if t[1] is not None] remote_data = [t for t in zip(times, remote) if t[1] is not None] self._createdb(self.db, schema, remote_data) self._createdb(testdb, schema, have_data) heal_metric(self.db, testdb, overwrite=True) final_data = whisper.fetch(testdb, 0) self.assertEqual(final_data[1], list(range(1, 21)))
def test_heal_mixed_data(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] have = [1, 2, 3, None, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, None, 17, 18, 19, None] remote = [1, 2, 3, 4, 5, 6, None, None, None, None, 11, 12, 13, 14, 15, 16, 17, 18, None, 20] end = int(time.time()) + schema[0][0] start = end - (schema[0][1] * schema[0][0]) times = range(start, end, schema[0][0]) have_data = [t for t in zip(times, have) if t[1] is not None] remote_data = [t for t in zip(times, remote) if t[1] is not None] self._createdb(self.db, schema, remote_data) self._createdb(testdb, schema, have_data) heal_metric(self.db, testdb, overwrite=True) final_data = whisper.fetch(testdb, 0) self.assertEqual(final_data[1], list(range(1,21)))
def test_heal_target_corrupt(self): testdb = "/dev/null" self._removedb() schema = [(1, 20)] self._createdb(self.db, schema) original_data = whisper.fetch(self.db, 0) # This should log complaints but exit successfully as it cannot # heal its target /dev/null heal_metric(self.db, testdb) data = whisper.fetch(self.db, 0) self.assertEqual(original_data, data)
def test_heal_target_missing(self): testdb = "test-%s" % self.db try: os.unlink(testdb) except (IOError, OSError): pass self._removedb() schema = [(1, 20)] self._createdb(self.db, schema) original_data = whisper.fetch(self.db, 0) # This should log complaints but exit successfully as it cannot # heal its target /dev/null heal_metric(self.db, testdb) data = whisper.fetch(testdb, 0) self.assertEqual(original_data, data)
def _heal(batch): """Heal whisper files. This method will backfill data present in files in the staging dir if not present in the local files for points between 'start' and 'stop' (unix timestamps). """ for metric in batch.metrics_fs: src = os.path.join(batch.staging_dir, metric) dst = os.path.join(STORAGE_DIR, metric) try: kwargs = {} if HAVE_HEAL_WITH_TIME_RANGE: kwargs['start_time'] = batch.start_time kwargs['end_time'] = batch.end_time if HAVE_HEAL_WITH_OVERWRITE: kwargs['overwrite'] = batch.overwrite carbonate_sync.heal_metric(src, dst, **kwargs) except Exception as e: logging.exception("Failed to heal %s" % dst) return batch
def test_heal_empty(self): testdb = "test-%s" % self.db self._removedb() try: os.unlink(testdb) except (IOError, OSError): pass schema = [(1, 20)] emptyData = [] self._createdb(self.db, schema) self._createdb(testdb, schema, emptyData) heal_metric(self.db, testdb) original_data = whisper.fetch(self.db, 0) filled_data = whisper.fetch(testdb, 0) self.assertEqual(original_data, filled_data) # Heal again, should still be equal heal_metric(self.db, testdb) filled_data = whisper.fetch(testdb, 0) self.assertEqual(original_data, filled_data)