def test_WhisperReader_fetch(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') (_, values) = reader.fetch(self.start_ts-5, self.start_ts) self.assertEqual(values, [None, None, None, None, 1.0])
def test_WhisperReader_missing_file(self, log_exception): path = 'missing/file.wsp' reader = WhisperReader(path, 'hosts.worker2.cpu') self.assertEqual(reader.fetch(self.start_ts - 5, self.start_ts), None) log_exception.assert_called_with("Failed fetch of whisper file '%s'" % path)
def test_WhisperReader_fetch(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') (_, values) = reader.fetch(self.start_ts - 5, self.start_ts) self.assertEqual(values, [None, None, None, None, 1.0])
def test_WhisperReader_fetch_returns_no_data(self, whisper_fetch): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') whisper_fetch.return_value = None self.assertEqual(reader.fetch(self.start_ts - 5, self.start_ts), None)
def test_WhisperReader_fetch_returns_no_data(self, whisper_fetch): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') whisper_fetch.return_value = None self.assertEqual(reader.fetch(self.start_ts-5, self.start_ts), None)
def test_WhisperReader_get_intervals(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') intervals = reader.get_intervals() for interval in intervals: self.assertEqual(int(interval.start), self.start_ts-60) self.assertEqual(int(interval.end), self.start_ts)
def test_WhisperReader_get_intervals(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') intervals = reader.get_intervals() for interval in intervals: self.assertEqual(int(interval.start), self.start_ts - 60) self.assertEqual(int(interval.end), self.start_ts)
def test_WhisperReader_CarbonLinkQuery(self, carbonlink_query): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) carbonlink_query.return_value = {} reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') (_, values) = reader.fetch(self.start_ts-5, self.start_ts) self.assertEqual(values, [None, None, None, None, 1.0])
def test_WhisperReader_CarbonLinkQuery(self, carbonlink_query): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) carbonlink_query.return_value = {} reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') (_, values) = reader.fetch(self.start_ts - 5, self.start_ts) self.assertEqual(values, [None, None, None, None, 1.0])
def test_WhisperReader_get_raw_step(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') raw_step = reader.get_raw_step() self.assertEqual(int(raw_step), 1) # read it again to validate cache works raw_step = reader.get_raw_step() self.assertEqual(int(raw_step), 1)
def test_WhisperReader_get_default_retention(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') default_retention = reader.get_default_retention() self.assertEqual(int(default_retention), 1) # read it again to validate cache works default_retention = reader.get_default_retention() self.assertEqual(int(default_retention), 1)
def test_MultiReader_init(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) wr1 = WhisperReader(self.worker1, 'hosts.worker1.cpu') node1 = LeafNode('hosts.worker1.cpu', wr1) wr2 = WhisperReader(self.worker2, 'hosts.worker2.cpu') node2 = LeafNode('hosts.worker2.cpu', wr2) reader = MultiReader([node1, node2]) self.assertIsNotNone(reader)
def test_WhisperReader_broken_file(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) # Test broken whisper file f = open(self.worker2, 'rb+') f.seek(10) f.write(b'Bad Data') f.close() reader = WhisperReader(self.worker2, 'hosts.worker2.cpu') with self.assertRaises(Exception): reader.fetch(self.start_ts - 5, self.start_ts)
def test_WhisperReader_broken_file(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) # Test broken whisper file f = open(self.worker2, 'rb+') f.seek(10) f.write('Bad Data') f.close() reader = WhisperReader(self.worker2, 'hosts.worker2.cpu') with self.assertRaises(Exception): reader.fetch(self.start_ts-5, self.start_ts)
def test_MultiReader_get_intervals(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) wr1 = WhisperReader(self.worker1, 'hosts.worker1.cpu') node1 = LeafNode('hosts.worker1.cpu', wr1) wr2 = WhisperReader(self.worker2, 'hosts.worker2.cpu') node2 = LeafNode('hosts.worker2.cpu', wr2) reader = MultiReader([node1, node2]) intervals = reader.get_intervals() for interval in intervals: self.assertEqual(int(interval.start), self.start_ts - 60) self.assertEqual(int(interval.end), self.start_ts)
def test_WhisperReader_get_intervals(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') ts = int(time.time()) intervals = reader.get_intervals() for interval in intervals: self.assertEqual(int(interval.start), ts - 60) self.assertEqual(int(interval.end), ts) # read it again to validate cache works intervals = reader.get_intervals() for interval in intervals: self.assertEqual(int(interval.start), ts - 60) self.assertEqual(int(interval.end), ts)
def test_MultiReader_fetch(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) wr1 = WhisperReader(self.worker1, 'hosts.worker1.cpu') node1 = LeafNode('hosts.worker1.cpu', wr1) wr2 = WhisperReader(self.worker2, 'hosts.worker2.cpu') node2 = LeafNode('hosts.worker2.cpu', wr2) reader = MultiReader([node1, node2]) results = reader.fetch(self.start_ts - 5, self.start_ts) (_, values) = results self.assertEqual(values, [None, None, None, None, 1.0])
def test_MultiReader_merge_normal(self): results1 = ((1496252939, 1496252944, 1), [None, None, None, None, 1.0]) results2 = ((1496252939, 1496252944, 1), [1.0, 1.0, 1.0, 1.0, 1.0]) wr1 = WhisperReader(self.worker1, 'hosts.worker1.cpu') node1 = LeafNode('hosts.worker1.cpu', wr1) reader = MultiReader([node1]) (_, values) = reader.merge(results1, results2) self.assertEqual(values, [1.0, 1.0, 1.0, 1.0, 1.0])
def find_nodes(self, query, reqkey): log.info("running blablabla RRd") clean_pattern = query.pattern.replace('\\', '') pattern_parts = clean_pattern.split('.') for root_dir in self.directories: for absolute_path in self._find_paths(root_dir, pattern_parts): if basename(absolute_path).startswith('.'): continue if self.DATASOURCE_DELIMETER in basename(absolute_path): (absolute_path, datasource_pattern) = absolute_path.rsplit( self.DATASOURCE_DELIMETER, 1) else: datasource_pattern = None relative_path = absolute_path[len(root_dir):].lstrip('/') metric_path = fs_to_metric(relative_path) real_metric_path = get_real_metric_path( absolute_path, metric_path) metric_path_parts = metric_path.split('.') for field_index in find_escaped_pattern_fields(query.pattern): metric_path_parts[field_index] = pattern_parts[ field_index].replace('\\', '') metric_path = '.'.join(metric_path_parts) # Now we construct and yield an appropriate Node object if isdir(absolute_path): yield BranchNode(metric_path) elif isfile(absolute_path): if absolute_path.endswith( '.wsp') and WhisperReader.supported: reader = WhisperReader(absolute_path, real_metric_path) yield LeafNode(metric_path, reader) elif absolute_path.endswith( '.wsp.gz') and GzippedWhisperReader.supported: reader = GzippedWhisperReader(absolute_path, real_metric_path) yield LeafNode(metric_path, reader) elif absolute_path.endswith( '.rrd') and RRDReader.supported: if datasource_pattern is None: yield BranchNode(metric_path) else: for datasource_name in RRDReader.get_datasources( absolute_path): if match_entries([datasource_name], datasource_pattern): reader = RRDReader(absolute_path, datasource_name) yield LeafNode( metric_path + "." + datasource_name, reader)
def test_WhisperReader_init(self): self.create_whisper_hosts() self.addCleanup(self.wipe_whisper_hosts) reader = WhisperReader(self.worker1, 'hosts.worker1.cpu') self.assertIsNotNone(reader)
def find_nodes(self, query): clean_pattern = query.pattern.replace('\\', '') # translate query pattern if it is tagged tagged = not query.pattern.startswith( '_tagged.') and ';' in query.pattern if tagged: # tagged series are stored in whisper using encoded names, so to retrieve them we need to # encode the query pattern using the same scheme used in carbon when they are written. encoded_paths = [ TaggedSeries.encode(query.pattern, sep=os.sep, hash_only=True), TaggedSeries.encode(query.pattern, sep=os.sep, hash_only=False), ] pattern_parts = clean_pattern.split('.') for root_dir in self.directories: if tagged: relative_paths = [] for pattern in encoded_paths: entries = [ pattern + '.wsp', pattern + '.wsp.gz', pattern + '.rrd', ] for entry in entries: if isfile(join(root_dir, entry)): relative_paths.append(entry) else: relative_paths = self._find_paths(root_dir, pattern_parts) for relative_path in relative_paths: if basename(relative_path).startswith('.'): continue if self.DATASOURCE_DELIMITER in basename(relative_path): (relative_path, datasource_pattern) = relative_path.rsplit( self.DATASOURCE_DELIMITER, 1) else: datasource_pattern = None absolute_path = join(root_dir, relative_path) metric_path = fs_to_metric(relative_path) real_metric_path = get_real_metric_path( absolute_path, metric_path) # if we're finding by tag, return the proper metric path if tagged: metric_path = query.pattern else: metric_path_parts = metric_path.split('.') for field_index in find_escaped_pattern_fields( query.pattern): metric_path_parts[field_index] = pattern_parts[ field_index].replace('\\', '') metric_path = '.'.join(metric_path_parts) # Now we construct and yield an appropriate Node object if isdir(absolute_path): yield BranchNode(metric_path) elif absolute_path.endswith( '.wsp') and WhisperReader.supported: reader = WhisperReader(absolute_path, real_metric_path) yield LeafNode(metric_path, reader) elif absolute_path.endswith( '.wsp.gz') and GzippedWhisperReader.supported: reader = GzippedWhisperReader(absolute_path, real_metric_path) yield LeafNode(metric_path, reader) elif absolute_path.endswith('.rrd') and RRDReader.supported: if datasource_pattern is None: yield BranchNode(metric_path) else: for datasource_name in RRDReader.get_datasources( absolute_path): if match_entries([datasource_name], datasource_pattern): reader = RRDReader(absolute_path, datasource_name) yield LeafNode( metric_path + "." + datasource_name, reader)