def test_WhisperReader_fetch(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
(_, values) = reader.fetch(self.start_ts-5, self.start_ts)
self.assertEqual(values, [None, None, None, None, 1.0])
def test_WhisperReader_missing_file(self, log_exception):
path = 'missing/file.wsp'
reader = WhisperReader(path, 'hosts.worker2.cpu')
self.assertEqual(reader.fetch(self.start_ts - 5, self.start_ts), None)
log_exception.assert_called_with("Failed fetch of whisper file '%s'" %
path)
def test_WhisperReader_fetch(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
(_, values) = reader.fetch(self.start_ts - 5, self.start_ts)
self.assertEqual(values, [None, None, None, None, 1.0])
def test_WhisperReader_fetch_returns_no_data(self, whisper_fetch):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
whisper_fetch.return_value = None
self.assertEqual(reader.fetch(self.start_ts - 5, self.start_ts), None)
def test_WhisperReader_fetch_returns_no_data(self, whisper_fetch):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
whisper_fetch.return_value = None
self.assertEqual(reader.fetch(self.start_ts-5, self.start_ts), None)
def test_WhisperReader_get_intervals(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
intervals = reader.get_intervals()
for interval in intervals:
self.assertEqual(int(interval.start), self.start_ts-60)
self.assertEqual(int(interval.end), self.start_ts)
def test_WhisperReader_get_intervals(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
intervals = reader.get_intervals()
for interval in intervals:
self.assertEqual(int(interval.start), self.start_ts - 60)
self.assertEqual(int(interval.end), self.start_ts)
def test_WhisperReader_CarbonLinkQuery(self, carbonlink_query):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
carbonlink_query.return_value = {}
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
(_, values) = reader.fetch(self.start_ts-5, self.start_ts)
self.assertEqual(values, [None, None, None, None, 1.0])
def test_WhisperReader_CarbonLinkQuery(self, carbonlink_query):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
carbonlink_query.return_value = {}
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
(_, values) = reader.fetch(self.start_ts - 5, self.start_ts)
self.assertEqual(values, [None, None, None, None, 1.0])
def test_WhisperReader_get_raw_step(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
raw_step = reader.get_raw_step()
self.assertEqual(int(raw_step), 1)
# read it again to validate cache works
raw_step = reader.get_raw_step()
self.assertEqual(int(raw_step), 1)
def test_WhisperReader_get_default_retention(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
default_retention = reader.get_default_retention()
self.assertEqual(int(default_retention), 1)
# read it again to validate cache works
default_retention = reader.get_default_retention()
self.assertEqual(int(default_retention), 1)
def test_MultiReader_init(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
wr1 = WhisperReader(self.worker1, 'hosts.worker1.cpu')
node1 = LeafNode('hosts.worker1.cpu', wr1)
wr2 = WhisperReader(self.worker2, 'hosts.worker2.cpu')
node2 = LeafNode('hosts.worker2.cpu', wr2)
reader = MultiReader([node1, node2])
self.assertIsNotNone(reader)
def test_WhisperReader_broken_file(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
# Test broken whisper file
f = open(self.worker2, 'rb+')
f.seek(10)
f.write(b'Bad Data')
f.close()
reader = WhisperReader(self.worker2, 'hosts.worker2.cpu')
with self.assertRaises(Exception):
reader.fetch(self.start_ts - 5, self.start_ts)
def test_WhisperReader_broken_file(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
# Test broken whisper file
f = open(self.worker2, 'rb+')
f.seek(10)
f.write('Bad Data')
f.close()
reader = WhisperReader(self.worker2, 'hosts.worker2.cpu')
with self.assertRaises(Exception):
reader.fetch(self.start_ts-5, self.start_ts)
def test_MultiReader_get_intervals(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
wr1 = WhisperReader(self.worker1, 'hosts.worker1.cpu')
node1 = LeafNode('hosts.worker1.cpu', wr1)
wr2 = WhisperReader(self.worker2, 'hosts.worker2.cpu')
node2 = LeafNode('hosts.worker2.cpu', wr2)
reader = MultiReader([node1, node2])
intervals = reader.get_intervals()
for interval in intervals:
self.assertEqual(int(interval.start), self.start_ts - 60)
self.assertEqual(int(interval.end), self.start_ts)
def test_WhisperReader_get_intervals(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
ts = int(time.time())
intervals = reader.get_intervals()
for interval in intervals:
self.assertEqual(int(interval.start), ts - 60)
self.assertEqual(int(interval.end), ts)
# read it again to validate cache works
intervals = reader.get_intervals()
for interval in intervals:
self.assertEqual(int(interval.start), ts - 60)
self.assertEqual(int(interval.end), ts)
def test_MultiReader_fetch(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
wr1 = WhisperReader(self.worker1, 'hosts.worker1.cpu')
node1 = LeafNode('hosts.worker1.cpu', wr1)
wr2 = WhisperReader(self.worker2, 'hosts.worker2.cpu')
node2 = LeafNode('hosts.worker2.cpu', wr2)
reader = MultiReader([node1, node2])
results = reader.fetch(self.start_ts - 5, self.start_ts)
(_, values) = results
self.assertEqual(values, [None, None, None, None, 1.0])
def test_MultiReader_merge_normal(self):
results1 = ((1496252939, 1496252944, 1), [None, None, None, None, 1.0])
results2 = ((1496252939, 1496252944, 1), [1.0, 1.0, 1.0, 1.0, 1.0])
wr1 = WhisperReader(self.worker1, 'hosts.worker1.cpu')
node1 = LeafNode('hosts.worker1.cpu', wr1)
reader = MultiReader([node1])
(_, values) = reader.merge(results1, results2)
self.assertEqual(values, [1.0, 1.0, 1.0, 1.0, 1.0])
def find_nodes(self, query, reqkey):
log.info("running blablabla RRd")
clean_pattern = query.pattern.replace('\\', '')
pattern_parts = clean_pattern.split('.')
for root_dir in self.directories:
for absolute_path in self._find_paths(root_dir, pattern_parts):
if basename(absolute_path).startswith('.'):
continue
if self.DATASOURCE_DELIMETER in basename(absolute_path):
(absolute_path, datasource_pattern) = absolute_path.rsplit(
self.DATASOURCE_DELIMETER, 1)
else:
datasource_pattern = None
relative_path = absolute_path[len(root_dir):].lstrip('/')
metric_path = fs_to_metric(relative_path)
real_metric_path = get_real_metric_path(
absolute_path, metric_path)
metric_path_parts = metric_path.split('.')
for field_index in find_escaped_pattern_fields(query.pattern):
metric_path_parts[field_index] = pattern_parts[
field_index].replace('\\', '')
metric_path = '.'.join(metric_path_parts)
# Now we construct and yield an appropriate Node object
if isdir(absolute_path):
yield BranchNode(metric_path)
elif isfile(absolute_path):
if absolute_path.endswith(
'.wsp') and WhisperReader.supported:
reader = WhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith(
'.wsp.gz') and GzippedWhisperReader.supported:
reader = GzippedWhisperReader(absolute_path,
real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith(
'.rrd') and RRDReader.supported:
if datasource_pattern is None:
yield BranchNode(metric_path)
else:
for datasource_name in RRDReader.get_datasources(
absolute_path):
if match_entries([datasource_name],
datasource_pattern):
reader = RRDReader(absolute_path,
datasource_name)
yield LeafNode(
metric_path + "." + datasource_name,
reader)
def test_WhisperReader_init(self):
self.create_whisper_hosts()
self.addCleanup(self.wipe_whisper_hosts)
reader = WhisperReader(self.worker1, 'hosts.worker1.cpu')
self.assertIsNotNone(reader)
def find_nodes(self, query):
clean_pattern = query.pattern.replace('\\', '')
# translate query pattern if it is tagged
tagged = not query.pattern.startswith(
'_tagged.') and ';' in query.pattern
if tagged:
# tagged series are stored in whisper using encoded names, so to retrieve them we need to
# encode the query pattern using the same scheme used in carbon when they are written.
encoded_paths = [
TaggedSeries.encode(query.pattern, sep=os.sep, hash_only=True),
TaggedSeries.encode(query.pattern, sep=os.sep,
hash_only=False),
]
pattern_parts = clean_pattern.split('.')
for root_dir in self.directories:
if tagged:
relative_paths = []
for pattern in encoded_paths:
entries = [
pattern + '.wsp',
pattern + '.wsp.gz',
pattern + '.rrd',
]
for entry in entries:
if isfile(join(root_dir, entry)):
relative_paths.append(entry)
else:
relative_paths = self._find_paths(root_dir, pattern_parts)
for relative_path in relative_paths:
if basename(relative_path).startswith('.'):
continue
if self.DATASOURCE_DELIMITER in basename(relative_path):
(relative_path, datasource_pattern) = relative_path.rsplit(
self.DATASOURCE_DELIMITER, 1)
else:
datasource_pattern = None
absolute_path = join(root_dir, relative_path)
metric_path = fs_to_metric(relative_path)
real_metric_path = get_real_metric_path(
absolute_path, metric_path)
# if we're finding by tag, return the proper metric path
if tagged:
metric_path = query.pattern
else:
metric_path_parts = metric_path.split('.')
for field_index in find_escaped_pattern_fields(
query.pattern):
metric_path_parts[field_index] = pattern_parts[
field_index].replace('\\', '')
metric_path = '.'.join(metric_path_parts)
# Now we construct and yield an appropriate Node object
if isdir(absolute_path):
yield BranchNode(metric_path)
elif absolute_path.endswith(
'.wsp') and WhisperReader.supported:
reader = WhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith(
'.wsp.gz') and GzippedWhisperReader.supported:
reader = GzippedWhisperReader(absolute_path,
real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.rrd') and RRDReader.supported:
if datasource_pattern is None:
yield BranchNode(metric_path)
else:
for datasource_name in RRDReader.get_datasources(
absolute_path):
if match_entries([datasource_name],
datasource_pattern):
reader = RRDReader(absolute_path,
datasource_name)
yield LeafNode(
metric_path + "." + datasource_name,
reader)