def get_index(self, requestContext):
matches = []
for root, _, files in walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, '')
for base_name in files:
if fnmatch.fnmatch(base_name, '*.wsp'):
match = join(root, base_name).replace('.wsp', '').replace('/', '.').lstrip('.')
bisect.insort_left(matches, match)
# unlike 0.9.x, we're going to use os.walk with followlinks
# since we require Python 2.7 and newer that supports it
if RRDReader.supported:
for root, _, files in walk(settings.RRD_DIR, followlinks=True):
root = root.replace(settings.RRD_DIR, '')
for base_name in files:
if fnmatch.fnmatch(base_name, '*.rrd'):
absolute_path = join(settings.RRD_DIR, root, base_name)
base_name = splitext(base_name)[0]
metric_path = join(root, base_name)
rrd = RRDReader(absolute_path, metric_path)
for datasource_name in rrd.get_datasources(absolute_path):
match = join(metric_path, datasource_name).replace('.rrd', '').replace('/', '.').lstrip('.')
if match not in matches:
bisect.insort_left(matches, match)
return matches
def get_index(self, requestContext):
matches = []
for root, _, files in walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, '')
for base_name in files:
if fnmatch.fnmatch(base_name, '*.wsp'):
match = join(root, base_name).replace('.wsp', '').replace(
'/', '.').lstrip('.')
bisect.insort_left(matches, match)
# unlike 0.9.x, we're going to use os.walk with followlinks
# since we require Python 2.7 and newer that supports it
if RRDReader.supported:
for root, _, files in walk(settings.RRD_DIR, followlinks=True):
root = root.replace(settings.RRD_DIR, '')
for base_name in files:
if fnmatch.fnmatch(base_name, '*.rrd'):
absolute_path = join(settings.RRD_DIR, root, base_name)
base_name = splitext(base_name)[0]
metric_path = join(root, base_name)
rrd = RRDReader(absolute_path, metric_path)
for datasource_name in rrd.get_datasources(
absolute_path):
match = join(metric_path, datasource_name).replace(
'.rrd', '').replace('/', '.').lstrip('.')
if match not in matches:
bisect.insort_left(matches, match)
return matches
def find_nodes(self, query, reqkey):
log.info("running blablabla RRd")
clean_pattern = query.pattern.replace('\\', '')
pattern_parts = clean_pattern.split('.')
for root_dir in self.directories:
for absolute_path in self._find_paths(root_dir, pattern_parts):
if basename(absolute_path).startswith('.'):
continue
if self.DATASOURCE_DELIMETER in basename(absolute_path):
(absolute_path, datasource_pattern) = absolute_path.rsplit(
self.DATASOURCE_DELIMETER, 1)
else:
datasource_pattern = None
relative_path = absolute_path[len(root_dir):].lstrip('/')
metric_path = fs_to_metric(relative_path)
real_metric_path = get_real_metric_path(
absolute_path, metric_path)
metric_path_parts = metric_path.split('.')
for field_index in find_escaped_pattern_fields(query.pattern):
metric_path_parts[field_index] = pattern_parts[
field_index].replace('\\', '')
metric_path = '.'.join(metric_path_parts)
# Now we construct and yield an appropriate Node object
if isdir(absolute_path):
yield BranchNode(metric_path)
elif isfile(absolute_path):
if absolute_path.endswith(
'.wsp') and WhisperReader.supported:
reader = WhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith(
'.wsp.gz') and GzippedWhisperReader.supported:
reader = GzippedWhisperReader(absolute_path,
real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith(
'.rrd') and RRDReader.supported:
if datasource_pattern is None:
yield BranchNode(metric_path)
else:
for datasource_name in RRDReader.get_datasources(
absolute_path):
if match_entries([datasource_name],
datasource_pattern):
reader = RRDReader(absolute_path,
datasource_name)
yield LeafNode(
metric_path + "." + datasource_name,
reader)
def test_RRDReader_get_intervals(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
reader = RRDReader(self.hostcpu, 'cpu')
# Intervals are calculated on the actual time so tolerate a 2 second
# deviation for delays caused between file creation and test.
for interval in reader.get_intervals():
self.assertAlmostEqual(interval.start,
self.start_ts - self.points * self.step,
delta=2)
self.assertAlmostEqual(interval.end, self.start_ts, delta=2)
def test_RRDReader_get_intervals(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
reader = RRDReader(self.hostcpu, 'cpu')
# Intervals are calculated on the actual time so tolerate a 2 second
# deviation for delays caused between file creation and test.
for interval in reader.get_intervals():
self.assertAlmostEqual(interval.start,
self.start_ts - self.points * self.step,
delta=2)
self.assertAlmostEqual(interval.end, self.start_ts, delta=2)
def test_RRDReader_fetch(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
# insert some data
for ts in range(self.start_ts + 60, self.start_ts + 10 * self.step,
self.step):
rrdtool.update(self.hostcpu, '{}:42'.format(ts))
reader = RRDReader(self.hostcpu, 'cpu')
(time_info, values) = reader.fetch(self.start_ts + self.step,
self.start_ts + self.step * 2)
self.assertEqual(list(values), [42.0])
def test_RRDReader_fetch(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
# insert some data
for ts in range(self.start_ts + 60, self.start_ts + 10 * self.step,
self.step):
rrdtool.update(self.hostcpu, '{}:42'.format(ts))
reader = RRDReader(self.hostcpu, 'cpu')
(time_info, values) = reader.fetch(self.start_ts + self.step,
self.start_ts + self.step * 2)
self.assertEqual(list(values), [42.0])
def find_matches():
matches = []
for root, dirs, files in os.walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.wsp'):
matches.append(os.path.join(root, basename))
for root, dirs, files in os.walk(settings.CERES_DIR):
root = root.replace(settings.CERES_DIR, '')
for filename in files:
if filename == '.ceres-node':
matches.append(root)
# unlike 0.9.x, we're going to use os.walk with followlinks
# since we require Python 2.7 and newer that supports it
if RRDReader.supported:
for root, dirs, files in os.walk(settings.RRD_DIR,
followlinks=True):
root = root.replace(settings.RRD_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.rrd'):
absolute_path = os.path.join(settings.RRD_DIR, root,
basename)
(basename, extension) = os.path.splitext(basename)
metric_path = os.path.join(root, basename)
rrd = RRDReader(absolute_path, metric_path)
for datasource_name in rrd.get_datasources(
absolute_path):
matches.append(
os.path.join(metric_path, datasource_name))
matches = [
m.replace('.wsp', '').replace('.rrd', '').replace('/',
'.').lstrip('.')
for m in sorted(matches)
]
return matches
def find_matches():
matches = []
for root, dirs, files in os.walk(settings.WHISPER_DIR):
root = root.replace(settings.WHISPER_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.wsp'):
matches.append(os.path.join(root, basename))
for root, dirs, files in os.walk(settings.CERES_DIR):
root = root.replace(settings.CERES_DIR, '')
for filename in files:
if filename == '.ceres-node':
matches.append(root)
# unlike 0.9.x, we're going to use os.walk with followlinks
# since we require Python 2.7 and newer that supports it
if RRDReader.supported:
for root, dirs, files in os.walk(settings.RRD_DIR, followlinks=True):
root = root.replace(settings.RRD_DIR, '')
for basename in files:
if fnmatch.fnmatch(basename, '*.rrd'):
absolute_path = os.path.join(settings.RRD_DIR, root, basename)
(basename,extension) = os.path.splitext(basename)
metric_path = os.path.join(root, basename)
rrd = RRDReader(absolute_path, metric_path)
for datasource_name in rrd.get_datasources(absolute_path):
matches.append(os.path.join(metric_path, datasource_name))
matches = [
m
.replace('.wsp', '')
.replace('.rrd', '')
.replace('/', '.')
.lstrip('.')
for m in sorted(matches)
]
return matches
def find_nodes(self, query):
log.info("running blablabla RRd")
clean_pattern = query.pattern.replace('\\', '')
pattern_parts = clean_pattern.split('.')
for root_dir in self.directories:
for absolute_path in self._find_paths(root_dir, pattern_parts):
if basename(absolute_path).startswith('.'):
continue
if self.DATASOURCE_DELIMETER in basename(absolute_path):
(absolute_path, datasource_pattern) = absolute_path.rsplit(self.DATASOURCE_DELIMETER, 1)
else:
datasource_pattern = None
relative_path = absolute_path[ len(root_dir): ].lstrip('/')
metric_path = fs_to_metric(relative_path)
real_metric_path = get_real_metric_path(absolute_path, metric_path)
metric_path_parts = metric_path.split('.')
for field_index in find_escaped_pattern_fields(query.pattern):
metric_path_parts[field_index] = pattern_parts[field_index].replace('\\', '')
metric_path = '.'.join(metric_path_parts)
# Now we construct and yield an appropriate Node object
if isdir(absolute_path):
yield BranchNode(metric_path)
elif isfile(absolute_path):
if absolute_path.endswith('.wsp') and WhisperReader.supported:
reader = WhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.wsp.gz') and GzippedWhisperReader.supported:
reader = GzippedWhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.rrd') and RRDReader.supported:
if datasource_pattern is None:
yield BranchNode(metric_path)
else:
for datasource_name in RRDReader.get_datasources(absolute_path):
if match_entries([datasource_name], datasource_pattern):
reader = RRDReader(absolute_path, datasource_name)
yield LeafNode(metric_path + "." + datasource_name, reader)
def find_nodes(self, query):
clean_pattern = query.pattern.replace('\\', '')
# translate query pattern if it is tagged
tagged = not query.pattern.startswith(
'_tagged.') and ';' in query.pattern
if tagged:
# tagged series are stored in whisper using encoded names, so to retrieve them we need to
# encode the query pattern using the same scheme used in carbon when they are written.
encoded_paths = [
TaggedSeries.encode(query.pattern, sep=os.sep, hash_only=True),
TaggedSeries.encode(query.pattern, sep=os.sep,
hash_only=False),
]
pattern_parts = clean_pattern.split('.')
for root_dir in self.directories:
if tagged:
relative_paths = []
for pattern in encoded_paths:
entries = [
pattern + '.wsp',
pattern + '.wsp.gz',
pattern + '.rrd',
]
for entry in entries:
if isfile(join(root_dir, entry)):
relative_paths.append(entry)
else:
relative_paths = self._find_paths(root_dir, pattern_parts)
for relative_path in relative_paths:
if basename(relative_path).startswith('.'):
continue
if self.DATASOURCE_DELIMITER in basename(relative_path):
(relative_path, datasource_pattern) = relative_path.rsplit(
self.DATASOURCE_DELIMITER, 1)
else:
datasource_pattern = None
absolute_path = join(root_dir, relative_path)
metric_path = fs_to_metric(relative_path)
real_metric_path = get_real_metric_path(
absolute_path, metric_path)
# if we're finding by tag, return the proper metric path
if tagged:
metric_path = query.pattern
else:
metric_path_parts = metric_path.split('.')
for field_index in find_escaped_pattern_fields(
query.pattern):
metric_path_parts[field_index] = pattern_parts[
field_index].replace('\\', '')
metric_path = '.'.join(metric_path_parts)
# Now we construct and yield an appropriate Node object
if isdir(absolute_path):
yield BranchNode(metric_path)
elif absolute_path.endswith(
'.wsp') and WhisperReader.supported:
reader = WhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith(
'.wsp.gz') and GzippedWhisperReader.supported:
reader = GzippedWhisperReader(absolute_path,
real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.rrd') and RRDReader.supported:
if datasource_pattern is None:
yield BranchNode(metric_path)
else:
for datasource_name in RRDReader.get_datasources(
absolute_path):
if match_entries([datasource_name],
datasource_pattern):
reader = RRDReader(absolute_path,
datasource_name)
yield LeafNode(
metric_path + "." + datasource_name,
reader)
def test_RRDReader_get_datasources(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
datasource = RRDReader.get_datasources(self.hostcpu)
self.assertEqual(datasource, ['cpu'])
def test_RRDReader_init(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
reader = RRDReader(self.hostcpu, 'hosts.worker1.cpu')
self.assertIsNotNone(reader)
def test_RRDReader_get_retention(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
retentions = RRDReader.get_retention(self.hostcpu)
self.assertEqual(retentions, self.points * self.step)
def test_RRDReader_convert_fs_path(self):
path = RRDReader._convert_fs_path(six.u(self.hostcpu))
self.assertIsInstance(path, str)
def test_RRDReader_get_retention(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
retentions = RRDReader.get_retention(self.hostcpu)
self.assertEqual(retentions, self.points * self.step)
def test_RRDReader_convert_fs_path(self):
path = RRDReader._convert_fs_path(six.u(self.hostcpu))
self.assertIsInstance(path, str)
def find_nodes(self, query):
clean_pattern = query.pattern.replace('\\', '')
# translate query pattern if it is tagged
tagged = not query.pattern.startswith('_tagged.') and ';' in query.pattern
if tagged:
# tagged series are stored in whisper using encoded names, so to retrieve them we need to encode the
# query pattern using the same scheme used in carbon when they are written.
clean_pattern = TaggedSeries.encode(query.pattern)
pattern_parts = clean_pattern.split('.')
for root_dir in self.directories:
for absolute_path in self._find_paths(root_dir, pattern_parts):
if basename(absolute_path).startswith('.'):
continue
if self.DATASOURCE_DELIMITER in basename(absolute_path):
(absolute_path, datasource_pattern) = absolute_path.rsplit(
self.DATASOURCE_DELIMITER, 1)
else:
datasource_pattern = None
relative_path = absolute_path[len(root_dir):].lstrip('/')
metric_path = fs_to_metric(relative_path)
real_metric_path = get_real_metric_path(
absolute_path, metric_path)
metric_path_parts = metric_path.split('.')
for field_index in find_escaped_pattern_fields(query.pattern):
metric_path_parts[field_index] = pattern_parts[field_index].replace(
'\\', '')
metric_path = '.'.join(metric_path_parts)
# if we're finding by tag, return the proper metric path
if tagged:
metric_path = query.pattern
# Now we construct and yield an appropriate Node object
if isdir(absolute_path):
yield BranchNode(metric_path)
elif isfile(absolute_path):
if absolute_path.endswith(
'.wsp') and WhisperReader.supported:
reader = WhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.wsp.gz') and GzippedWhisperReader.supported:
reader = GzippedWhisperReader(
absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.rrd') and RRDReader.supported:
if datasource_pattern is None:
yield BranchNode(metric_path)
else:
for datasource_name in RRDReader.get_datasources(
absolute_path):
if match_entries(
[datasource_name], datasource_pattern):
reader = RRDReader(
absolute_path, datasource_name)
yield LeafNode(metric_path + "." + datasource_name, reader)
def test_RRDReader_get_datasources(self):
self.create_rrd()
self.addCleanup(self.wipe_rrd)
datasource = RRDReader.get_datasources(self.hostcpu)
self.assertEqual(datasource, ['cpu'])