def data_get():
""" Get data from graphite """
shinken_contact = current_user.shinken_contact
permissions= utils.get_contact_permissions(shinken_contact)
reg= re.compile(r'\W')
probes = json.loads(request.args.get('probes'))
start = request.args.get('from') or time() - 3600 * 24 * 28
end = request.args.get('until') or time()
start = int(start)
end = int(end)
data = {}
for probe in probes:
#check if the current user is allowed to retreive probe's data
tmp= probe.split('.')
checkHost= next((i for i in permissions['hosts'] if reg.sub('_',i) == tmp[0]), False)
checkService= next((i for i in permissions['services'] if reg.sub('_',i) == tmp[1]), False)
if not checkHost or not checkService:
data[probe] = {
'error': 'Shinken contact %s is not allowed to retreive data from %s.%s'%(shinken_contact,tmp[0],tmp[1]),
'code': 403
}
continue
results= query.query(**{'target': probe, 'from': _format_time(start), 'until': _format_time(end)})
if(len(results)):
data[probe] = _parse_query_result(results[0])
else:
data[probe] = {
'error': 'No data found for %s'%probe,
'code': 404
}
return jsonify(data);
def test_old_whisper_data(self):
wsp_file = os.path.join(settings.WHISPER_DIR, 'test.wsp')
current_time = os.stat(wsp_file).st_mtime
# Now, modify the modification time to be one day old
old_time = current_time - 60*60*24
os.utime(wsp_file, (old_time, old_time))
data = query.query(**{'target': 'test'})
# This test fails on purpose, as it's unsure whether it's a bug or
# a feature
self.assertNotEqual(data, [])
def test_change_STANDARD_DIRS(self):
" A test against a bug that was found in StandardFinder.directories "
# This sets up whatever was the default storage directory layout
# We remove the default storage dir
shutil.rmtree(settings.STORAGE_DIR, ignore_errors=True)
# Now change the storage directory
STORAGE_DIR_new = settings.STORAGE_DIR + '_new'
# Remove if exists
shutil.rmtree(STORAGE_DIR_new, ignore_errors=True)
settings.setup_storage_variables(STORAGE_DIR_new, create_directories=True)
self.populate_data()
data = query.query(**{'target': 'test'})
self.assertTrue(data[0])
def test_generate_and_query_data(self):
data = query.query(**{'target': 'test'})
end = data[0]#[-4:]
match = False
# We iterate through all values and check
# _test_data against 3 consecutive values
# because sometimes whisper adds None
# value(s) to the end (depending on time)
for i, value in enumerate(end):
if value == self._test_data[0]:
self.assertEqual(end[i:i+3], self._test_data)
match = True
break
self.assertTrue(match)
def data_get():
""" Get data from graphite """
shinken_contact = current_user.shinken_contact
permissions = utils.get_contact_permissions(shinken_contact)
probes = json.loads(request.args.get('probes'))
start = request.args.get('from') or time() - 3600 * 24 * 28
end = request.args.get('until') or time()
start = int(start)
end = int(end)
data = {}
separator = getattr(app.config, 'GRAPHITE_SEP', '[SEP]')
for probe in probes:
#check if the current user is allowed to retreive probe's data
tmp = probe.split(separator)
checkHost = next((i for i in permissions['hosts'] if i == tmp[0]),
False)
checkService = next(
(i for i in permissions['services'] if i == tmp[1]), False)
if ('__HOST__' == tmp[1]):
if tmp[0] not in permissions['hosts_with_services']:
checkService = '__HOST__'
if not checkHost or not checkService:
data[probe] = {
'error':
'Shinken contact %s is not allowed to retreive data from %s.%s'
% (shinken_contact, tmp[0], tmp[1]),
'code':
403
}
continue
reg = re.compile(r'\W')
tmp = [reg.sub('_', t) for t in tmp]
results = query.query(
**{
'target': '.'.join(tmp),
'from': _format_time(start),
'until': _format_time(end)
})
if (len(results)):
data[probe] = _parse_query_result(results[0])
else:
data[probe] = {
'error': 'No data found for %s' % probe,
'code': 404
}
return jsonify(data)
def get_graphite_data(metric_name,
from_hours,
remove_nones=True,
expand=False):
"""
Returns the data stored in graphite for the period between now and N hours ago
If expand is True, then the function will add None values to fill areas of time that
are required but not returned by Graphite
If remove_nones is True, the all None values in the series will be replaced by the previous value.
If the first value is None, it will be set to zero.
The function returns a set containing :
- The step interval (in seconds)
- The list of values
If no data was found, the step will be 0 and the values will be an empty list
"""
now = time.time()
results = query.query(target=metric_name,
from_time='-' + str(from_hours) + 'h')
if len(results):
data = results[0].getInfo()['values']
step = results[0].step
end_data = results[0].end
start_data = results[0].start
else:
# TODO : No data found for this target... What do ?
data = []
step = 0
end_data = 0
start_data = 0
if len(data) > 0 and expand:
end_ts = now
start_ts = now - (from_hours * 3600)
start_diff = start_data - start_ts
end_diff = end_ts - end_data
start_diff = int(start_diff / step)
end_diff = int(end_diff / step)
if start_diff > 0:
data = [None for i in xrange(start_diff)] + data
start_data -= (start_diff * step)
if end_diff > 0:
data = data + [None for i in xrange(end_diff)]
end_data += (end_diff * step)
if remove_nones:
# So we have to get sure that no unknown value is left in the data
# TODO : Make sure that the array returned enough values to cover the entire requested period
last_state = 0
for i in xrange(len(data)):
if data[i] is None:
data[i] = last_state
else:
last_state = data[i]
return (start_data, end_data, step, data)
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from graphitequery import query
from datetime import datetime
import argparse
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('path')
args = parser.parse_args()
print 'Fetching path ' + args.path
results = query.query(target=args.path, from_time='-31d')
if len(results) == 0:
print 'Not found'
else:
current = results[0].start
for val in results[0].getInfo()['values']:
current = current + results[0].step
if val:
print "%s %d" % (str(datetime.fromtimestamp(current)), val)
print 'Starting: %s' % str(datetime.fromtimestamp(results[0].start))
print 'Ending: ' + str(datetime.fromtimestamp(results[0].end))
print 'Step: ' + str(results[0].step)
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from graphitequery import query
from datetime import datetime
import argparse
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('path')
args = parser.parse_args()
print 'Fetching path ' + args.path
results = query.query(target=args.path, from_time='-31d')
if len(results) == 0:
print 'Not found'
else:
current = results[0].start
for val in results[0].getInfo()['values']:
current = current + results[0].step
if val:
print "%s %d"%(str(datetime.fromtimestamp(current)),val)
print 'Starting: %s'% str(datetime.fromtimestamp(results[0].start))
print 'Ending: ' + str(datetime.fromtimestamp(results[0].end))
print 'Step: ' + str(results[0].step)
def test_all_None(self):
whisper.create(os.path.join(settings.WHISPER_DIR, 'test_all_None.wsp'), [(1, 60)])
data = query.query(**{'target': 'test_all_None'})
self.assertNotEqual(data, [])
def test_eval_qs(self):
self.assertEqual(query.query('test'),
query.eval_qs('"format=raw&target=test"'))
