def test_input_parameter_error_with_panel_id(self):
e = InputParameterError('exception details')
e.setSourceIdHeaders({'X-PANEL-ID': 12})
self.assertEqual(
str(e),
'Invalid parameters (exception details); source: "X-PANEL-ID: 12"',
)
def test_input_parameter_error_with_grafana_org_id(self):
e = InputParameterError('exception details')
e.setSourceIdHeaders({'X-GRAFANA-ORG-ID': 3})
self.assertEqual(
str(e),
'Invalid parameters (exception details); source: "X-GRAFANA-ORG-ID: 3"',
)
def test_input_parameter_error_with_dashboard_id(self):
e = InputParameterError('exception details')
e.setSourceIdHeaders({'X-DASHBOARD-ID': 'abcde123'})
self.assertEqual(
str(e),
'Invalid parameters (exception details); source: "X-DASHBOARD-ID: abcde123"',
)
def test_input_parameter_error_with_multiple_targets(self):
e = InputParameterError('exception details')
e.setTargets(['some_func(a.b.c)', 'otherfunc(c.b.a)'])
self.assertEqual(
str(e),
'Invalid parameters (exception details); targets: "some_func(a.b.c), otherfunc(c.b.a)"',
)
def validateParams(func, params, args, kwargs):
valid_args = []
if len(params) == 0 or params[len(params) - 1].multiple is False:
if len(args) + len(kwargs) > len(params):
raise InputParameterError(
'Too many parameters specified for function "{func}"'.format(
func=func))
for i in range(len(params)):
if len(args) <= i:
# requirement is satisfied from "kwargs"
value = kwargs.get(params[i].name, None)
if value is None:
if params[i].required:
# required parameter is missing
raise InputParameterError(
'Missing required parameter "{param}" for function "{func}"'
.format(param=params[i].name, func=func))
else:
# got multiple values for keyword argument
if params[i].name in valid_args:
raise InputParameterError(
'Keyword parameter "{param}" specified multiple times for function "{func}"'
.format(param=params[i].name, func=func))
else:
# requirement is satisfied from "args"
value = args[i]
params[i].validateValue(value, func)
valid_args.append(params[i].name)
return True
def test_input_parameter_error_with_func_and_args(self):
e = InputParameterError('exception details')
e.setFunction('test_func', [1, 'a'], {})
self.assertEqual(
str(e),
'Invalid parameters (exception details); func: "test_func(1, \'a\')"',
)
def test_input_parameter_error_with_func_args_kwargs(self):
e = InputParameterError('exception details')
e.setFunction('test_func', [3, 'd'], {'a': 1, 'b': 'b'})
self.assertEqual(
str(e),
'Invalid parameters (exception details); func: "test_func(3, \'d\', a=1, b=\'b\')"',
)
def validateValue(self, value, func):
# if value isn't specified and there's a default then the default will be used,
# we don't need to validate the default value because we trust that it is valid
if value is None and self.default is not None:
return True
# None is ok for optional params
if not self.required and value is None:
return True
# parameter is restricted to a defined set of values, but value is not in it
if self.options and value not in self.options:
raise InputParameterError(
'Invalid option specified for function "{func}" parameter "{param}": {value}'
.format(func=func, param=self.name, value=repr(value)))
if not self.type.isValid(value):
raise InputParameterError(
'Invalid "{type}" value specified for function "{func}" parameter "{param}": {value}'
.format(type=self.type.name,
func=func,
param=self.name,
value=repr(value)))
return True
def validateParams(func, params, args, kwargs):
valid_args = []
valid_kwargs = {}
# total number of args + kwargs might be larger than number of params if
# the last param allows to be specified multiple times
if len(args) + len(kwargs) > len(params):
if not params[-1].multiple:
raise InputParameterError(
'Too many parameters specified for function "{func}"'.format(func=func))
# if args has more values than params and the last param allows multiple values,
# then we're going to validate all paramas from the args value list
args_params = params
kwargs_params = []
else:
# take the first len(args) params and use them to validate the args values,
# use the remaining params to validate the kwargs values
args_params = params[:len(args)]
kwargs_params = params[len(args):]
# validate the args
for (i, arg) in enumerate(args):
if i >= len(args_params):
# last parameter must be allowing multiple,
# so we're using it to validate this arg
param = args_params[-1]
else:
param = args_params[i]
valid_args.append(param.validateValue(arg, func))
# validate the kwargs
for param in kwargs_params:
value = kwargs.get(param.name, None)
if value is None:
if param.required:
raise InputParameterError(
'Missing required parameter "{param}" for function "{func}"'
.format(param=param.name, func=func))
continue
valid_kwargs[param.name] = param.validateValue(value, func)
if len(kwargs) > len(valid_kwargs):
unexpected_keys = []
for name in kwargs.keys():
if name not in valid_kwargs:
unexpected_keys.append(name)
raise InputParameterError(
'Unexpected key word arguments: {keys}'.format(
keys=', '.join(
key
for key in kwargs.keys()
if key not in valid_kwargs.keys()
)))
return (valid_args, valid_kwargs)
def getAggFunc(func, rawFunc=None):
if func in aggFuncs:
return aggFuncs[func]
if func in aggFuncAliases:
return aggFuncAliases[func]
raise InputParameterError('Unsupported aggregation function: %s' %
(rawFunc or func))
def validateParams(func, params, args, kwargs):
valid_args = []
if len(params) == 0 or params[len(params) - 1].multiple is False:
if len(args) + len(kwargs) > len(params):
raise InputParameterError(
'Too many parameters specified for function "{func}"'.format(
func=func))
for i in range(len(params)):
if len(args) <= i:
# requirement is satisfied from "kwargs"
value = kwargs.get(params[i].name, None)
if value is None:
if params[i].required:
# required parameter is missing
raise InputParameterError(
'Missing required parameter "{param}" for function "{func}"'
.format(param=params[i].name, func=func))
else:
# got multiple values for keyword argument
if params[i].name in valid_args:
raise InputParameterError(
'Keyword parameter "{param}" specified multiple times for function "{func}"'
.format(param=params[i].name, func=func))
else:
# requirement is satisfied from "args"
value = args[i]
# parameter is restricted to a defined set of values, but value is not in it
if params[i].options and value not in params[i].options:
raise InputParameterError(
'Invalid option "{value}" specified for function "{func}" parameter "{param}"'
.format(value=value, param=params[i].name, func=func))
if not params[i].validateValue(value):
raise InputParameterError(
'Invalid {type} value specified for function "{func}" parameter "{param}": {value}'
.format(type=params[i].type.name,
func=func,
param=params[i].name,
value=value))
valid_args.append(params[i].name)
return True
def queryParamAsInt(queryParams, name, default):
if name not in queryParams:
return default
try:
return int(queryParams[name])
except Exception as e:
raise InputParameterError(
'Invalid int value {value} for param {name}: {err}'.format(
value=repr(queryParams[name]), name=name, err=str(e)))
def evaluateScalarTokens(tokens):
if tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
if tokens.number.float:
return float(tokens.number.float)
if tokens.number.scientific:
return float(tokens.number.scientific[0])
raise InputParameterError("unknown numeric type in target evaluator")
if tokens.string:
return tokens.string[1:-1]
if tokens.boolean:
return tokens.boolean[0] == 'true'
if tokens.none:
return None
raise InputParameterError("unknown token in target evaluator")
def test_input_parameter_error_with_all_source_id(self):
e = InputParameterError('exception details')
e.setSourceIdHeaders({'X-GRAFANA-ORG-ID': 25, 'X-DASHBOARD-ID': 12})
e.setSourceIdHeaders({'X-PANEL-ID': 3})
self.assertEqual(
str(e),
'Invalid parameters (exception details); source: "X-DASHBOARD-ID: 12, X-GRAFANA-ORG-ID: 25, X-PANEL-ID: 3"',
)
def find(self,
pattern,
startTime=None,
endTime=None,
local=False,
headers=None,
leaves_only=False):
try:
query = FindQuery(pattern,
startTime,
endTime,
local=local,
headers=headers,
leaves_only=leaves_only)
except Exception as e:
raise InputParameterError(
'Failed to instantiate find query: {err}'.format(err=str(e)))
warn_threshold = settings.METRICS_FIND_WARNING_THRESHOLD
fail_threshold = settings.METRICS_FIND_FAILURE_THRESHOLD
matched_leafs = 0
for match in self._find(query):
if isinstance(match, LeafNode):
matched_leafs += 1
elif leaves_only:
continue
if matched_leafs > fail_threshold:
raise Exception(
("Query %s yields too many results and failed "
"(failure threshold is %d)") % (pattern, fail_threshold))
yield match
if matched_leafs > warn_threshold:
log.warning(("Query %s yields large number of results up to %d "
"(warning threshold is %d)") %
(pattern, matched_leafs, warn_threshold))
def test_input_parameter_error_basic(self):
e = InputParameterError('exception details')
self.assertEqual(
str(e),
'Invalid parameters (exception details)',
)
def renderView(request):
start = time()
try:
# we consider exceptions thrown by the option
# parsing to be due to user input error
(graphOptions, requestOptions) = parseOptions(request)
except Exception as e:
raise InputParameterError(str(e))
useCache = 'noCache' not in requestOptions
cacheTimeout = requestOptions['cacheTimeout']
# TODO: Make that a namedtuple or a class.
requestContext = {
'startTime': requestOptions['startTime'],
'endTime': requestOptions['endTime'],
'now': requestOptions['now'],
'localOnly': requestOptions['localOnly'],
'template': requestOptions['template'],
'tzinfo': requestOptions['tzinfo'],
'forwardHeaders': requestOptions['forwardHeaders'],
'sourceIdHeaders': requestOptions['sourceIdHeaders'],
'data': [],
'prefetched': {},
'xFilesFactor': requestOptions['xFilesFactor'],
'maxDataPoints': requestOptions.get('maxDataPoints', None),
}
data = requestContext['data']
response = None
# First we check the request cache
if useCache:
requestKey = hashRequest(request)
response = cache.get(requestKey)
if response:
log.cache('Request-Cache hit [%s]' % requestKey)
log.rendering('Returned cached response in %.6f' %
(time() - start))
return response
log.cache('Request-Cache miss [%s]' % requestKey)
# Now we prepare the requested data
if requestOptions['graphType'] == 'pie':
for target in requestOptions['targets']:
if target.find(':') >= 0:
try:
name, value = target.split(':', 1)
value = float(value)
except ValueError:
raise ValueError("Invalid target '%s'" % target)
data.append((name, value))
else:
seriesList = evaluateTarget(requestContext, target)
for series in seriesList:
func = PieFunction(requestOptions['pieMode'])
data.append((series.name, func(requestContext, series)
or 0))
elif requestOptions['graphType'] == 'line':
# Let's see if at least our data is cached
cachedData = None
if useCache:
targets = requestOptions['targets']
startTime = requestOptions['startTime']
endTime = requestOptions['endTime']
dataKey = hashData(targets, startTime, endTime,
requestOptions['xFilesFactor'])
cachedData = cache.get(dataKey)
if cachedData:
log.cache("Data-Cache hit [%s]" % dataKey)
else:
log.cache("Data-Cache miss [%s]" % dataKey)
if cachedData is not None:
requestContext['data'] = data = cachedData
else: # Have to actually retrieve the data now
targets = requestOptions['targets']
data.extend(evaluateTarget(requestContext, targets))
if useCache:
cache.add(dataKey, data, cacheTimeout)
renderStart = time()
format = requestOptions.get('format')
if format == 'csv':
response = renderViewCsv(requestOptions, data)
elif format == 'json':
response = renderViewJson(requestOptions, data)
elif format == 'dygraph':
response = renderViewDygraph(requestOptions, data)
elif format == 'rickshaw':
response = renderViewRickshaw(requestOptions, data)
elif format == 'raw':
response = renderViewRaw(requestOptions, data)
elif format == 'pickle':
response = renderViewPickle(requestOptions, data)
elif format == 'msgpack':
response = renderViewMsgPack(requestOptions, data)
# if response wasn't generated above, render a graph image
if not response:
format = 'image'
renderStart = time()
response = renderViewGraph(graphOptions, requestOptions, data)
if useCache:
cache.add(requestKey, response, cacheTimeout)
patch_response_headers(response, cache_timeout=cacheTimeout)
else:
add_never_cache_headers(response)
log.rendering('%s rendering time %6f' % (format, time() - renderStart))
log.rendering('Total request processing time %6f' % (time() - start))
return response
def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None):
if tokens.template:
arglist = dict()
if tokens.template.kwargs:
arglist.update(dict([(kwarg.argname, evaluateScalarTokens(kwarg.args[0])) for kwarg in tokens.template.kwargs]))
if tokens.template.args:
arglist.update(dict([(str(i+1), evaluateScalarTokens(arg)) for i, arg in enumerate(tokens.template.args)]))
if 'template' in requestContext:
arglist.update(requestContext['template'])
return evaluateTokens(requestContext, tokens.template, arglist)
if tokens.expression:
if tokens.expression.pipedCalls:
# when the expression has piped calls, we pop the right-most call and pass the remaining
# expression into it via pipedArg, to get the same result as a nested call
rightMost = tokens.expression.pipedCalls.pop()
return evaluateTokens(requestContext, rightMost, replacements, tokens)
return evaluateTokens(requestContext, tokens.expression, replacements)
if tokens.pathExpression:
expression = tokens.pathExpression
if replacements:
for name in replacements:
if expression == '$'+name:
val = replacements[name]
if not isinstance(val, six.string_types):
return val
elif re.match(r'^-?[\d.]+$', val):
return float(val)
else:
return val
else:
expression = expression.replace('$'+name, str(replacements[name]))
return fetchData(requestContext, expression)
if tokens.call:
if tokens.call.funcname == 'template':
# if template propagates down here, it means the grammar didn't match the invocation
# as tokens.template. this generally happens if you try to pass non-numeric/string args
raise InputParameterError("invalid template() syntax, only string/numeric arguments are allowed")
if tokens.call.funcname == 'seriesByTag':
return fetchData(requestContext, tokens.call.raw)
try:
func = SeriesFunction(tokens.call.funcname)
except KeyError:
raise InputParameterError('Received request for unknown function: {func}'.format(func=tokens.call.funcname))
rawArgs = tokens.call.args or []
if pipedArg is not None:
rawArgs.insert(0, pipedArg)
args = [evaluateTokens(requestContext, arg, replacements) for arg in rawArgs]
requestContext['args'] = rawArgs
kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements))
for kwarg in tokens.call.kwargs])
def handleInvalidParameters(e):
e.setSourceIdHeaders(requestContext.get('sourceIdHeaders', {}))
e.setTargets(requestContext.get('targets', []))
e.setFunction(tokens.call.funcname, args, kwargs)
if settings.ENFORCE_INPUT_VALIDATION:
raise e
if not getattr(handleInvalidParameters, 'alreadyLogged', False):
log.warning('%s', str(e))
# only log invalid parameters once
setattr(handleInvalidParameters, 'alreadyLogged', True)
if hasattr(func, 'params'):
try:
(args, kwargs) = validateParams(tokens.call.funcname, func.params, args, kwargs)
except InputParameterError as e:
handleInvalidParameters(e)
try:
return func(requestContext, *args, **kwargs)
except NormalizeEmptyResultError:
return []
except InputParameterError as e:
handleInvalidParameters(e)
return evaluateScalarTokens(tokens)
def find_view(request):
"View for finding metrics matching a given pattern"
queryParams = request.GET.copy()
queryParams.update(request.POST)
format = queryParams.get('format', 'treejson')
leaves_only = queryParamAsInt(queryParams, 'leavesOnly', 0)
local_only = queryParamAsInt(queryParams, 'local', 0)
wildcards = queryParamAsInt(queryParams, 'wildcards', 0)
tzinfo = pytz.timezone(settings.TIME_ZONE)
if 'tz' in queryParams:
try:
value = queryParams['tz']
tzinfo = pytz.timezone(value)
except pytz.UnknownTimeZoneError:
pass
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param tz: {err}'.format(
value=repr(value), err=str(e)))
if 'now' in queryParams:
try:
value = queryParams['now']
now = parseATTime(value, tzinfo)
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param now: {err}'.format(
value=repr(value), err=str(e)))
else:
now = datetime.now(tzinfo)
if 'from' in queryParams and str(queryParams['from']) != '-1':
try:
value = queryParams['from']
fromTime = int(epoch(parseATTime(value, tzinfo, now)))
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param from: {err}'.format(
value=repr(value), err=str(e)))
else:
fromTime = -1
if 'until' in queryParams and str(queryParams['until']) != '-1':
try:
value = queryParams['until']
untilTime = int(epoch(parseATTime(value, tzinfo, now)))
except Exception as e:
raise InputParameterError(
'Invalid value {value} for param until: {err}'.format(
value=repr(value), err=str(e)))
else:
untilTime = -1
nodePosition = queryParamAsInt(queryParams, 'position', -1)
jsonp = queryParams.get('jsonp', False)
forward_headers = extractForwardHeaders(request)
if fromTime == -1:
fromTime = None
if untilTime == -1:
untilTime = None
automatic_variants = queryParamAsInt(queryParams, 'automatic_variants', 0)
try:
query = str(queryParams['query'])
except KeyError:
raise InputParameterError('Missing required parameter \'query\'')
if query == '':
raise InputParameterError('Required parameter \'query\' is empty')
if '.' in query:
base_path = query.rsplit('.', 1)[0] + '.'
else:
base_path = ''
if format == 'completer':
query = query.replace('..', '*.')
if not query.endswith('*'):
query += '*'
if automatic_variants:
query_parts = query.split('.')
for i, part in enumerate(query_parts):
if ',' in part and '{' not in part:
query_parts[i] = '{%s}' % part
query = '.'.join(query_parts)
try:
matches = list(
STORE.find(
query,
fromTime,
untilTime,
local=local_only,
headers=forward_headers,
leaves_only=leaves_only,
))
except Exception:
log.exception()
raise
log.info('find_view query=%s local_only=%s matches=%d' %
(query, local_only, len(matches)))
matches.sort(key=lambda node: node.name)
log.info(
"received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d"
% (query, fromTime, untilTime, local_only, format, len(matches)))
if format == 'treejson':
profile = getProfile(request)
content = tree_json(matches,
base_path,
wildcards=profile.advancedUI or wildcards)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'nodelist':
content = nodes_by_position(matches, nodePosition)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'pickle':
content = pickle_nodes(matches)
response = HttpResponse(content, content_type='application/pickle')
elif format == 'msgpack':
content = msgpack_nodes(matches)
response = HttpResponse(content, content_type='application/x-msgpack')
elif format == 'json':
content = json_nodes(matches)
response = json_response_for(request, content, jsonp=jsonp)
elif format == 'completer':
results = []
for node in matches:
node_info = dict(path=node.path,
name=node.name,
is_leaf=str(int(node.is_leaf)))
if not node.is_leaf:
node_info['path'] += '.'
results.append(node_info)
if len(results) > 1 and wildcards:
wildcardNode = {'name': '*'}
results.append(wildcardNode)
response = json_response_for(request, {'metrics': results},
jsonp=jsonp)
else:
return HttpResponseBadRequest(
content="Invalid value for 'format' parameter",
content_type='text/plain')
response['Pragma'] = 'no-cache'
response['Cache-Control'] = 'no-cache'
return response
def test_input_parameter_error_with_all_properties(self):
e = InputParameterError('exception details')
e.setSourceIdHeaders({'X-DASHBOARD-ID': 'a'})
e.setSourceIdHeaders({'X-GRAFANA-ORG-ID': 'b'})
e.setSourceIdHeaders({'X-PANEL-ID': 'c'})
e.setTargets(['some(target, extra="value")'])
e.setFunction('some', ['target'], {'extra': 'value'})
self.assertEqual(
str(e), 'Invalid parameters (exception details)'
'; targets: "some(target, extra="value")"'
'; source: "X-DASHBOARD-ID: a, X-GRAFANA-ORG-ID: b, X-PANEL-ID: c"'
'; func: "some(\'target\', extra=\'value\')"')