def test_input_parameter_error_with_panel_id(self): e = InputParameterError('exception details') e.setSourceIdHeaders({'X-PANEL-ID': 12}) self.assertEqual( str(e), 'Invalid parameters (exception details); source: "X-PANEL-ID: 12"', )
def test_input_parameter_error_with_grafana_org_id(self): e = InputParameterError('exception details') e.setSourceIdHeaders({'X-GRAFANA-ORG-ID': 3}) self.assertEqual( str(e), 'Invalid parameters (exception details); source: "X-GRAFANA-ORG-ID: 3"', )
def test_input_parameter_error_with_dashboard_id(self): e = InputParameterError('exception details') e.setSourceIdHeaders({'X-DASHBOARD-ID': 'abcde123'}) self.assertEqual( str(e), 'Invalid parameters (exception details); source: "X-DASHBOARD-ID: abcde123"', )
def test_input_parameter_error_with_multiple_targets(self): e = InputParameterError('exception details') e.setTargets(['some_func(a.b.c)', 'otherfunc(c.b.a)']) self.assertEqual( str(e), 'Invalid parameters (exception details); targets: "some_func(a.b.c), otherfunc(c.b.a)"', )
def validateParams(func, params, args, kwargs): valid_args = [] if len(params) == 0 or params[len(params) - 1].multiple is False: if len(args) + len(kwargs) > len(params): raise InputParameterError( 'Too many parameters specified for function "{func}"'.format( func=func)) for i in range(len(params)): if len(args) <= i: # requirement is satisfied from "kwargs" value = kwargs.get(params[i].name, None) if value is None: if params[i].required: # required parameter is missing raise InputParameterError( 'Missing required parameter "{param}" for function "{func}"' .format(param=params[i].name, func=func)) else: # got multiple values for keyword argument if params[i].name in valid_args: raise InputParameterError( 'Keyword parameter "{param}" specified multiple times for function "{func}"' .format(param=params[i].name, func=func)) else: # requirement is satisfied from "args" value = args[i] params[i].validateValue(value, func) valid_args.append(params[i].name) return True
def test_input_parameter_error_with_func_and_args(self): e = InputParameterError('exception details') e.setFunction('test_func', [1, 'a'], {}) self.assertEqual( str(e), 'Invalid parameters (exception details); func: "test_func(1, \'a\')"', )
def test_input_parameter_error_with_func_args_kwargs(self): e = InputParameterError('exception details') e.setFunction('test_func', [3, 'd'], {'a': 1, 'b': 'b'}) self.assertEqual( str(e), 'Invalid parameters (exception details); func: "test_func(3, \'d\', a=1, b=\'b\')"', )
def validateValue(self, value, func): # if value isn't specified and there's a default then the default will be used, # we don't need to validate the default value because we trust that it is valid if value is None and self.default is not None: return True # None is ok for optional params if not self.required and value is None: return True # parameter is restricted to a defined set of values, but value is not in it if self.options and value not in self.options: raise InputParameterError( 'Invalid option specified for function "{func}" parameter "{param}": {value}' .format(func=func, param=self.name, value=repr(value))) if not self.type.isValid(value): raise InputParameterError( 'Invalid "{type}" value specified for function "{func}" parameter "{param}": {value}' .format(type=self.type.name, func=func, param=self.name, value=repr(value))) return True
def validateParams(func, params, args, kwargs): valid_args = [] valid_kwargs = {} # total number of args + kwargs might be larger than number of params if # the last param allows to be specified multiple times if len(args) + len(kwargs) > len(params): if not params[-1].multiple: raise InputParameterError( 'Too many parameters specified for function "{func}"'.format(func=func)) # if args has more values than params and the last param allows multiple values, # then we're going to validate all paramas from the args value list args_params = params kwargs_params = [] else: # take the first len(args) params and use them to validate the args values, # use the remaining params to validate the kwargs values args_params = params[:len(args)] kwargs_params = params[len(args):] # validate the args for (i, arg) in enumerate(args): if i >= len(args_params): # last parameter must be allowing multiple, # so we're using it to validate this arg param = args_params[-1] else: param = args_params[i] valid_args.append(param.validateValue(arg, func)) # validate the kwargs for param in kwargs_params: value = kwargs.get(param.name, None) if value is None: if param.required: raise InputParameterError( 'Missing required parameter "{param}" for function "{func}"' .format(param=param.name, func=func)) continue valid_kwargs[param.name] = param.validateValue(value, func) if len(kwargs) > len(valid_kwargs): unexpected_keys = [] for name in kwargs.keys(): if name not in valid_kwargs: unexpected_keys.append(name) raise InputParameterError( 'Unexpected key word arguments: {keys}'.format( keys=', '.join( key for key in kwargs.keys() if key not in valid_kwargs.keys() ))) return (valid_args, valid_kwargs)
def getAggFunc(func, rawFunc=None): if func in aggFuncs: return aggFuncs[func] if func in aggFuncAliases: return aggFuncAliases[func] raise InputParameterError('Unsupported aggregation function: %s' % (rawFunc or func))
def validateParams(func, params, args, kwargs): valid_args = [] if len(params) == 0 or params[len(params) - 1].multiple is False: if len(args) + len(kwargs) > len(params): raise InputParameterError( 'Too many parameters specified for function "{func}"'.format( func=func)) for i in range(len(params)): if len(args) <= i: # requirement is satisfied from "kwargs" value = kwargs.get(params[i].name, None) if value is None: if params[i].required: # required parameter is missing raise InputParameterError( 'Missing required parameter "{param}" for function "{func}"' .format(param=params[i].name, func=func)) else: # got multiple values for keyword argument if params[i].name in valid_args: raise InputParameterError( 'Keyword parameter "{param}" specified multiple times for function "{func}"' .format(param=params[i].name, func=func)) else: # requirement is satisfied from "args" value = args[i] # parameter is restricted to a defined set of values, but value is not in it if params[i].options and value not in params[i].options: raise InputParameterError( 'Invalid option "{value}" specified for function "{func}" parameter "{param}"' .format(value=value, param=params[i].name, func=func)) if not params[i].validateValue(value): raise InputParameterError( 'Invalid {type} value specified for function "{func}" parameter "{param}": {value}' .format(type=params[i].type.name, func=func, param=params[i].name, value=value)) valid_args.append(params[i].name) return True
def queryParamAsInt(queryParams, name, default): if name not in queryParams: return default try: return int(queryParams[name]) except Exception as e: raise InputParameterError( 'Invalid int value {value} for param {name}: {err}'.format( value=repr(queryParams[name]), name=name, err=str(e)))
def evaluateScalarTokens(tokens): if tokens.number: if tokens.number.integer: return int(tokens.number.integer) if tokens.number.float: return float(tokens.number.float) if tokens.number.scientific: return float(tokens.number.scientific[0]) raise InputParameterError("unknown numeric type in target evaluator") if tokens.string: return tokens.string[1:-1] if tokens.boolean: return tokens.boolean[0] == 'true' if tokens.none: return None raise InputParameterError("unknown token in target evaluator")
def test_input_parameter_error_with_all_source_id(self): e = InputParameterError('exception details') e.setSourceIdHeaders({'X-GRAFANA-ORG-ID': 25, 'X-DASHBOARD-ID': 12}) e.setSourceIdHeaders({'X-PANEL-ID': 3}) self.assertEqual( str(e), 'Invalid parameters (exception details); source: "X-DASHBOARD-ID: 12, X-GRAFANA-ORG-ID: 25, X-PANEL-ID: 3"', )
def find(self, pattern, startTime=None, endTime=None, local=False, headers=None, leaves_only=False): try: query = FindQuery(pattern, startTime, endTime, local=local, headers=headers, leaves_only=leaves_only) except Exception as e: raise InputParameterError( 'Failed to instantiate find query: {err}'.format(err=str(e))) warn_threshold = settings.METRICS_FIND_WARNING_THRESHOLD fail_threshold = settings.METRICS_FIND_FAILURE_THRESHOLD matched_leafs = 0 for match in self._find(query): if isinstance(match, LeafNode): matched_leafs += 1 elif leaves_only: continue if matched_leafs > fail_threshold: raise Exception( ("Query %s yields too many results and failed " "(failure threshold is %d)") % (pattern, fail_threshold)) yield match if matched_leafs > warn_threshold: log.warning(("Query %s yields large number of results up to %d " "(warning threshold is %d)") % (pattern, matched_leafs, warn_threshold))
def test_input_parameter_error_basic(self): e = InputParameterError('exception details') self.assertEqual( str(e), 'Invalid parameters (exception details)', )
def renderView(request): start = time() try: # we consider exceptions thrown by the option # parsing to be due to user input error (graphOptions, requestOptions) = parseOptions(request) except Exception as e: raise InputParameterError(str(e)) useCache = 'noCache' not in requestOptions cacheTimeout = requestOptions['cacheTimeout'] # TODO: Make that a namedtuple or a class. requestContext = { 'startTime': requestOptions['startTime'], 'endTime': requestOptions['endTime'], 'now': requestOptions['now'], 'localOnly': requestOptions['localOnly'], 'template': requestOptions['template'], 'tzinfo': requestOptions['tzinfo'], 'forwardHeaders': requestOptions['forwardHeaders'], 'sourceIdHeaders': requestOptions['sourceIdHeaders'], 'data': [], 'prefetched': {}, 'xFilesFactor': requestOptions['xFilesFactor'], 'maxDataPoints': requestOptions.get('maxDataPoints', None), } data = requestContext['data'] response = None # First we check the request cache if useCache: requestKey = hashRequest(request) response = cache.get(requestKey) if response: log.cache('Request-Cache hit [%s]' % requestKey) log.rendering('Returned cached response in %.6f' % (time() - start)) return response log.cache('Request-Cache miss [%s]' % requestKey) # Now we prepare the requested data if requestOptions['graphType'] == 'pie': for target in requestOptions['targets']: if target.find(':') >= 0: try: name, value = target.split(':', 1) value = float(value) except ValueError: raise ValueError("Invalid target '%s'" % target) data.append((name, value)) else: seriesList = evaluateTarget(requestContext, target) for series in seriesList: func = PieFunction(requestOptions['pieMode']) data.append((series.name, func(requestContext, series) or 0)) elif requestOptions['graphType'] == 'line': # Let's see if at least our data is cached cachedData = None if useCache: targets = requestOptions['targets'] startTime = requestOptions['startTime'] endTime = requestOptions['endTime'] dataKey = hashData(targets, startTime, endTime, requestOptions['xFilesFactor']) cachedData = cache.get(dataKey) if cachedData: log.cache("Data-Cache hit [%s]" % dataKey) else: log.cache("Data-Cache miss [%s]" % dataKey) if cachedData is not None: requestContext['data'] = data = cachedData else: # Have to actually retrieve the data now targets = requestOptions['targets'] data.extend(evaluateTarget(requestContext, targets)) if useCache: cache.add(dataKey, data, cacheTimeout) renderStart = time() format = requestOptions.get('format') if format == 'csv': response = renderViewCsv(requestOptions, data) elif format == 'json': response = renderViewJson(requestOptions, data) elif format == 'dygraph': response = renderViewDygraph(requestOptions, data) elif format == 'rickshaw': response = renderViewRickshaw(requestOptions, data) elif format == 'raw': response = renderViewRaw(requestOptions, data) elif format == 'pickle': response = renderViewPickle(requestOptions, data) elif format == 'msgpack': response = renderViewMsgPack(requestOptions, data) # if response wasn't generated above, render a graph image if not response: format = 'image' renderStart = time() response = renderViewGraph(graphOptions, requestOptions, data) if useCache: cache.add(requestKey, response, cacheTimeout) patch_response_headers(response, cache_timeout=cacheTimeout) else: add_never_cache_headers(response) log.rendering('%s rendering time %6f' % (format, time() - renderStart)) log.rendering('Total request processing time %6f' % (time() - start)) return response
def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None): if tokens.template: arglist = dict() if tokens.template.kwargs: arglist.update(dict([(kwarg.argname, evaluateScalarTokens(kwarg.args[0])) for kwarg in tokens.template.kwargs])) if tokens.template.args: arglist.update(dict([(str(i+1), evaluateScalarTokens(arg)) for i, arg in enumerate(tokens.template.args)])) if 'template' in requestContext: arglist.update(requestContext['template']) return evaluateTokens(requestContext, tokens.template, arglist) if tokens.expression: if tokens.expression.pipedCalls: # when the expression has piped calls, we pop the right-most call and pass the remaining # expression into it via pipedArg, to get the same result as a nested call rightMost = tokens.expression.pipedCalls.pop() return evaluateTokens(requestContext, rightMost, replacements, tokens) return evaluateTokens(requestContext, tokens.expression, replacements) if tokens.pathExpression: expression = tokens.pathExpression if replacements: for name in replacements: if expression == '$'+name: val = replacements[name] if not isinstance(val, six.string_types): return val elif re.match(r'^-?[\d.]+$', val): return float(val) else: return val else: expression = expression.replace('$'+name, str(replacements[name])) return fetchData(requestContext, expression) if tokens.call: if tokens.call.funcname == 'template': # if template propagates down here, it means the grammar didn't match the invocation # as tokens.template. this generally happens if you try to pass non-numeric/string args raise InputParameterError("invalid template() syntax, only string/numeric arguments are allowed") if tokens.call.funcname == 'seriesByTag': return fetchData(requestContext, tokens.call.raw) try: func = SeriesFunction(tokens.call.funcname) except KeyError: raise InputParameterError('Received request for unknown function: {func}'.format(func=tokens.call.funcname)) rawArgs = tokens.call.args or [] if pipedArg is not None: rawArgs.insert(0, pipedArg) args = [evaluateTokens(requestContext, arg, replacements) for arg in rawArgs] requestContext['args'] = rawArgs kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements)) for kwarg in tokens.call.kwargs]) def handleInvalidParameters(e): e.setSourceIdHeaders(requestContext.get('sourceIdHeaders', {})) e.setTargets(requestContext.get('targets', [])) e.setFunction(tokens.call.funcname, args, kwargs) if settings.ENFORCE_INPUT_VALIDATION: raise e if not getattr(handleInvalidParameters, 'alreadyLogged', False): log.warning('%s', str(e)) # only log invalid parameters once setattr(handleInvalidParameters, 'alreadyLogged', True) if hasattr(func, 'params'): try: (args, kwargs) = validateParams(tokens.call.funcname, func.params, args, kwargs) except InputParameterError as e: handleInvalidParameters(e) try: return func(requestContext, *args, **kwargs) except NormalizeEmptyResultError: return [] except InputParameterError as e: handleInvalidParameters(e) return evaluateScalarTokens(tokens)
def find_view(request): "View for finding metrics matching a given pattern" queryParams = request.GET.copy() queryParams.update(request.POST) format = queryParams.get('format', 'treejson') leaves_only = queryParamAsInt(queryParams, 'leavesOnly', 0) local_only = queryParamAsInt(queryParams, 'local', 0) wildcards = queryParamAsInt(queryParams, 'wildcards', 0) tzinfo = pytz.timezone(settings.TIME_ZONE) if 'tz' in queryParams: try: value = queryParams['tz'] tzinfo = pytz.timezone(value) except pytz.UnknownTimeZoneError: pass except Exception as e: raise InputParameterError( 'Invalid value {value} for param tz: {err}'.format( value=repr(value), err=str(e))) if 'now' in queryParams: try: value = queryParams['now'] now = parseATTime(value, tzinfo) except Exception as e: raise InputParameterError( 'Invalid value {value} for param now: {err}'.format( value=repr(value), err=str(e))) else: now = datetime.now(tzinfo) if 'from' in queryParams and str(queryParams['from']) != '-1': try: value = queryParams['from'] fromTime = int(epoch(parseATTime(value, tzinfo, now))) except Exception as e: raise InputParameterError( 'Invalid value {value} for param from: {err}'.format( value=repr(value), err=str(e))) else: fromTime = -1 if 'until' in queryParams and str(queryParams['until']) != '-1': try: value = queryParams['until'] untilTime = int(epoch(parseATTime(value, tzinfo, now))) except Exception as e: raise InputParameterError( 'Invalid value {value} for param until: {err}'.format( value=repr(value), err=str(e))) else: untilTime = -1 nodePosition = queryParamAsInt(queryParams, 'position', -1) jsonp = queryParams.get('jsonp', False) forward_headers = extractForwardHeaders(request) if fromTime == -1: fromTime = None if untilTime == -1: untilTime = None automatic_variants = queryParamAsInt(queryParams, 'automatic_variants', 0) try: query = str(queryParams['query']) except KeyError: raise InputParameterError('Missing required parameter \'query\'') if query == '': raise InputParameterError('Required parameter \'query\' is empty') if '.' in query: base_path = query.rsplit('.', 1)[0] + '.' else: base_path = '' if format == 'completer': query = query.replace('..', '*.') if not query.endswith('*'): query += '*' if automatic_variants: query_parts = query.split('.') for i, part in enumerate(query_parts): if ',' in part and '{' not in part: query_parts[i] = '{%s}' % part query = '.'.join(query_parts) try: matches = list( STORE.find( query, fromTime, untilTime, local=local_only, headers=forward_headers, leaves_only=leaves_only, )) except Exception: log.exception() raise log.info('find_view query=%s local_only=%s matches=%d' % (query, local_only, len(matches))) matches.sort(key=lambda node: node.name) log.info( "received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d" % (query, fromTime, untilTime, local_only, format, len(matches))) if format == 'treejson': profile = getProfile(request) content = tree_json(matches, base_path, wildcards=profile.advancedUI or wildcards) response = json_response_for(request, content, jsonp=jsonp) elif format == 'nodelist': content = nodes_by_position(matches, nodePosition) response = json_response_for(request, content, jsonp=jsonp) elif format == 'pickle': content = pickle_nodes(matches) response = HttpResponse(content, content_type='application/pickle') elif format == 'msgpack': content = msgpack_nodes(matches) response = HttpResponse(content, content_type='application/x-msgpack') elif format == 'json': content = json_nodes(matches) response = json_response_for(request, content, jsonp=jsonp) elif format == 'completer': results = [] for node in matches: node_info = dict(path=node.path, name=node.name, is_leaf=str(int(node.is_leaf))) if not node.is_leaf: node_info['path'] += '.' results.append(node_info) if len(results) > 1 and wildcards: wildcardNode = {'name': '*'} results.append(wildcardNode) response = json_response_for(request, {'metrics': results}, jsonp=jsonp) else: return HttpResponseBadRequest( content="Invalid value for 'format' parameter", content_type='text/plain') response['Pragma'] = 'no-cache' response['Cache-Control'] = 'no-cache' return response
def test_input_parameter_error_with_all_properties(self): e = InputParameterError('exception details') e.setSourceIdHeaders({'X-DASHBOARD-ID': 'a'}) e.setSourceIdHeaders({'X-GRAFANA-ORG-ID': 'b'}) e.setSourceIdHeaders({'X-PANEL-ID': 'c'}) e.setTargets(['some(target, extra="value")']) e.setFunction('some', ['target'], {'extra': 'value'}) self.assertEqual( str(e), 'Invalid parameters (exception details)' '; targets: "some(target, extra="value")"' '; source: "X-DASHBOARD-ID: a, X-GRAFANA-ORG-ID: b, X-PANEL-ID: c"' '; func: "some(\'target\', extra=\'value\')"')