def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None):
if tokens.template:
arglist = dict()
if tokens.template.kwargs:
arglist.update(dict([(kwarg.argname, evaluateScalarTokens(kwarg.args[0])) for kwarg in tokens.template.kwargs]))
if tokens.template.args:
arglist.update(dict([(str(i+1), evaluateScalarTokens(arg)) for i, arg in enumerate(tokens.template.args)]))
if 'template' in requestContext:
arglist.update(requestContext['template'])
return evaluateTokens(requestContext, tokens.template, arglist)
if tokens.expression:
if tokens.expression.pipedCalls:
# when the expression has piped calls, we pop the right-most call and pass the remaining
# expression into it via pipedArg, to get the same result as a nested call
rightMost = tokens.expression.pipedCalls.pop()
return evaluateTokens(requestContext, rightMost, replacements, tokens)
return evaluateTokens(requestContext, tokens.expression, replacements)
if tokens.pathExpression:
expression = tokens.pathExpression
if replacements:
for name in replacements:
if expression == '$'+name:
val = replacements[name]
if not isinstance(val, six.string_types):
return val
elif re.match('^-?[\d.]+$', val):
return float(val)
else:
return val
else:
expression = expression.replace('$'+name, str(replacements[name]))
return fetchData(requestContext, expression)
if tokens.call:
if tokens.call.funcname == 'template':
# if template propagates down here, it means the grammar didn't match the invocation
# as tokens.template. this generally happens if you try to pass non-numeric/string args
raise ValueError("invalid template() syntax, only string/numeric arguments are allowed")
if tokens.call.funcname == 'seriesByTag':
return fetchData(requestContext, tokens.call.raw)
func = SeriesFunction(tokens.call.funcname)
rawArgs = tokens.call.args or []
if pipedArg is not None:
rawArgs.insert(0, pipedArg)
args = [evaluateTokens(requestContext, arg, replacements) for arg in rawArgs]
requestContext['args'] = rawArgs
kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements))
for kwarg in tokens.call.kwargs])
try:
return func(requestContext, *args, **kwargs)
except NormalizeEmptyResultError:
return []
return evaluateScalarTokens(tokens)
def evaluateTokens(requestContext, tokens):
if tokens.expression:
return evaluateTokens(requestContext, tokens.expression)
elif tokens.pathExpression:
return fetchData(requestContext, tokens.pathExpression)
elif tokens.call:
func = SeriesFunctions[tokens.call.func]
args = [evaluateTokens(requestContext, arg) for arg in tokens.call.args]
return func(requestContext, *args)
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.number.scientific:
return float(tokens.number.scientific[0])
elif tokens.string:
return str(tokens.string)[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
def evaluateTokens(requestContext, tokens):
if tokens.expression:
return evaluateTokens(requestContext, tokens.expression)
elif tokens.pathExpression:
return fetchData(requestContext, tokens.pathExpression)
elif tokens.call:
func = SeriesFunctions[tokens.call.funcname]
args = [evaluateTokens(requestContext, arg) for arg in tokens.call.args]
kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0]))
for kwarg in tokens.call.kwargs])
try:
return func(requestContext, *args, **kwargs)
except NormalizeEmptyResultError:
return []
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.number.scientific:
return float(tokens.number.scientific[0])
elif tokens.string:
return tokens.string[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
def evaluateTokens(requestContext, tokens):
if tokens.expression:
return evaluateTokens(requestContext, tokens.expression)
elif tokens.pathExpression:
return fetchData(requestContext, tokens.pathExpression)
elif tokens.call:
try:
func = SeriesFunctions[tokens.call.func]
args = [
evaluateTokens(requestContext, arg) for arg in tokens.call.args
]
return func(requestContext, *args)
except ValueError:
log.exception('value error when render')
return []
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.number.scientific:
return float(tokens.number.scientific[0])
elif tokens.string:
return tokens.string[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
def evaluateTokens(requestContext, tokens):
if tokens.expression:
return evaluateTokens(requestContext, tokens.expression)
elif tokens.pathExpression:
return fetchData(requestContext, tokens.pathExpression)
elif tokens.call:
func = SeriesFunctions[tokens.call.func]
args = [
evaluateTokens(requestContext, arg) for arg in tokens.call.args
]
return func(requestContext, *args)
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.string:
return str(tokens.string)[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
def evaluateTokens(requestContext, tokens):
if tokens.expression:
return evaluateTokens(requestContext, tokens.expression)
elif tokens.pathExpression:
return fetchData(requestContext, tokens.pathExpression)
elif tokens.call:
func = SeriesFunctions[tokens.call.func]
args = [
evaluateTokens(requestContext, arg) for arg in tokens.call.args
]
try:
return func(requestContext, *args)
except NormalizeEmptyResultError:
return []
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.number.scientific:
return float(tokens.number.scientific[0])
elif tokens.string:
return str(tokens.string)[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
def evaluateTokens(requestContext, tokens):
if tokens.expression:
return evaluateTokens(requestContext, tokens.expression)
elif tokens.pathExpression:
return fetchData(requestContext, tokens.pathExpression)
elif tokens.call:
func = SeriesFunctions[tokens.call.funcname]
args = [
evaluateTokens(requestContext, arg) for arg in tokens.call.args
]
kwargs = dict([(kwarg.argname,
evaluateTokens(requestContext, kwarg.args[0]))
for kwarg in tokens.call.kwargs])
return func(requestContext, *args, **kwargs)
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.number.scientific:
return float(tokens.number.scientific[0])
elif tokens.string:
return tokens.string[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
def evaluateTokens(requestContext, tokens):
if tokens.expression:
return evaluateTokens(requestContext, tokens.expression)
elif tokens.pathExpression:
return fetchData(requestContext, tokens.pathExpression)
elif tokens.call:
try:
func = SeriesFunctions[tokens.call.func]
args = [evaluateTokens(requestContext, arg) for arg in tokens.call.args]
return func(requestContext, *args)
except ValueError:
log.exception('value error when render')
return []
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.number.scientific:
return float(tokens.number.scientific[0])
elif tokens.string:
return tokens.string[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
def test_fetchData(self):
pathExpr = 'collectd.test-db.load.value'
startTime=datetime(1970, 1, 1, 0, 10, 0, 0, pytz.timezone(settings.TIME_ZONE))
endTime=datetime(1970, 1, 1, 0, 20, 0, 0, pytz.timezone(settings.TIME_ZONE))
requestContext = self._build_requestContext(startTime, endTime)
requestContext['now'] = endTime
requestContext['forwardHeaders'] = None
results = fetchData(requestContext, pathExpr)
expectedResults = []
self.assertEqual(results, expectedResults)
def test_fetchData(self):
pathExpr = 'collectd.test-db.load.value'
startTime=datetime(1970, 1, 1, 0, 10, 0, 0, pytz.timezone(settings.TIME_ZONE))
endTime=datetime(1970, 1, 1, 0, 20, 0, 0, pytz.timezone(settings.TIME_ZONE))
requestContext = self._build_requestContext(startTime, endTime)
requestContext['now'] = endTime
requestContext['forwardHeaders'] = None
results = fetchData(requestContext, pathExpr)
expectedResults = []
self.assertEqual(results, expectedResults)
def _get(from_time):
series = fetchData({
'startTime': from_time,
'endTime': until_time,
'now': until_time,
'localOnly': False},
metric
)
try:
return [k for k in series[0] if k is not None][-1]
except IndexError:
return None
def _get(from_time):
series = fetchData(
{
'startTime': from_time,
'endTime': until_time,
'now': until_time,
'localOnly': False
}, metric)
try:
return [k for k in series[0] if k is not None][-1]
except IndexError:
return None
def test_fetchData_wildcard(self):
pathExpr = 'collectd.test-db.*.value'
startTime = datetime(1970, 1, 1, 0, 10, 0, 0,
pytz.timezone(settings.TIME_ZONE))
endTime = datetime(1970, 1, 1, 0, 20, 0, 0,
pytz.timezone(settings.TIME_ZONE))
startTime = datetime(2016, 6, 16, 15, 55, 38)
endTime = datetime(2016, 6, 17, 15, 55, 38)
requestContext = self._build_requestContext(startTime, endTime)
requestContext['now'] = endTime
print("requestContext: {}".format(requestContext))
results = fetchData(requestContext, pathExpr)
expectedResults = []
print("result_fetchdata: {}".format(results))
print("expr: {}".format(pathExpr))
self.assertEqual(results, expectedResults)
def test_fetchData_failed_fetch(self, log_exception):
pathExpr = 'collectd.test-db.load.value'
startTime = datetime(1970, 1, 1, 0, 10, 0, 0,
pytz.timezone(settings.TIME_ZONE))
endTime = datetime(1970, 1, 1, 0, 20, 0, 0,
pytz.timezone(settings.TIME_ZONE))
requestContext = self._build_requestContext(startTime, endTime)
requestContext['now'] = endTime
requestContext['forwardHeaders'] = None
def mock__fetchData(pathExpr, startTime, endTime, now, requestContext,
seriesList):
raise Exception('mock failure')
with self.assertRaises(Exception):
with patch('graphite.render.datalib._fetchData', mock__fetchData):
results = fetchData(requestContext, pathExpr)
def evaluateTokens(requestContext, tokens, replacements=None):
if tokens.template:
arglist = dict()
if tokens.template.kwargs:
arglist.update(
dict([(kwarg.argname,
evaluateTokens(requestContext, kwarg.args[0]))
for kwarg in tokens.template.kwargs]))
if tokens.template.args:
arglist.update(
dict([(str(i + 1), evaluateTokens(requestContext, arg))
for i, arg in enumerate(tokens.template.args)]))
if 'template' in requestContext:
arglist.update(requestContext['template'])
return evaluateTokens(requestContext, tokens.template, arglist)
elif tokens.expression:
return evaluateTokens(requestContext, tokens.expression, replacements)
elif tokens.pathExpression:
expression = tokens.pathExpression
if replacements:
for name in replacements:
if expression == '$' + name:
val = replacements[name]
if not isinstance(val, str) and not isinstance(
val, basestring):
return val
elif re.match('^-?[\d.]+$', val):
return float(val)
else:
return val
else:
expression = expression.replace('$' + name,
str(replacements[name]))
return fetchData(requestContext, expression)
elif tokens.call:
if tokens.call.funcname == 'template':
# if template propagates down here, it means the grammar didn't match the invocation
# as tokens.template. this generally happens if you try to pass non-numeric/string args
raise ValueError(
"invalid template() syntax, only string/numeric arguments are allowed"
)
func = SeriesFunctions[tokens.call.funcname]
args = [
evaluateTokens(requestContext, arg, replacements)
for arg in tokens.call.args
]
kwargs = dict([(kwarg.argname,
evaluateTokens(requestContext, kwarg.args[0],
replacements))
for kwarg in tokens.call.kwargs])
try:
return func(requestContext, *args, **kwargs)
except NormalizeEmptyResultError:
return []
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.number.scientific:
return float(tokens.number.scientific[0])
elif tokens.string:
return tokens.string[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
else:
raise ValueError("unknown token in target evaluator")
def evaluateTokens(requestContext, tokens, replacements=None):
if tokens.template:
arglist = dict()
if tokens.template.kwargs:
arglist.update(dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0])) for kwarg in tokens.template.kwargs]))
if tokens.template.args:
arglist.update(dict([(str(i + 1), evaluateTokens(requestContext, arg)) for i, arg in enumerate(tokens.template.args)]))
if 'template' in requestContext:
arglist.update(requestContext['template'])
return evaluateTokens(requestContext, tokens.template, arglist)
elif tokens.expression:
return evaluateTokens(requestContext, tokens.expression, replacements)
elif tokens.pathExpression:
expression = tokens.pathExpression
if replacements:
for name in replacements:
if expression == '$' + name:
val = replacements[name]
if not isinstance(val, str) and not isinstance(val, basestring):
return val
elif re.match('^-?[\d.]+$', val):
return float(val)
else:
return val
else:
expression = expression.replace('$' + name, str(replacements[name]))
return fetchData(requestContext, expression)
elif tokens.call:
if tokens.call.funcname == 'template':
# if template propagates down here, it means the grammar didn't match the invocation
# as tokens.template. this generally happens if you try to pass non-numeric/string args
raise ValueError("invaild template() syntax, only string/numeric arguments are allowed")
func = SeriesFunctions[tokens.call.funcname]
args = [evaluateTokens(requestContext, arg, replacements) for arg in tokens.call.args]
kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements))
for kwarg in tokens.call.kwargs])
try:
return func(requestContext, *args, **kwargs)
except NormalizeEmptyResultError:
return []
elif tokens.number:
if tokens.number.integer:
return int(tokens.number.integer)
elif tokens.number.float:
return float(tokens.number.float)
elif tokens.number.scientific:
return float(tokens.number.scientific[0])
elif tokens.string:
return tokens.string[1:-1]
elif tokens.boolean:
return tokens.boolean[0] == 'true'
else:
raise ValueError("unknown token in target evaulator")
def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None):
if tokens.template:
arglist = dict()
if tokens.template.kwargs:
arglist.update(dict([(kwarg.argname, evaluateScalarTokens(kwarg.args[0])) for kwarg in tokens.template.kwargs]))
if tokens.template.args:
arglist.update(dict([(str(i+1), evaluateScalarTokens(arg)) for i, arg in enumerate(tokens.template.args)]))
if 'template' in requestContext:
arglist.update(requestContext['template'])
return evaluateTokens(requestContext, tokens.template, arglist)
if tokens.expression:
if tokens.expression.pipedCalls:
# when the expression has piped calls, we pop the right-most call and pass the remaining
# expression into it via pipedArg, to get the same result as a nested call
rightMost = tokens.expression.pipedCalls.pop()
return evaluateTokens(requestContext, rightMost, replacements, tokens)
return evaluateTokens(requestContext, tokens.expression, replacements)
if tokens.pathExpression:
expression = tokens.pathExpression
if replacements:
for name in replacements:
if expression == '$'+name:
val = replacements[name]
if not isinstance(val, six.string_types):
return val
elif re.match(r'^-?[\d.]+$', val):
return float(val)
else:
return val
else:
expression = expression.replace('$'+name, str(replacements[name]))
return fetchData(requestContext, expression)
if tokens.call:
if tokens.call.funcname == 'template':
# if template propagates down here, it means the grammar didn't match the invocation
# as tokens.template. this generally happens if you try to pass non-numeric/string args
raise InputParameterError("invalid template() syntax, only string/numeric arguments are allowed")
if tokens.call.funcname == 'seriesByTag':
return fetchData(requestContext, tokens.call.raw)
try:
func = SeriesFunction(tokens.call.funcname)
except KeyError:
raise InputParameterError('Received request for unknown function: {func}'.format(func=tokens.call.funcname))
rawArgs = tokens.call.args or []
if pipedArg is not None:
rawArgs.insert(0, pipedArg)
args = [evaluateTokens(requestContext, arg, replacements) for arg in rawArgs]
requestContext['args'] = rawArgs
kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements))
for kwarg in tokens.call.kwargs])
def handleInvalidParameters(e):
e.setSourceIdHeaders(requestContext.get('sourceIdHeaders', {}))
e.setTargets(requestContext.get('targets', []))
e.setFunction(tokens.call.funcname, args, kwargs)
if settings.ENFORCE_INPUT_VALIDATION:
raise e
if not getattr(handleInvalidParameters, 'alreadyLogged', False):
log.warning('%s', str(e))
# only log invalid parameters once
setattr(handleInvalidParameters, 'alreadyLogged', True)
if hasattr(func, 'params'):
try:
(args, kwargs) = validateParams(tokens.call.funcname, func.params, args, kwargs)
except InputParameterError as e:
handleInvalidParameters(e)
try:
return func(requestContext, *args, **kwargs)
except NormalizeEmptyResultError:
return []
except InputParameterError as e:
handleInvalidParameters(e)
return evaluateScalarTokens(tokens)