def new_f(*args, **kwargs):
try:
return f(*args, **kwargs)
except InputParameterError as e:
msgStr = str(e)
log.warning('%s', msgStr)
return HttpResponseBadRequest(msgStr)
def find(self,
pattern,
startTime=None,
endTime=None,
local=False,
headers=None,
leaves_only=False):
query = FindQuery(pattern,
startTime,
endTime,
local=local,
headers=headers,
leaves_only=leaves_only)
warn_threshold = settings.METRICS_FIND_WARNING_THRESHOLD
fail_threshold = settings.METRICS_FIND_FAILURE_THRESHOLD
matched_leafs = 0
for match in self._find(query):
if isinstance(match, LeafNode):
matched_leafs += 1
elif leaves_only:
continue
if matched_leafs > fail_threshold:
raise Exception(
("Query %s yields too many results and failed "
"(failure threshold is %d)") % (pattern, fail_threshold))
yield match
if matched_leafs > warn_threshold:
log.warning(("Query %s yields large number of results up to %d "
"(warning threshold is %d)") %
(pattern, matched_leafs, warn_threshold))
def find(self, pattern, startTime=None, endTime=None, local=False, headers=None, leaves_only=False):
query = FindQuery(
pattern, startTime, endTime,
local=local,
headers=headers,
leaves_only=leaves_only
)
warn_threshold = settings.METRICS_FIND_WARNING_THRESHOLD
fail_threshold = settings.METRICS_FIND_FAILURE_THRESHOLD
matched_leafs = 0
for match in self._find(query):
if isinstance(match, LeafNode):
matched_leafs += 1
elif leaves_only:
continue
if matched_leafs > fail_threshold:
raise Exception(
("Query %s yields too many results and failed "
"(failure threshold is %d)") % (pattern, fail_threshold))
yield match
if matched_leafs > warn_threshold:
log.warning(
("Query %s yields large number of results up to %d "
"(warning threshold is %d)") % (
pattern, matched_leafs, warn_threshold))
def validateAggFuncs(self, value):
if value in self.getValidAggFuncs():
return value
if value in self.getDeprecatedAggFuncs():
log.warning('Deprecated aggregation function: {value}'.format(value=repr(value)))
return value
raise ValueError('Invalid aggregation function: {value}'.format(value=repr(value)))
def epoch(dt):
"""
Returns the epoch timestamp of a timezone-aware datetime object.
"""
if not dt.tzinfo:
tb = traceback.extract_stack(None, 2)
log.warning('epoch() called with non-timezone-aware datetime in %s at %s:%d' % (tb[0][2], tb[0][0], tb[0][1]))
return calendar.timegm(make_aware(dt, pytz.timezone(settings.TIME_ZONE)).astimezone(pytz.utc).timetuple())
return calendar.timegm(dt.astimezone(pytz.utc).timetuple())
def epoch(dt):
"""
Returns the epoch timestamp of a timezone-aware datetime object.
"""
if not dt.tzinfo:
tb = traceback.extract_stack(None, 2)
log.warning('epoch() called with non-timezone-aware datetime in %s at %s:%d' % (tb[0][2], tb[0][0], tb[0][1]))
return calendar.timegm(make_aware(dt, pytz.timezone(settings.TIME_ZONE)).astimezone(pytz.utc).timetuple())
return calendar.timegm(dt.astimezone(pytz.utc).timetuple())
def validateAggFuncs(self, value):
if value in self.getValidAggFuncs():
return True
if value in self.getDeprecatedAggFuncs():
log.warning('Deprecated aggregation function "{value}" used'.format(value=value))
return True
return False
def handleInvalidParameters(e):
if not getattr(handleInvalidParameters, 'alreadyLogged', False):
log.warning(
invalidParamLogMsg(requestContext, str(e),
tokens.call.funcname, args, kwargs))
# only log invalid parameters once
setattr(handleInvalidParameters, 'alreadyLogged', True)
if settings.ENFORCE_INPUT_VALIDATION:
raise
def handleInvalidParameters(e):
e.setSourceIdHeaders(requestContext.get('sourceIdHeaders', {}))
e.setTargets(requestContext.get('targets', []))
e.setFunction(tokens.call.funcname, args, kwargs)
if settings.ENFORCE_INPUT_VALIDATION:
raise e
if not getattr(handleInvalidParameters, 'alreadyLogged', False):
log.warning('%s', str(e))
# only log invalid parameters once
setattr(handleInvalidParameters, 'alreadyLogged', True)
def _add(self, key, val):
realval = None
try:
if val != '':
realval = int(val)
except:
log.warning('unexpected val for %s: %s' % (key, val))
branches = key.split('.')
cursor = self.tree
leaf = branches.pop()
for branch in branches:
if branch not in cursor[1]:
cursor[1][branch] = ({}, {}) # (leaves, children)
cursor = cursor[1][branch]
cursor[0][leaf] = realval # add leaf
def handleInvalidParameters(e):
if not getattr(handleInvalidParameters, 'alreadyLogged', False):
log.warning(
'Received invalid parameters ({msg}): {func} ({args})'.
format(msg=str(e),
func=tokens.call.funcname,
args=', '.join(argList for argList in [
', '.join(str(arg) for arg in args),
', '.join('{k}={v}'.format(k=str(k), v=str(v))
for k, v in kwargs.items()),
] if argList)))
# only log invalid parameters once
setattr(handleInvalidParameters, 'alreadyLogged', True)
if settings.ENFORCE_INPUT_VALIDATION:
raise
def loadFunctions(force=False):
if _SeriesFunctions and not force:
return
from graphite.render import functions
_SeriesFunctions.clear()
_SeriesFunctions.update(functions.SeriesFunctions)
_PieFunctions.clear()
_PieFunctions.update(functions.PieFunctions)
custom_modules = []
for filename in listdir(customDir):
module_name, extension = splitext(filename)
if extension != '.py' or module_name == '__init__':
continue
custom_modules.append(customModPrefix + module_name)
for module_name in custom_modules + settings.FUNCTION_PLUGINS:
try:
module = import_module(module_name)
except Exception as e:
log.warning('Error loading function plugin %s: %s' %
(module_name, e))
continue
for func_name, func in getattr(module, 'SeriesFunctions', {}).items():
try:
addFunction(_SeriesFunctions, func, func_name)
except Exception as e:
log.warning('Error loading function plugin %s: %s' %
(module_name, e))
for func_name, func in getattr(module, 'PieFunctions', {}).items():
try:
addFunction(_PieFunctions, func, func_name)
except Exception as e:
log.warning('Error loading function plugin %s: %s' %
(module_name, e))
def fetch(self, path, reqkey, start_time, end_time):
if path in self.result:
return self.result[path]
if reqkey is None:
reqkey = str(uuid.uuid4())
profilingTime = {'start': time.time()}
try:
query = {'start': start_time, 'end': end_time, 'reqKey': reqkey}
data = {'metrics': ','.join(self.paths)}
request_url = graphouse_url + "/metricData"
request = requests.post(request_url, params=query, data=data)
log.debug(
'DEBUG:graphouse_data_query: {} parameters {}, data {}'.format(
request_url, query, data))
request.raise_for_status()
except (requests.exceptions.ConnectionError,
requests.exceptions.Timeout) as e:
log.warning(
'CRITICAL:graphouse_data_query: Connection error: {}'.format(
str(e)))
raise
except requests.exceptions.HTTPError as e:
log.warning(
'CRITICAL:graphouse_data_query: {}, message: {}'.format(
str(e), request.text))
raise
except Exception:
log.warning('Unexpected exception!', exc_info=True)
raise
profilingTime['fetch'] = time.time()
try:
metrics_object = request.json()
except Exception:
log.warning(
'CRITICAL:graphouse_parse: '
"can't parse json from graphouse answer, got '{}'".format(
request.text))
raise
profilingTime['parse'] = time.time()
for node in self.nodes:
metric_object = metrics_object.get(node.path)
if metric_object is None:
self.result[node.path] = ((start_time, end_time, 1), [])
else:
self.result[node.path] = (
(metric_object.get("start"), metric_object.get("end"),
metric_object.get("step")),
metric_object.get("points"),
)
profilingTime['convert'] = time.time()
log.debug('DEBUG:graphouse_time:[{}] full = {} '
'fetch = {}, parse = {}, convert = {}'.format(
reqkey,
profilingTime['convert'] - profilingTime['start'],
profilingTime['fetch'] - profilingTime['start'],
profilingTime['parse'] - profilingTime['fetch'],
profilingTime['convert'] - profilingTime['parse']))
result = self.result.get(path)
if result is None:
log.warning(
'WARNING:graphouse_data: something strange: '
"path {} doesn't exist in graphouse response".format(path))
raise ValueError(
"path {} doesn't exist in graphouse response".format(path))
else:
return result
def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None):
if tokens.template:
arglist = dict()
if tokens.template.kwargs:
arglist.update(
dict([(kwarg.argname, evaluateScalarTokens(kwarg.args[0]))
for kwarg in tokens.template.kwargs]))
if tokens.template.args:
arglist.update(
dict([(str(i + 1), evaluateScalarTokens(arg))
for i, arg in enumerate(tokens.template.args)]))
if 'template' in requestContext:
arglist.update(requestContext['template'])
return evaluateTokens(requestContext, tokens.template, arglist)
if tokens.expression:
if tokens.expression.pipedCalls:
# when the expression has piped calls, we pop the right-most call and pass the remaining
# expression into it via pipedArg, to get the same result as a nested call
rightMost = tokens.expression.pipedCalls.pop()
return evaluateTokens(requestContext, rightMost, replacements,
tokens)
return evaluateTokens(requestContext, tokens.expression, replacements)
if tokens.pathExpression:
expression = tokens.pathExpression
if replacements:
for name in replacements:
if expression == '$' + name:
val = replacements[name]
if not isinstance(val, six.string_types):
return val
elif re.match(r'^-?[\d.]+$', val):
return float(val)
else:
return val
else:
expression = expression.replace('$' + name,
str(replacements[name]))
return fetchData(requestContext, expression)
if tokens.call:
if tokens.call.funcname == 'template':
# if template propagates down here, it means the grammar didn't match the invocation
# as tokens.template. this generally happens if you try to pass non-numeric/string args
raise InputParameterError(
"invalid template() syntax, only string/numeric arguments are allowed"
)
if tokens.call.funcname == 'seriesByTag':
return fetchData(requestContext, tokens.call.raw)
try:
func = SeriesFunction(tokens.call.funcname)
except KeyError:
msg = 'Received request for unknown function: {func}'.format(
func=tokens.call.funcname)
log.warning(msg)
# even if input validation enforcement is disabled, there's nothing else we can do here
raise InputParameterError(msg)
rawArgs = tokens.call.args or []
if pipedArg is not None:
rawArgs.insert(0, pipedArg)
args = [
evaluateTokens(requestContext, arg, replacements)
for arg in rawArgs
]
requestContext['args'] = rawArgs
kwargs = dict([(kwarg.argname,
evaluateTokens(requestContext, kwarg.args[0],
replacements))
for kwarg in tokens.call.kwargs])
def handleInvalidParameters(e):
if not getattr(handleInvalidParameters, 'alreadyLogged', False):
log.warning(
'Received invalid parameters ({msg}): {func} ({args})'.
format(msg=str(e),
func=tokens.call.funcname,
args=', '.join(argList for argList in [
', '.join(str(arg) for arg in args),
', '.join('{k}={v}'.format(k=str(k), v=str(v))
for k, v in kwargs.items()),
] if argList)))
# only log invalid parameters once
setattr(handleInvalidParameters, 'alreadyLogged', True)
if settings.ENFORCE_INPUT_VALIDATION:
raise
if hasattr(func, 'params'):
try:
validateParams(tokens.call.funcname, func.params, args, kwargs)
except InputParameterError as e:
handleInvalidParameters(e)
try:
return func(requestContext, *args, **kwargs)
except NormalizeEmptyResultError:
return []
except InputParameterError as e:
handleInvalidParameters(e)
return evaluateScalarTokens(tokens)
def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None):
if tokens.template:
arglist = dict()
if tokens.template.kwargs:
arglist.update(
dict([(kwarg.argname, evaluateScalarTokens(kwarg.args[0]))
for kwarg in tokens.template.kwargs]))
if tokens.template.args:
arglist.update(
dict([(str(i + 1), evaluateScalarTokens(arg))
for i, arg in enumerate(tokens.template.args)]))
if 'template' in requestContext:
arglist.update(requestContext['template'])
return evaluateTokens(requestContext, tokens.template, arglist)
if tokens.expression:
if tokens.expression.pipedCalls.asList():
# when the expression has piped calls, we pop the right-most call and pass the remaining
# expression into it via pipedArg, to get the same result as a nested call
rightMost = tokens.expression.pipedCalls.pop()
return evaluateTokens(requestContext, rightMost, replacements,
tokens)
return evaluateTokens(requestContext, tokens.expression, replacements)
if tokens.pathExpression:
expression = tokens.pathExpression
if replacements:
for name in replacements:
if expression == '$' + name:
val = replacements[name]
if not isinstance(val, six.string_types):
return val
elif re.match(r'^-?[\d.]+$', val):
return float(val)
else:
return val
else:
expression = expression.replace('$' + name,
str(replacements[name]))
return fetchData(requestContext, expression)
if tokens.call:
if tokens.call.funcname == 'template':
# if template propagates down here, it means the grammar didn't match the invocation
# as tokens.template. this generally happens if you try to pass non-numeric/string args
raise InputParameterError(
"invalid template() syntax, only string/numeric arguments are allowed"
)
if tokens.call.funcname == 'seriesByTag':
return fetchData(requestContext, tokens.call.raw)
try:
func = SeriesFunction(tokens.call.funcname)
except KeyError:
raise InputParameterError(
'Received request for unknown function: {func}'.format(
func=tokens.call.funcname))
rawArgs = tokens.call.args or []
if pipedArg is not None:
rawArgs.insert(0, pipedArg)
args = [
evaluateTokens(requestContext, arg, replacements)
for arg in rawArgs
]
requestContext['args'] = rawArgs
kwargs = dict([(kwarg.argname,
evaluateTokens(requestContext, kwarg.args[0],
replacements))
for kwarg in tokens.call.kwargs])
if hasattr(func, 'params'):
try:
(args, kwargs) = validateParams(tokens.call.funcname,
func.params, args, kwargs)
except InputParameterError as e:
e.setSourceIdHeaders(requestContext.get('sourceIdHeaders', {}))
e.setTargets(requestContext.get('targets', []))
e.setFunction(tokens.call.funcname, args, kwargs)
if settings.ENFORCE_INPUT_VALIDATION:
raise
else:
log.warning('Validation Error: %s', str(e))
try:
return func(requestContext, *args, **kwargs)
except NormalizeEmptyResultError:
return []
except InputParameterError as e:
e.setSourceIdHeaders(requestContext.get('sourceIdHeaders', {}))
e.setTargets(requestContext.get('targets', []))
e.setFunction(tokens.call.funcname, args, kwargs)
raise
return evaluateScalarTokens(tokens)