def set_metadata_view(request):
results = {}
if request.method == 'GET':
metric = request.GET['metric']
key = request.GET['key']
value = request.GET['value']
try:
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key))
elif request.method == 'POST':
if request.META.get('CONTENT_TYPE') == 'application/json':
operations = json.loads( request.body )
else:
operations = json.loads( request.POST['operations'] )
for op in operations:
metric = None
try:
metric, key, value = op['metric'], op['key'], op['value']
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
if metric:
results[metric] = dict(error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric)
else:
results = dict(error='Invalid request method')
return json_response_for(request, results)
def fetch(self, startTime, endTime):
try:
data = whisper.fetch(self.fs_path, startTime, endTime)
except IOError:
log.exception("Failed fetch of whisper file '%s'" % self.fs_path)
return None
if not data:
return None
time_info, values = data
(start, end, step) = time_info
meta_info = whisper.info(self.fs_path)
aggregation_method = meta_info['aggregationMethod']
lowest_step = min(
[i['secondsPerPoint'] for i in meta_info['archives']])
# Merge in data from carbon's cache
cached_datapoints = []
try:
cached_datapoints = CarbonLink().query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" %
self.real_metric_path)
cached_datapoints = []
if isinstance(cached_datapoints, dict):
cached_datapoints = cached_datapoints.items()
values = merge_with_cache(cached_datapoints, start, step, values,
aggregation_method)
return time_info, values
def set_metadata_view(request):
results = {}
if request.method == "GET":
metric = request.GET["metric"]
key = request.GET["key"]
value = request.GET["value"]
try:
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key))
elif request.method == "POST":
if request.META.get("CONTENT_TYPE") == "application/json":
operations = json.loads(request.body)
else:
operations = json.loads(request.POST["operations"])
for op in operations:
metric = None
try:
metric, key, value = op["metric"], op["key"], op["value"]
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
if metric:
results[metric] = dict(
error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric
)
else:
results = dict(error="Invalid request method")
return json_response_for(request, results)
def fetch(self, startTime, endTime):
try:
data = whisper.fetch(self.fs_path, startTime, endTime)
except IOError:
log.exception("Failed fetch of whisper file '%s'" % self.fs_path)
return None
if not data:
return None
time_info, values = data
(start,end,step) = time_info
meta_info = whisper.info(self.fs_path)
aggregation_method = meta_info['aggregationMethod']
lowest_step = min([i['secondsPerPoint'] for i in meta_info['archives']])
# Merge in data from carbon's cache
cached_datapoints = []
try:
cached_datapoints = CarbonLink().query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
if isinstance(cached_datapoints, dict):
cached_datapoints = cached_datapoints.items()
values = merge_with_cache(cached_datapoints,
start,
step,
values,
aggregation_method)
return time_info, values
def set_metadata_view(request):
results = {}
if request.method == 'GET':
metric = request.GET['metric']
key = request.GET['key']
value = request.GET['value']
try:
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key))
elif request.method == 'POST':
if request.META.get('CONTENT_TYPE') == 'application/json':
operations = json.loads( request.body )
else:
operations = json.loads( request.POST['operations'] )
for op in operations:
metric = None
try:
metric, key, value = op['metric'], op['key'], op['value']
results[metric] = CarbonLink.set_metadata(metric, key, value)
except:
log.exception()
if metric:
results[metric] = dict(error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric)
else:
results = dict(error="Invalid request method")
return json_response_for(request, results)
def merge_with_carbonlink(metric, start, step, values, aggregation_method=None, raw_step=None):
"""Get points from carbonlink and merge them with existing values."""
cached_datapoints = []
try:
cached_datapoints = CarbonLink().query(metric)
except BaseException:
log.exception("Failed CarbonLink query '%s'" % metric)
cached_datapoints = []
if isinstance(cached_datapoints, dict):
cached_datapoints = list(cached_datapoints.items())
return merge_with_cache(
cached_datapoints, start, step, values,
func=aggregation_method, raw_step=raw_step)
def merge_with_carbonlink(metric, start, step, values, aggregation_method=None, raw_step=None):
"""Get points from carbonlink and merge them with existing values."""
cached_datapoints = []
try:
cached_datapoints = CarbonLink().query(metric)
except BaseException:
log.exception("Failed CarbonLink query '%s'" % metric)
cached_datapoints = []
if isinstance(cached_datapoints, dict):
cached_datapoints = list(cached_datapoints.items())
return merge_with_cache(
cached_datapoints, start, step, values,
func=aggregation_method, raw_step=raw_step)
def find_nodes(self, query, cache_incomplete_nodes=None):
clean_patterns = query.pattern.replace('\\', '')
has_wildcard = clean_patterns.find('{') > -1 or clean_patterns.find(
'[') > -1 or clean_patterns.find('*') > -1 or clean_patterns.find(
'?') > -1
if cache_incomplete_nodes is None:
cache_incomplete_nodes = {}
# CarbonLink has some hosts
if CarbonLink.hosts:
metric = clean_patterns
# Let's combine these two cases:
# 1) has_wildcard
# 2) single metric query
# Expand queries in CarbonLink
# we will get back a list of tuples (metric_name, is_leaf) here.
# For example,
# [(metric1, False), (metric2, True)]
metrics = CarbonLink.expand_query(metric)
# dedup, because of BranchNodes
metrics = list(set(metrics))
# check all metrics in same valid query range
prechecks = []
for m, is_leaf in metrics:
if is_leaf:
prechecks.append(CarbonLink.precheck(m, query.startTime))
else: # return True for BranchNode
prechecks.append((True, True))
exists = all((exist for exist, partial_exist in prechecks))
partial_exists = all(
(partial_exist for exist, partial_exist in prechecks))
if exists:
for metric, is_leaf in metrics:
if is_leaf:
reader = CarbonCacheReader(metric)
yield LeafNode(metric, reader)
else:
yield BranchNode(metric)
elif partial_exists:
for metric, is_leaf in metrics:
if is_leaf:
reader = CarbonCacheReader(metric)
cache_incomplete_nodes[metric] = LeafNode(
metric, reader)
else:
cache_incomplete_nodes[metric] = BranchNode(metric)
def fetch(self, startTime, endTime):
data = self.cassandra_node.read(startTime, endTime)
time_info = (data.startTime, data.endTime, data.timeStep)
values = list(data.values)
#log.exception("{0}, {1}, {2}".format(data.startTime, data.endTime, data.timeStep))
#log.exception(values)
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" %
self.real_metric_path)
cached_datapoints = []
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % data.timeStep)
try:
i = int(interval - data.startTime) / data.timeStep
values[i] = value
except:
pass
return (time_info, values)
def fetch(self, start_time, end_time):
# Fetch data from carbon cache through CarbonLink
schema = CarbonLink.get_storage_schema(self.metric)
archives = schema["archives"]
# Get lowest step
lowest_step = min([arch[0] for arch in archives])
now = int(time.time())
max_retention = max([arch[0] * arch[1] for arch in archives])
oldest_time = now - max_retention
# format and extract from/until time
from_and_until_time = self._format_and_extract_time(
start_time, end_time, max_retention)
if not from_and_until_time:
return None
from_time, until_time = from_and_until_time
# calcucate step
diff = now - from_time
# sorted_archives = sorted(archives, key=lambda x: x[0] * x[1])
step = self._calculate_step(archives, diff)
if not step:
return None
# Only check carbon-cache if step == lowest_step
if step == lowest_step:
return self._query_and_format_cache_data(from_time, until_time,
step)
return None
def fetch(self, startTime, endTime, now=None, requestContext=None):
try:
data = whisper.fetch(self.fs_path, startTime, endTime, now)
except IOError:
log.exception("Failed fetch of whisper file '%s'" % self.fs_path)
return None
if not data:
return None
time_info, values = data
(start, end, step) = time_info
meta_info = whisper.info(self.fs_path)
aggregation_method = meta_info['aggregationMethod']
# Merge in data from carbon's cache
cached_datapoints = []
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" %
self.real_metric_path)
cached_datapoints = []
if isinstance(cached_datapoints, dict):
cached_datapoints = cached_datapoints.items()
values = merge_with_cache(cached_datapoints, start, step, values,
aggregation_method)
return time_info, values
def fetch(self, startTime, endTime):
data = whisper.fetch(self.fs_path, startTime, endTime)
if not data:
return None
time_info, values = data
(start,end,step) = time_info
meta_info = whisper.info(self.fs_path)
lowest_step = min([i['secondsPerPoint'] for i in meta_info['archives']])
# Merge in data from carbon's cache
cached_datapoints = []
try:
if step == lowest_step:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
if isinstance(cached_datapoints, dict):
cached_datapoints = cached_datapoints.items()
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % step)
try:
i = int(interval - start) / step
values[i] = value
except:
pass
return (time_info, values)
def fetch(self, startTime, endTime):
data = whisper.fetch(self.fs_path, startTime, endTime)
if not data:
return None
time_info, values = data
(start, end, step) = time_info
meta_info = whisper.info(self.fs_path)
lowest_step = min([i['secondsPerPoint'] for i in meta_info['archives']])
# Merge in data from carbon's cache
cached_datapoints = []
try:
if step == lowest_step:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
if isinstance(cached_datapoints, dict):
cached_datapoints = cached_datapoints.items()
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % step)
try:
i = int(interval - start) / step
values[i] = value
except:
pass
return (time_info, values)
def fetch(self, startTime, endTime):
data = whisper.fetch(self.fs_path, startTime, endTime)
if not data:
return None
time_info, values = data
(start,end,step) = time_info
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % step)
try:
i = int(interval - start) / step
values[i] = value
except:
pass
return (time_info, values)
def fetch(self, startTime, endTime):
data = whisper.fetch(self.fs_path, startTime, endTime)
if not data:
return None
consolidationFunc = ""
whisper_info = whisper.info(self.fs_path)
if "aggregationMethod" in whisper_info:
aggregationMethod = whisper_info["aggregationMethod"]
if aggregationMethod == 'min' or aggregationMethod == 'max':
consolidationFunc = aggregationMethod
time_info, values = data
(start,end,step) = time_info
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % step)
try:
i = int(interval - start) / step
values[i] = value
except:
pass
return (time_info, values, consolidationFunc)
def fetch(self, startTime, endTime):
data = whisper.fetch(self.fs_path, startTime, endTime)
if not data:
return None
time_info, values = data
(start, end, step) = time_info
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" %
self.real_metric_path)
cached_datapoints = []
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % step)
try:
i = int(interval - start) / step
values[i] = value
except:
pass
return (time_info, values)
def fetch(self, startTime, endTime):
data = whisper.fetch(self.fs_path, startTime, endTime)
if not data:
return None
time_info, values = data
(start, end, step) = time_info
# Merge in data from carbon's cache
if settings.REPLICATION_FACTOR != 0:
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" %
self.real_metric_path)
cached_datapoints = []
else:
cached_datapoints = []
if isinstance(cached_datapoints, dict):
cached_datapoints = cached_datapoints.items()
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % step)
try:
i = int(interval - start) / step
values[i] = value
except:
pass
return (time_info, values)
def fetch(self, startTime, endTime):
data = self.ceres_node.read(startTime, endTime)
time_info = (data.startTime, data.endTime, data.timeStep)
values = list(data.values)
# Merge in data from carbon's cache
if settings.REPLICATION_FACTOR != 0:
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" %
self.real_metric_path)
cached_datapoints = []
else:
cached_datapoints = []
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % data.timeStep)
try:
i = int(interval - data.startTime) / data.timeStep
values[i] = value
except:
pass
return (time_info, values)
def cache_metric(request):
queryParams = request.GET.copy()
metric = queryParams.get('metric', 'yun.test')
datapoints = CarbonLink.query(metric)
result_json_obj = {"target": metric, "datapoints": datapoints}
response = HttpResponse(content=json.dumps(result_json_obj),
content_type='application/json')
return response
def get_metadata_view(request):
key = request.REQUEST['key']
metrics = request.REQUEST.getlist('metric')
results = {}
for metric in metrics:
try:
results[metric] = CarbonLink.get_metadata(metric, key)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.get_metadata(%s, %s)" % (metric, key))
return HttpResponse(json.dumps(results), mimetype='application/json')
def get_metadata_view(request):
key = request.REQUEST['key']
metrics = request.REQUEST.getlist('metric')
results = {}
for metric in metrics:
try:
results[metric] = CarbonLink.get_metadata(metric, key)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.get_metadata(%s, %s)" % (metric, key))
return json_response_for(request, results)
def get_metadata_view(request):
key = request.REQUEST['key']
metrics = request.REQUEST.getlist('metric')
results = {}
for metric in metrics:
try:
results[metric] = CarbonLink.get_metadata(metric, key)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.get_metadata(%s, %s)" % (metric, key))
return json_response_for(request, results)
def get_metadata_view(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
key = queryParams.get('key')
metrics = queryParams.getlist('metric')
results = {}
for metric in metrics:
try:
results[metric] = CarbonLink.get_metadata(metric, key)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.get_metadata(%s, %s)" % (metric, key))
return json_response_for(request, results)
def get_metadata_view(request):
queryParams = request.GET.copy()
queryParams.update(request.POST)
key = queryParams.get('key')
metrics = queryParams.getlist('metric')
results = {}
for metric in metrics:
try:
results[metric] = CarbonLink.get_metadata(metric, key)
except:
log.exception()
results[metric] = dict(error="Unexpected error occurred in CarbonLink.get_metadata(%s, %s)" % (metric, key))
return json_response_for(request, results)
def get_metadata_view(request):
key = request.REQUEST['key']
metrics = request.REQUEST.getlist('metric')
results = {}
for metric in metrics:
try:
results[metric] = CarbonLink.get_metadata(metric, key)
except:
log.exception()
results[metric] = dict(
error=
"Unexpected error occurred in CarbonLink.get_metadata(%s, %s)"
% (metric, key))
return HttpResponse(json.dumps(results), mimetype='application/json')
def _query_and_format_cache_data(self, from_time, until_time, step):
cached_results = CarbonLink.query(self.metric)
if cached_results:
from_interval = int(from_time - (from_time % step)) + step
until_interval = int(until_time - (until_time % step)) + step
if from_interval == until_interval:
until_interval += step
points = (until_interval - from_interval) // step
values = [None] * points
time_info = (from_interval, until_interval, step)
for (timestamp, value) in cached_results:
interval = int(timestamp - (timestamp % step))
index = (interval - from_interval) / step
if index < 0 or index >= points:
continue
values[index] = value
return time_info, values
def fetch(self, startTime, endTime):
data = self.ceres_node.read(startTime, endTime)
time_info = (data.startTime, data.endTime, data.timeStep)
values = list(data.values)
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" %
self.real_metric_path)
cached_datapoints = []
values = merge_with_cache(cached_datapoints, data.startTime,
data.timeStep, values)
return time_info, values
def fetch(self, startTime, endTime):
data = self.ceres_node.read(startTime, endTime)
time_info = (data.startTime, data.endTime, data.timeStep)
values = list(data.values)
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
values = merge_with_cache(cached_datapoints,
data.startTime,
data.timeStep,
values)
return time_info, values
def fetch(self, startTime, endTime):
data = self.ceres_node.read(startTime, endTime)
time_info = (data.startTime, data.endTime, data.timeStep)
values = list(data.values)
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % data.timeStep)
try:
i = int(interval - data.startTime) / data.timeStep
values[i] = value
except:
pass
return (time_info, values)
def CarbonLink():
"""Return a carbonlink instance."""
# Late import to avoid pulling out too many dependencies with
# readers.py which is usually imported by plugins.
from graphite.carbonlink import CarbonLink
return CarbonLink()
