def send(self):
log.info("FindRequest.send(host=%s, query=%s) called" %
(self.store.host, self.query))
self.cachedResult = cache.get(self.cacheKey)
if self.cachedResult is not None:
log.info("FindRequest(host=%s, query=%s) using cached result" %
(self.store.host, self.query))
return
self.connection = HTTPConnectionWithTimeout(self.store.host)
self.connection.timeout = settings.REMOTE_FIND_TIMEOUT
query_params = [
('local', '1'),
('format', 'pickle'),
('query', self.query.pattern),
]
if self.query.startTime:
query_params.append(('from', self.query.startTime))
if self.query.endTime:
query_params.append(('until', self.query.endTime))
query_string = urlencode(query_params)
try:
self.connection.request('GET', '/metrics/find/?' + query_string)
except:
log.exception(
"FindRequest.send(host=%s, query=%s) exception during request"
% (self.store.host, self.query))
self.store.fail()
self.failed = True
def fetch(self, startTime, endTime):
data = whisper.fetch(self.fs_path, startTime, endTime)
if not data:
return None
time_info, values = data
(start,end,step) = time_info
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % step)
try:
i = int(interval - start) / step
values[i] = value
except:
pass
return (time_info, values)
def get_results(self):
if self.failed:
return
if self.cachedResult is not None:
results = self.cachedResult
else:
if self.connection is None:
self.send()
try:
response = self.connection.getresponse()
assert response.status == 200, "received error response %s - %s" % (response.status, response.reason)
result_data = response.read()
results = unpickle.loads(result_data)
except:
log.exception("FindRequest.get_results(host=%s, query=%s) exception processing response" % (self.store.host, self.query))
self.store.fail()
return
cache.set(self.cacheKey, results, settings.FIND_CACHE_DURATION)
for node_info in results:
if node_info.get('is_leaf'):
reader = RemoteReader(self.store, node_info, bulk_query=self.query.pattern)
node = LeafNode(node_info['path'], reader)
else:
node = BranchNode(node_info['path'])
node.local = False
yield node
def wait_for_results():
if wait_lock.acquire(
False
): # the FetchInProgress that gets waited on waits for the actual completion
try:
response = connection.getresponse()
if response.status != 200:
raise Exception(
"Error response %d %s from %s" %
(response.status, response.reason, url))
pickled_response = response.read()
results = unpickle.loads(pickled_response)
self.cache_lock.acquire()
self.request_cache[url] = results
self.cache_lock.release()
completion_event.set()
return results
except:
completion_event.set()
self.store.fail()
log.exception("Error requesting %s" % url)
raise
else: # otherwise we just wait on the completion_event
completion_event.wait(settings.REMOTE_FETCH_TIMEOUT)
cached_results = self.request_cache.get(url)
if cached_results is None:
raise Exception(
"Passive remote fetch failed to find cached results")
else:
return cached_results
def wait_for_results():
if wait_lock.acquire(False): # the FetchInProgress that gets waited on waits for the actual completion
try:
response = connection.getresponse()
if response.status != 200:
raise Exception("Error response %d %s from %s" % (response.status, response.reason, url))
pickled_response = response.read()
results = unpickle.loads(pickled_response)
self.cache_lock.acquire()
self.request_cache[url] = results
self.cache_lock.release()
completion_event.set()
return results
except:
completion_event.set()
self.store.fail()
log.exception("Error requesting %s" % url)
raise
else: # otherwise we just wait on the completion_event
completion_event.wait(settings.REMOTE_FETCH_TIMEOUT)
cached_results = self.request_cache.get(url)
if cached_results is None:
raise Exception("Passive remote fetch failed to find cached results")
else:
return cached_results
def send(self):
log.info("FindRequest.send(host=%s, query=%s) called" % (self.store.host, self.query))
self.cachedResult = cache.get(self.cacheKey)
if self.cachedResult is not None:
log.info("FindRequest(host=%s, query=%s) using cached result" % (self.store.host, self.query))
return
self.connection = HTTPConnectionWithTimeout(self.store.host)
self.connection.timeout = settings.REMOTE_FIND_TIMEOUT
query_params = [
('local', '1'),
('format', 'pickle'),
('query', self.query.pattern),
]
if self.query.startTime:
query_params.append( ('from', self.query.startTime) )
if self.query.endTime:
query_params.append( ('until', self.query.endTime) )
query_string = urlencode(query_params)
try:
self.connection.request('GET', '/metrics/find/?' + query_string)
except:
log.exception("FindRequest.send(host=%s, query=%s) exception during request" % (self.store.host, self.query))
self.store.fail()
self.failed = True
def is_local_interface(host):
if ':' in host:
host = host.split(':', 1)[0]
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.connect((host, 4242))
local_ip = sock.getsockname()[0]
sock.close()
except:
log.exception("Failed to open socket with %s" % host)
raise
if local_ip == host:
return True
return False
def fetch(self, startTime, endTime):
# Start the fetch on each node
results = [ n.fetch(startTime, endTime) for n in self.nodes ]
# Wait for any asynchronous operations to complete
for i, result in enumerate(results):
if isinstance(result, FetchInProgress):
try:
results[i] = result.waitForResults()
except:
log.exception("Failed to complete subfetch")
results[i] = None
results = [r for r in results if r is not None]
if not results:
raise Exception("All sub-fetches failed")
return reduce(self.merge, results)
def _find_paths(self, current_dir, patterns):
"""Recursively generates absolute paths whose components underneath current_dir
match the corresponding pattern in patterns"""
pattern = patterns[0]
patterns = patterns[1:]
try:
entries = os.listdir(current_dir)
except OSError as e:
log.exception(e)
entries = []
subdirs = [e for e in entries if isdir(join(current_dir, e))]
matching_subdirs = match_entries(subdirs, pattern)
if len(
patterns
) == 1 and RRDReader.supported: #the last pattern may apply to RRD data sources
files = [e for e in entries if isfile(join(current_dir, e))]
rrd_files = match_entries(files, pattern + ".rrd")
if rrd_files: #let's assume it does
datasource_pattern = patterns[0]
for rrd_file in rrd_files:
absolute_path = join(current_dir, rrd_file)
yield absolute_path + self.DATASOURCE_DELIMETER + datasource_pattern
if patterns: #we've still got more directories to traverse
for subdir in matching_subdirs:
absolute_path = join(current_dir, subdir)
for match in self._find_paths(absolute_path, patterns):
yield match
else: #we've got the last pattern
files = [e for e in entries if isfile(join(current_dir, e))]
matching_files = match_entries(files, pattern + '.*')
for basename in matching_files + matching_subdirs:
yield join(current_dir, basename)
def fetch(self, startTime, endTime):
data = self.ceres_node.read(startTime, endTime)
time_info = (data.startTime, data.endTime, data.timeStep)
values = list(data.values)
# Merge in data from carbon's cache
try:
cached_datapoints = CarbonLink.query(self.real_metric_path)
except:
log.exception("Failed CarbonLink query '%s'" % self.real_metric_path)
cached_datapoints = []
for (timestamp, value) in cached_datapoints:
interval = timestamp - (timestamp % data.timeStep)
try:
i = int(interval - data.startTime) / data.timeStep
values[i] = value
except:
pass
return (time_info, values)
def _find_paths(self, current_dir, patterns):
"""Recursively generates absolute paths whose components underneath current_dir
match the corresponding pattern in patterns"""
pattern = patterns[0]
patterns = patterns[1:]
try:
entries = os.listdir(current_dir)
except OSError as e:
log.exception(e)
entries = []
subdirs = [e for e in entries if isdir( join(current_dir,e) )]
matching_subdirs = match_entries(subdirs, pattern)
if len(patterns) == 1 and RRDReader.supported: #the last pattern may apply to RRD data sources
files = [e for e in entries if isfile( join(current_dir,e) )]
rrd_files = match_entries(files, pattern + ".rrd")
if rrd_files: #let's assume it does
datasource_pattern = patterns[0]
for rrd_file in rrd_files:
absolute_path = join(current_dir, rrd_file)
yield absolute_path + self.DATASOURCE_DELIMETER + datasource_pattern
if patterns: #we've still got more directories to traverse
for subdir in matching_subdirs:
absolute_path = join(current_dir, subdir)
for match in self._find_paths(absolute_path, patterns):
yield match
else: #we've got the last pattern
files = [e for e in entries if isfile( join(current_dir,e) )]
matching_files = match_entries(files, pattern + '.*')
for basename in matching_files + matching_subdirs:
yield join(current_dir, basename)
def get_results(self):
if self.failed:
return
if self.cachedResult is not None:
results = self.cachedResult
else:
if self.connection is None:
self.send()
try:
response = self.connection.getresponse()
assert response.status == 200, "received error response %s - %s" % (
response.status, response.reason)
result_data = response.read()
results = unpickle.loads(result_data)
except:
log.exception(
"FindRequest.get_results(host=%s, query=%s) exception processing response"
% (self.store.host, self.query))
self.store.fail()
return
cache.set(self.cacheKey, results, settings.FIND_CACHE_DURATION)
for node_info in results:
if node_info.get('is_leaf'):
reader = RemoteReader(self.store,
node_info,
bulk_query=self.query.pattern)
node = LeafNode(node_info['path'], reader)
else:
node = BranchNode(node_info['path'])
node.local = False
yield node
def fetch(self, startTime, endTime):
query_params = [('target', self.query), ('format', 'pickle'),
('local', '1'), ('noCache', '1'),
('from', str(int(startTime))),
('until', str(int(endTime)))]
query_string = urlencode(query_params)
urlpath = '/render/?' + query_string
url = "http://%s%s" % (self.store.host, urlpath)
# Quick cache check up front
self.clean_cache()
cached_results = self.request_cache.get(url)
if cached_results:
for series in cached_results:
if series['name'] == self.metric_path:
time_info = (series['start'], series['end'],
series['step'])
return (time_info, series['values'])
# Synchronize with other RemoteReaders using the same bulk query.
# Despite our use of thread synchronization primitives, the common
# case is for synchronizing asynchronous fetch operations within
# a single thread.
(request_lock, wait_lock,
completion_event) = self.get_request_locks(url)
if request_lock.acquire(
False): # we only send the request the first time we're called
try:
log.info("RemoteReader.request_data :: requesting %s" % url)
connection = HTTPConnectionWithTimeout(self.store.host)
connection.timeout = settings.REMOTE_FETCH_TIMEOUT
connection.request('GET', urlpath)
except:
completion_event.set()
self.store.fail()
log.exception("Error requesting %s" % url)
raise
def wait_for_results():
if wait_lock.acquire(
False
): # the FetchInProgress that gets waited on waits for the actual completion
try:
response = connection.getresponse()
if response.status != 200:
raise Exception(
"Error response %d %s from %s" %
(response.status, response.reason, url))
pickled_response = response.read()
results = unpickle.loads(pickled_response)
self.cache_lock.acquire()
self.request_cache[url] = results
self.cache_lock.release()
completion_event.set()
return results
except:
completion_event.set()
self.store.fail()
log.exception("Error requesting %s" % url)
raise
else: # otherwise we just wait on the completion_event
completion_event.wait(settings.REMOTE_FETCH_TIMEOUT)
cached_results = self.request_cache.get(url)
if cached_results is None:
raise Exception(
"Passive remote fetch failed to find cached results")
else:
return cached_results
def extract_my_results():
for series in wait_for_results():
if series['name'] == self.metric_path:
time_info = (series['start'], series['end'],
series['step'])
return (time_info, series['values'])
return graphitequery.readers.FetchInProgress(extract_my_results)
if series_with_duplicate_names == empty_duplicates and len(empty_duplicates) > 0: # if they're all empty
empty_duplicates.pop() # make sure we leave one in seriesList
for series in empty_duplicates:
seriesList.remove(series)
return seriesList
retries = 1 # start counting at one to make log output and settings more readable
while True:
try:
seriesList = _fetchData(pathExpr,startTime, endTime, requestContext, seriesList)
return seriesList
except Exception, e:
if retries >= settings.MAX_FETCH_RETRIES:
log.exception("Failed after %i retry! See: %s" % (settings.MAX_FETCH_RETRIES, e))
raise Exception("Failed after %i retry! See: %s" % (settings.MAX_FETCH_RETRIES, e))
else:
log.exception("Got an exception when fetching data! See: %s Will do it again! Run: %i of %i" %
(e, retries, settings.MAX_FETCH_RETRIES))
retries += 1
def nonempty(series):
for value in series:
if value is not None:
return True
return False
def fetch(self, startTime, endTime):
query_params = [
('target', self.query),
('format', 'pickle'),
('local', '1'),
('noCache', '1'),
('from', str( int(startTime) )),
('until', str( int(endTime) ))
]
query_string = urlencode(query_params)
urlpath = '/render/?' + query_string
url = "http://%s%s" % (self.store.host, urlpath)
# Quick cache check up front
self.clean_cache()
cached_results = self.request_cache.get(url)
if cached_results:
for series in cached_results:
if series['name'] == self.metric_path:
time_info = (series['start'], series['end'], series['step'])
return (time_info, series['values'])
# Synchronize with other RemoteReaders using the same bulk query.
# Despite our use of thread synchronization primitives, the common
# case is for synchronizing asynchronous fetch operations within
# a single thread.
(request_lock, wait_lock, completion_event) = self.get_request_locks(url)
if request_lock.acquire(False): # we only send the request the first time we're called
try:
log.info("RemoteReader.request_data :: requesting %s" % url)
connection = HTTPConnectionWithTimeout(self.store.host)
connection.timeout = settings.REMOTE_FETCH_TIMEOUT
connection.request('GET', urlpath)
except:
completion_event.set()
self.store.fail()
log.exception("Error requesting %s" % url)
raise
def wait_for_results():
if wait_lock.acquire(False): # the FetchInProgress that gets waited on waits for the actual completion
try:
response = connection.getresponse()
if response.status != 200:
raise Exception("Error response %d %s from %s" % (response.status, response.reason, url))
pickled_response = response.read()
results = unpickle.loads(pickled_response)
self.cache_lock.acquire()
self.request_cache[url] = results
self.cache_lock.release()
completion_event.set()
return results
except:
completion_event.set()
self.store.fail()
log.exception("Error requesting %s" % url)
raise
else: # otherwise we just wait on the completion_event
completion_event.wait(settings.REMOTE_FETCH_TIMEOUT)
cached_results = self.request_cache.get(url)
if cached_results is None:
raise Exception("Passive remote fetch failed to find cached results")
else:
return cached_results
def extract_my_results():
for series in wait_for_results():
if series['name'] == self.metric_path:
time_info = (series['start'], series['end'], series['step'])
return (time_info, series['values'])
return graphitequery.readers.FetchInProgress(extract_my_results)
empty_duplicates.pop() # make sure we leave one in seriesList
for series in empty_duplicates:
seriesList.remove(series)
return seriesList
retries = 1 # start counting at one to make log output and settings more readable
while True:
try:
seriesList = _fetchData(pathExpr, startTime, endTime,
requestContext, seriesList)
return seriesList
except Exception, e:
if retries >= settings.MAX_FETCH_RETRIES:
log.exception("Failed after %i retry! See: %s" %
(settings.MAX_FETCH_RETRIES, e))
raise Exception("Failed after %i retry! See: %s" %
(settings.MAX_FETCH_RETRIES, e))
else:
log.exception(
"Got an exception when fetching data! See: %s Will do it again! Run: %i of %i"
% (e, retries, settings.MAX_FETCH_RETRIES))
retries += 1
def nonempty(series):
for value in series:
if value is not None:
return True
return False