def _read_log_error(self):
st = ZTCStore('php-fpm.record_log_error', self.options)
#st.clear()
self._err_old_status = st.get()
if self._err_old_status == None:
self._err_old_status = {}
try:
for f in self.config.get('err_log', '/var/log/php5-fpm.log').split(','):
log_f = open(f, 'r')
if self._err_old_status.has_key(f):
_offset = self._err_old_status[f]
if os.fstat(log_f.fileno()).st_size >= _offset:
log_f.seek(self._err_old_status[f])
else:
log_f.seek(0)
self._lines_err_record += ''.join(log_f.readlines())
else:
self._lines_err_record += ''.join(log_f.readlines())
_position = log_f.tell()
self._err_new_status[f] = _position
log_f.close()
st.set(self._err_new_status)
return True
except IOError as ioerr:
self.logger.exception('failed to open nginx access log')
return False
except Exception as err:
self.logger.exception('unkowon error')
return False
def _read_http_status(self):
""" urlopen and save to _page_data text of status page """
if self._http_new_status is not None:
# we've already retrieved it
return True
st = ZTCStore('nginx.http_status_page', self.options)
# get old http status
self._http_old_status = st.get()
try:
read_start = time.time()
url = "%s://%s:%s%s?auto" % (
self.config.get('proto', 'http'),
self.config.get('host', 'localhost'),
self.config.get('port', '80'),
self.config.get('status',
'/status-codes'))
try:
u = urllib2.urlopen(url, None, 1)
except TypeError:
u = urllib2.urlopen(url, None)
self._http_new_status = u.readlines()
u.close()
st.set(self._http_new_status)
return True
except urllib2.URLError:
self.logger.exception('failed to load test page')
# status page read failed
self._http_new_status = st.get()
self.ping_time = 0 # status page read failed
return False
def _read_log_error(self):
st = ZTCStore('php-fpm.record_log_error', self.options)
#st.clear()
self._err_old_status = st.get()
if self._err_old_status == None:
self._err_old_status = {}
try:
for f in self.config.get('err_log',
'/var/log/php5-fpm.log').split(','):
log_f = open(f, 'r')
if self._err_old_status.has_key(f):
_offset = self._err_old_status[f]
if os.fstat(log_f.fileno()).st_size >= _offset:
log_f.seek(self._err_old_status[f])
else:
log_f.seek(0)
self._lines_err_record += ''.join(log_f.readlines())
else:
self._lines_err_record += ''.join(log_f.readlines())
_position = log_f.tell()
self._err_new_status[f] = _position
log_f.close()
st.set(self._err_new_status)
return True
except IOError as ioerr:
self.logger.exception('failed to open nginx access log')
return False
except Exception as err:
self.logger.exception('unkowon error')
return False
def _read_status(self):
""" urlopen and save to _page_data text of status page """
if self._page_data is not None:
# we've already retrieved it
return True
st = ZTCStore('nginx.status_page', self.options)
try:
read_start = time.time()
url = "%s://%s:%s%s?auto" % (self.config.get(
'proto', 'http'), self.config.get(
'host', 'localhost'), self.config.get('port', '8080'),
self.config.get(
'resource', '/server-status'))
try:
u = urllib2.urlopen(url, None, 1)
except TypeError:
u = urllib2.urlopen(url, None)
self._page_data = u.readlines()
u.close()
st.set(self._page_data)
# calulate how many time was required:
self.ping_time = time.time() - read_start
return True
except urllib2.URLError:
self.logger.exception('failed to load test page')
# status page read failed
self._page_data = st.get()
self.ping_time = 0 # status page read failed
return False
def _read_status(self):
""" urlopen and save to _page_data text of status page """
if self._page_data is not None:
# we've already retrieved it
return True
st = ZTCStore('nginx.status_page', self.options)
try:
read_start = time.time()
url = "%s://%s:%s%s?auto" % (
self.config.get('proto', 'http'),
self.config.get('host', 'localhost'),
self.config.get('port', '8080'),
self.config.get('resource',
'/server-status'))
try:
u = urllib2.urlopen(url, None, 1)
except TypeError:
u = urllib2.urlopen(url, None)
self._page_data = u.readlines()
u.close()
st.set(self._page_data)
# calulate how many time was required:
self.ping_time = time.time() - read_start
return True
except urllib2.URLError:
self.logger.exception('failed to load test page')
# status page read failed
self._page_data = st.get()
self.ping_time = 0 # status page read failed
return False
def get_heap(self, metric):
""" get terracotta heap memory metrics """
st = ZTCStore('java.terracotta.heap', self.options)
st.ttl = 60
data = st.get()
if not data:
# no cache, get from jmx
data = self.get_prop('java.lang:type=Memory', 'HeapMemoryUsage')
st.set(data)
rt = self.extract_val_from_dict(data, metric)
if rt is None:
raise CheckFail('no such memory mertic')
else:
return rt
def get_codecache(self, metric):
""" get java codecache memory (non-heap) metrics """
self.logger.debug('in get_codecache')
st = ZTCStore('java.terracotta.codecache', self.options)
st.ttl = 60
data = st.get()
if not data:
# no cache, get from jmx
data = self.get_prop('java.lang:name=Code Cache,type=MemoryPool',
'Usage')
st.set(data)
rt = self.extract_val_from_dict(data, metric)
if rt is None:
raise CheckFail('no such memory mertic')
else:
return rt
def _load_jmap_heap(self):
"""
Load jmap from cache or runs it
"""
key = "jmap_heap_%i" % (self.pid,)
c = ZTCStore(key, self.options, self.logger, ttl=60)
jmap_heap = c.get()
if jmap_heap:
ret = jmap_heap.data
else:
# no data in cache
jmap_heap_data = self._run_jmap_heap()
if jmap_heap_data:
c.set(jmap_heap_data)
ret = jmap_heap
else:
ret = None
return ret
def get_dbstats(self, dbname, metric):
""" get db.stats metric for specified database """
self._connect()
# get database object
if dbname not in self.dbs:
self.dbs[dbname] = self.connection[dbname]
db = self.dbs[dbname]
# load dbstats for specified database
c = ZTCStore('mongodb_dbstats_' + dbname, self.options, 120)
dbstats = c.get()
if not dbstats:
dbstats = db.command('dbstats')
c.set(dbstats)
ret = dbstats[metric]
if metric == 'nsSizeMB':
ret = ret * 1024 * 1024
return ret
def get_dbstats(self, dbname, metric):
""" get db.stats metric for specified database """
self._connect()
# get database object
if dbname not in self.dbs:
self.dbs[dbname] = self.connection[dbname]
db = self.dbs[dbname]
# load dbstats for specified database
c = ZTCStore('mongodb_dbstats_' + dbname, self.options, 120)
dbstats = c.get()
if not dbstats:
dbstats = db.command('dbstats')
c.set(dbstats)
ret = dbstats[metric]
if metric == 'nsSizeMB':
ret = ret * 1024 * 1024
return ret
def _read_http_status(self):
""" urlopen and save to _page_data text of status page """
if self._http_new_status is not None:
# we've already retrieved it
return True
st = ZTCStore('nginx.http_status_page', self.options)
# get old http status
self._http_old_status = st.get()
try:
read_start = time.time()
url = "%s://%s:%s%s?auto" % (self.config.get(
'proto', 'http'), self.config.get(
'host', 'localhost'), self.config.get('port', '80'),
self.config.get(
'status', '/status-codes'))
try:
u = urllib2.urlopen(url, None, 1)
except TypeError:
u = urllib2.urlopen(url, None)
self._http_new_status = u.readlines()
u.close()
st.set(self._http_new_status)
return True
except urllib2.URLError:
self.logger.exception('failed to load test page')
# status page read failed
self._http_new_status = st.get()
self.ping_time = 0 # status page read failed
return False
def _read_http_status(self):
st = ZTCStore('nginx.http_status_page', self.options)
# store dict, ex. dict = { file:offset }
# get old http status
self._http_old_status = st.get()
if self._http_old_status == None:
self._http_old_status = {}
try:
for f in self.config.get('logfile', '/var/log/nginx/access.log').split(','):
log_f = open(f,'r')
if self._http_old_status.has_key(f):
# get _offset value
_offset = self._http_old_status[f]
# comp file size and _offset
if os.fstat(log_f.fileno()).st_size >= _offset:
# 当前文件size大于offset
log_f.seek(self._http_old_status[f])
else:
# 当日志清空或被切割时,当前size小于offset
log_f.seek(0)
self._lines_record += ''.join(log_f.readlines())
else:
self._lines_record += ''.join(log_f.readlines())
# 得到当前文件的偏移位
_position = log_f.tell()
self._http_new_status[f] = _position
log_f.close()
st.set(self._http_new_status)
return True
except IOError as ioerr:
self.logger.exception('failed to open nginx access log')
return False
except Exception as err:
self.logger.exception('unkowon error')
return False
def get_heap(self, metric):
""" get terracotta heap memory metrics """
st = ZTCStore('java.terracotta.heap', self.options)
st.ttl = 60
data = st.get()
if not data:
# no cache, get from jmx
data = self.get_prop('java.lang:type=Memory', 'HeapMemoryUsage')
st.set(data)
rt = self.extract_val_from_dict(data, metric)
if rt is None:
raise CheckFail('no such memory mertic')
else:
return rt
def get_codecache(self, metric):
""" get java codecache memory (non-heap) metrics """
self.logger.debug('in get_codecache')
st = ZTCStore('java.terracotta.codecache', self.options)
st.ttl = 60
data = st.get()
if not data:
# no cache, get from jmx
data = self.get_prop('java.lang:name=Code Cache,type=MemoryPool',
'Usage')
st.set(data)
rt = self.extract_val_from_dict(data, metric)
if rt is None:
raise CheckFail('no such memory mertic')
else:
return rt
def clear_store(self):
""" urlopen and save to _page_data text of status page """
st = ZTCStore('nginx.http_status_page', self.options)
# get old http status
st.clear()
def clear_store(self):
""" urlopen and save to _page_data text of status page """
st = ZTCStore('nginx.http_status_page', self.options)
# get old http status
st.clear()
def save_to_store(self, data):
st = ZTCStore('nginx_statuses', self.options)
st.set(data)
def read_from_store(self):
st = ZTCStore('nginx_statuses', self.options)
return st.get()
def save_to_store(self, data):
st = ZTCStore('nginx_reqtime', self.options)
st.set(data)
def read_from_store(self):
st = ZTCStore('nginx_reqtime', self.options)
return st.get()
def read_from_store(self):
""" Read from ZTCStore cache """
st = ZTCStore('nginx_statuses', self.options)
return st.get()
def read_from_store(self):
st = ZTCStore('nginx_reqtime', self.options)
return st.get()
def _save_metrics_to_cache(self, data):
st = ZTCStore('apache_reqtime', self.options)
st.set(data)
def _get_metrics_from_cache(self):
""" load page response metrics from ZTCStore key apache_reqtime """
st = ZTCStore('apache_reqtime', self.options)
return st.get()
def save_to_store(self, data):
st = ZTCStore('nginx_reqtime', self.options)
st.set(data)
def save_to_store(self, data):
""" Save fetched status to ZTCStore cache. """
st = ZTCStore('nginx_statuses', self.options)
st.set(data)
def _save_metrics_to_cache(self, data):
st = ZTCStore('apache_reqtime', self.options)
st.set(data)
def _get_metrics_from_cache(self):
""" load page response metrics from ZTCStore key apache_reqtime """
st = ZTCStore('apache_reqtime', self.options)
return st.get()
