def clusterValidate():
serverAddress = config.get('RootContainerSection', 'serverAddress')
rootContainer = config.get('RootContainerSection', 'rootContainer')
jlkUserName = config.get('RootContainerSection', 'jlkUserName')
jlkUserPassword = config.get('RootContainerSection', 'jlkUserPassword')
admPort = config.get('RootContainerSection', rootContainer + '_adm_port')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
result = sock.connect_ex((serverAddress, int(admPort)))
if result == 0:
try:
j4p = Jolokia('http://' + serverAddress + ':' + admPort +
'/jolokia/')
j4p.auth(httpusername=decode(jlkUserName),
httppassword=decode(jlkUserPassword))
strClusterStatus = j4p.request(type='read',
mbean='io.fabric8:service=Health',
attribute='CurrentStatus')
if strClusterStatus['value'] == 'Good':
clusterStatus = 0
else:
clusterStatus = 1
except Exception:
clusterStatus = 1
else:
clusterStatus = 1
print rootContainer + ' ' + str(clusterStatus)
def activeMQValidate():
serverAddress = config.get('RootContainerSection', 'serverAddress')
appContainers = config.get('AppContainerSection', 'appContainers')
jlkUserName = config.get('RootContainerSection', 'jlkUserName')
jlkUserPassword = config.get('RootContainerSection', 'jlkUserPassword')
arrayContainersInstances = []
arrayContainersInstances.extend(appContainers.split(','))
for contSplit in arrayContainersInstances:
admPort = config.get('AppContainerSection', contSplit + '_adm_port')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
result = sock.connect_ex((serverAddress, int(admPort)))
if result == 0:
try:
j4p = Jolokia('http://' + serverAddress + ':' + admPort +
'/jolokia/')
j4p.auth(httpusername=decode(jlkUserName),
httppassword=decode(jlkUserPassword))
reqActiveMQStatus = j4p.request(type='read', \
mbean='org.apache.activemq:brokerName=' + contSplit + ',service=Health,type=Broker', \
attribute='CurrentStatus')
strActiveMQStatus = reqActiveMQStatus['value']
if strActiveMQStatus == 'Good':
activeMQStatus = 0
else:
activeMQStatus = 1
except Exception:
activeMQStatus = 1
else:
activeMQStatus = 1
print contSplit + ' ' + str(activeMQStatus)
def zookeeperValidate():
serverAddress = config.get('RootContainerSection', 'serverAddress')
rootContainer = config.get('RootContainerSection', 'rootContainer')
jlkUserName = config.get('RootContainerSection', 'jlkUserName')
jlkUserPassword = config.get('RootContainerSection', 'jlkUserPassword')
admPort = config.get('RootContainerSection', rootContainer + '_adm_port')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
result = sock.connect_ex((serverAddress, int(admPort)))
if result == 0:
try:
j4p = Jolokia('http://' + serverAddress + ':' + admPort +
'/jolokia/')
j4p.auth(httpusername=decode(jlkUserName),
httppassword=decode(jlkUserPassword))
reqZookeeperStatus = j4p.request(
type='read',
mbean='org.apache.zookeeper:name0=*,name1=*',
attribute='State')
strZookeeperStatus = reqZookeeperStatus['value'].items(
)[0][1]['State']
if strZookeeperStatus == 'following' or strZookeeperStatus == 'leading':
zookeeperStatus = 0
else:
zookeeperStatus = 1
except Exception:
zookeeperStatus = 1
else:
zookeeperStatus = 1
print rootContainer + ' ' + str(zookeeperStatus)
def __init__(self, url, user, pwd, beans=['*.*']):
self.j4p = Jolokia(url)
self.j4p.auth(httpusername=user, httppassword=pwd)
self.data = {}
for bean in beans:
d = JolokiaData(self.j4p, bean).get_data()
self.data.update(d)
def init_jolokia():
global JOLOKIA_CONNECTIONS
for connection in JOLOKIA_CONNECTIONS.keys():
j4p = Jolokia(connection)
for bean in JOLOKIA_CONNECTIONS[connection]['mbeans']:
j4p.add_request(type = 'read', mbean=bean)
JOLOKIA_CONNECTIONS[connection]['j4p'] = j4p
def init_jolokia():
global JOLOKIA_CONNECTIONS
for connection in JOLOKIA_CONNECTIONS.keys():
j4p = Jolokia(connection)
for bean in JOLOKIA_CONNECTIONS[connection]['mbeans']:
j4p.add_request(type='read', mbean=bean)
JOLOKIA_CONNECTIONS[connection]['j4p'] = j4p
def clear_dil_cache(url):
"""Performs removeALL() operation in the DIL cache Mbean - (clears eh cache)
Test Suite variable ${mbean_cache} is used to access mbean attribute type "cache"
:param url: Jolokia Agent URL installed in the web-server under test.
Example : http://rome.lhs-systems.com:29001/jolokia/
"""
j_url = Jolokia(url)
j_response = j_url.request(
type='exec',
mbean=builtin.get_variable_value("${mbean_cache}"),
operation='removeAll()')
if j_response.get('status') != 200:
raise RuntimeError(
"Jolokia response status is not 200, please check response!!")
return json.dumps(j_response, indent=4)
def get_on_disk_miss(url):
"""Returns on disk miss attribute value from the DIL cache Mbean
Test Suite variable ${mbean_cache_stats} is used to access mbean attribute type "CacheStatistics"
:param url: Jolokia Agent URL installed in the web-server under test.
Example : http://rome.lhs-systems.com:29001/jolokia/
"""
j_url = Jolokia(url)
j_response = j_url.request(
type='read',
mbean=builtin.get_variable_value("${mbean_cache_stats}"),
attribute='OnDiskMisses')
if j_response.get('status') != 200:
raise RuntimeError(
"Jolokia response Status is not 200, please check response!!")
print json.dumps(j_response, indent=4)
return j_response.get('value')
def jolokia_request(self,
url,
mbean="java.lang:type=Memory",
attribute="HeapMemoryUsage"):
""" |Usage|
To get memory usage using Jolokia request.
|Argument|
'url' : Jolokia url
'mbean' : By default, java.lang:type=Memory
'attribute' : By default,HeapMemoryUsage
"""
j4p = Jolokia(url)
data = j4p.request(type='read', mbean=mbean, attribute=attribute)
logger.debug("Response {}".format(data['value']))
return data['value']
def servicesCXFValidation():
serverAddress = config.get('RootContainerSection', 'serverAddress')
appContainers = config.get('AppContainerSection', 'appContainers')
jlkUserName = config.get('RootContainerSection', 'jlkUserName')
jlkUserPassword = config.get('RootContainerSection', 'jlkUserPassword')
arrayContainersInstances = []
arrayContainersInstances.extend(appContainers.split(','))
for contSplit in arrayContainersInstances:
admPort = config.get('AppContainerSection', contSplit + '_adm_port')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
result = sock.connect_ex((serverAddress, int(admPort)))
if result == 0:
try:
j4p = Jolokia('http://' + serverAddress + ':' + admPort +
'/jolokia/')
j4p.auth(httpusername=decode(jlkUserName),
httppassword=decode(jlkUserPassword))
reqApacheCXFStatus = j4p.request(
type='read',
mbean=
'io.fabric8.cxf:*,instance.id=*,port=*,service=*,type=Bus.Service.Endpoint',
attribute='State')
serviceCXFStatus = 0
for apacheCXF in reqApacheCXFStatus['value'].items():
splitApacheCXF = apacheCXF[0].split(',')
apacheCXFState = apacheCXF[1]
splitApacheCXFApp = str(splitApacheCXF[2]).split('=')
apacheCXFApp = splitApacheCXFApp[1].replace('"', '')
statusCXF = str(apacheCXFApp) + '=' + str(
apacheCXFState['State'])
if ("STARTED" not in statusCXF):
serviceCXFStatus = serviceCXFStatus + 1
except Exception:
serviceCXFStatus = serviceCXFStatus + 1
print contSplit + ' ' + str(serviceCXFStatus)
def _set_monitors(self, monitors, tags):
self.j4p = Jolokia(self.url)
self.j4p.auth(httpusername=self.auth['username'], httppassword=self.auth['password'])
self.j4p.config(ignoreErrors=True)
self.monitors = copy.deepcopy(monitors)
for m in self.monitors:
if 'tags' in m:
m['tags'].update(tags)
else:
m['tags'] = tags
if 'tags' in m and len(m['tags']) > 0:
m['taglist'] = ["%s=%s" % (k, v) for k, v in m['tags'].items()]
else:
m['taglist'] = []
# set a default not_tags to '' to catch empty tag keys and append
# the value to metric name instead
m['not_tags'] += ['']
self.j4p.add_request(type='read', mbean=m['mbean'])
def __init__(self,
proxyhost='localhost',
proxyport=8888,
timeout=5,
debug=False):
''''''
self.proxyhost = proxyhost
self.proxyport = proxyport
self.debug = debug
self.proxyaddress = '%s:%s' % (self.proxyhost, self.proxyport)
self.proxy = Jolokia('http://%s/jolokia/' % self.proxyaddress)
self.proxy.timeout = timeout
self.order = ['RMI', 'REMOTING-JMX', 'JMXMP']
# these outputs indicate JMX available, but bad credentials
self.jmxReplies = [
#'Connection refused',
'Invalid credentials',
'Authentication failed',
'SecurityException',
'does not have administration access',
'com.sun.enterprise.security.LoginException',
]
self.jmx = None
def main():
global j4p
(options, args) = configure_options()
if len(args) != 2:
parser.print_help()
sys.exit(UNKNOWN)
if options.reset:
release_lock()
jolokia_endpoint = args[0]
bundle_name = args[1]
jolokia_username = options.user
jolokia_password = options.password
j4p = Jolokia(jolokia_endpoint)
j4p.auth(httpusername=jolokia_username, httppassword=jolokia_password)
if not dictionary_exists():
create_dictionary()
state = check_state(bundle_name)
if state == "ACTIVE":
print "OK - %s - %s" % (bundle_name, state)
sys.exit(OK)
elif state == "RESOLVED":
print "CRITICAL - %s - %s" % (bundle_name, state)
sys.exit(CRITICAL)
elif state == None:
print "UNKNOWN - Bundle %s not recognized" % bundle_name
sys.exit(UNKNOWN)
else:
print "WARNING - %s - %s" % (bundle_name, state)
sys.exit(WARNING)
def main():
global j4p
(options, args) = configure_options()
if len(args) != 2:
parser.print_help()
sys.exit(UNKNOWN)
if options.reset:
release_lock()
jolokia_endpoint = args[0]
bundle_name = args[1]
jolokia_username = options.user
jolokia_password = options.password
j4p = Jolokia(jolokia_endpoint)
j4p.auth(httpusername=jolokia_username, httppassword=jolokia_password)
if not dictionary_exists():
create_dictionary()
state = check_state(bundle_name)
if state == "ACTIVE":
print "OK - %s - %s" % (bundle_name, state)
sys.exit(OK)
elif state == "RESOLVED":
print "CRITICAL - %s - %s" % (bundle_name, state)
sys.exit(CRITICAL)
elif state == None:
print "UNKNOWN - Bundle %s not recognized" % bundle_name
sys.exit(UNKNOWN)
else:
print "WARNING - %s - %s" % (bundle_name, state)
sys.exit(WARNING)
def apacheCamelValidate():
serverAddress = config.get('RootContainerSection', 'serverAddress')
appContainers = config.get('AppContainerSection', 'appContainers')
jlkUserName = config.get('RootContainerSection', 'jlkUserName')
jlkUserPassword = config.get('RootContainerSection', 'jlkUserPassword')
arrayContainersInstances = []
arrayContainersInstances.extend(appContainers.split(','))
for contSplit in arrayContainersInstances:
admPort = config.get('AppContainerSection', contSplit + '_adm_port')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
result = sock.connect_ex((serverAddress, int(admPort)))
if result == 0:
try:
j4p = Jolokia('http://' + serverAddress + ':' + admPort +
'/jolokia/')
j4p.auth(httpusername=decode(jlkUserName),
httppassword=decode(jlkUserPassword))
reqApacheCamelStatus = j4p.request(
type='read',
mbean='org.apache.camel:context=*,type=context,name=*',
attribute='State')
strApacheCamelStatus = reqApacheCamelStatus['value'].items(
)[0][1]['State']
if strApacheCamelStatus == 'Started':
apacheCamelStatus = 0
else:
apacheCamelStatus = 1
except Exception:
apacheCamelStatus = 1
else:
apacheCamelStatus = 1
print contSplit + ' ' + str(apacheCamelStatus)
def collect_common_data():
result = []
j4p = Jolokia(solr_config.jolokia_url)
#j4p.add_request(type='read', mbean='solr:id=NativeTrackerStats,type=NativeTrackerStats')
#data = j4p.getRequests()
#print(json.dumps(data, indent=4, separators=(',', ': ')))
#exit(1)
data = j4p.request(
type='read',
mbean='solr:id=NativeAllocatorStats,type=NativeAllocatorStats')
result.append(SolrKeyValue('numAlloc', data['value']['numAlloc'],
'derive'))
result.append(SolrKeyValue('numFree', data['value']['numFree'], 'derive'))
result.append(
SolrKeyValue('totalFreedMemoryInBytes',
data['value']['totalFreedMemoryInBytes'], 'derive'))
result.append(
SolrKeyValue('activeAllocatedMemoryInBytes',
data['value']['activeAllocatedMemoryInBytes']))
result.append(
SolrKeyValue('totalAllocatedMemoryInBytes',
data['value']['totalAllocatedMemoryInBytes'], 'derive'))
data = j4p.request(
type='read',
mbean='solr:id=NativeTrackerStats,type=NativeTrackerStats')
result.append(
SolrKeyValue('registeredThreads', data['value']['registeredThreads']))
result.append(
SolrKeyValue('trackedObjects', data['value']['trackedObjects']))
result.append(
SolrKeyValue('handedOffObjects', data['value']['handedOffObjects']))
return result
def run(self):
j4p = Jolokia('http://localhost:8161/api/jolokia/')
j4p.auth(httpusername='******', httppassword='******')
mbean = 'org.apache.activemq:type=Broker,brokerName=localhost,destinationType=Queue,destinationName=%s'
# queueOne
q1 = 'queueOne'
j4p.add_request(type='read', mbean=(mbean % q1), attribute='AverageEnqueueTime')
# queueTwo
q2 = 'queueTwo'
j4p.add_request(type='read', mbean=(mbean % q2), attribute='AverageEnqueueTime')
bulk_data = j4p.getRequests()
queue_one = next((x for x in bulk_data if q1 in x['request']['mbean']), None)
queue_two = next((x for x in bulk_data if q2 in x['request']['mbean']), None)
data = {q1: queue_one['value'], q2: queue_two['value']}
return data
def __init__(self, proxyhost='localhost', proxyport=8888, timeout=5, debug=False):
''''''
self.proxyhost = proxyhost
self.proxyport = proxyport
self.debug = debug
self.proxyaddress = '%s:%s' % (self.proxyhost, self.proxyport)
self.proxy = Jolokia('http://%s/jolokia/' % self.proxyaddress)
self.proxy.timeout = timeout
self.order = ['RMI','REMOTING-JMX','JMXMP']
# these outputs indicate JMX available, but bad credentials
self.jmxReplies = [
#'Connection refused',
'Invalid credentials',
'Authentication failed',
'SecurityException',
'does not have administration access',
'com.sun.enterprise.security.LoginException',
]
self.jmx = None
def _set_monitors(self, monitors, tags):
self.j4p = Jolokia(self.url)
self.j4p.auth(httpusername=self.auth['username'], httppassword=self.auth['password'])
self.j4p.config(ignoreErrors=True)
self.monitors = copy.deepcopy(monitors)
for m in self.monitors:
if 'tags' in m:
m['tags'].update(tags)
else:
m['tags'] = tags
if 'tags' in m and len(m['tags']) > 0:
m['taglist'] = ["%s=%s" % (k, v) for k, v in m['tags'].items()]
else:
m['taglist'] = []
# set a default not_tags to '' to catch empty tag keys and append
# the value to metric name instead
m['not_tags'] += ['']
self.j4p.add_request(type='read', mbean=m['mbean'])
class JolokiaProxyHandler(object):
""" Handler for jmxterm session """
def __init__(self,
proxyhost='localhost',
proxyport=8888,
timeout=5,
debug=False):
''''''
self.proxyhost = proxyhost
self.proxyport = proxyport
self.debug = debug
self.proxyaddress = '%s:%s' % (self.proxyhost, self.proxyport)
self.proxy = Jolokia('http://%s/jolokia/' % self.proxyaddress)
self.proxy.timeout = timeout
self.order = ['RMI', 'REMOTING-JMX', 'JMXMP']
# these outputs indicate JMX available, but bad credentials
self.jmxReplies = [
#'Connection refused',
'Invalid credentials',
'Authentication failed',
'SecurityException',
'does not have administration access',
'com.sun.enterprise.security.LoginException',
]
self.jmx = None
def reset(self):
''' reset the proxy parameters '''
self.proxy.proxy(url='', user='', password='')
self.jmx = None
def connectEval(self,
host=None,
port=None,
user=None,
password=None,
protocol=None):
'''poll the connected port for info'''
# establish the connection
self.connect(host, port, user, password, protocol)
# only return JMX port info
if self.jmx.protocol is not None:
# don't bother trying to get other info if we can't connect
if self.jmx.connected is True: self.getJVMDetails()
return self.jmx
return None
def connect(self,
host=None,
port=None,
user=None,
password=None,
protocol=None):
''' connect to jmx via proxy'''
self.reset()
self.jmx = JMXConnect(host, port, user, password, protocol)
self._connect(protocol)
def _connect(self, protocol=None):
'''connect with or without known protocol'''
if protocol is not None:
self.jmx.setURL(protocol)
self.proxy.proxy(url=self.jmx.url,
user=self.jmx.user,
password=self.jmx.password)
self.jmx.connected = self.isConnected()
self.jmx.eval()
else:
protocol = self.findProtocol()
# if found, retry with the known protocol
if protocol is not None:
self._connect(protocol)
# otherwise we couldn't find it so reset everything
else:
#self.reset()
self.jmx.url = ''
self.jmx.connected = False
self.jmx.protocol = None
self.proxy.proxy(url='', user='', password='')
def findProtocol(self):
'''try to determine what protocol a remote host uses'''
for protocol in self.order:
if self.usesProtocol(protocol) is True:
self.jmx.protocol = protocol
return protocol
return None
def usesProtocol(self, protocol=None):
'''decide if the remote jmx agent supports the given protocol'''
#print 'testing protocol: %s' % protocol
self._connect(protocol)
self.testConnection()
if self.jmx.result is not None and 'status' in self.jmx.result.keys():
if self.jmx.result['status'] == 200: return True
else:
if 'stacktrace' in self.jmx.result.keys():
for line in self.jmx.result['stacktrace'].split('\n'):
# any of these implies correct protocol even if wrong parameters were
for reply in self.jmxReplies:
if reply in str(line): return True
return False
def isConnected(self):
'''determine whether remote host is available'''
if self.jmx.result is None: self.testConnection()
return self.jmx.status()
def testConnection(self):
'''Test JMX connection by looking for generic mbean'''
self.jmx.result = None
try:
self.jmx.result = self.proxy.request(type='search',
mbean='java.lang:*')
except:
pass
def checkConnect(self,
host=None,
port=None,
user=None,
password=None,
protocol=None):
'''decide whether to start new connection'''
new = False
# completely new connection
if self.jmx is None: new = True
else:
if self.jmx.connected is False: new = True
else:
if self.jmx.host != host: new = True
if self.jmx.port != port: new = True
if new is True: self.connect(host, port, user, password, protocol)
def setJMX(self, jmx):
'''reuse a given JMXConnect object'''
self.jmx = jmx
self._connect(self.jmx.protocol)
def getJVMDetails(self):
''''''
self.jmx.javaversion = self.getJavaVersion()
self.jmx.vendorname = self.getJVMVendorName()
self.jmx.vendorproduct = self.getJVMVendorProduct(self.jmx.vendorname)
def getJavaVersion(self):
'''determine what version of Java is running'''
result = self.getBeanAttributeValues(
'java.lang:type=Runtime', ['VmName', 'VmVendor', 'VmVersion'])
if len(result.values()) > 0: return str(' '.join(result.values()))
else: return 'Unknown'
def getJVMVendorName(self):
'''determine who is the JVM vendor'''
vendor = "Unknown"
for v in VENDORSEARCH.keys():
# skip identifying as Sun unless there's no alternative
if v == 'Sun': continue
if self.beanExists(VENDORSEARCH[v]['search']) is True: vendor = v
if vendor == 'Unknown' and self.beanExists(
VENDORSEARCH['Sun']['search']) is True:
vendor = 'Sun'
return vendor
def getJVMVendorProduct(self, vendor='Unknown'):
'''determine what version of Java is running'''
product = "Java"
if vendor in VENDORSEARCH.keys():
for data in VENDORSEARCH[vendor]['jvmvendor']:
result = self.getBeanAttributeValues(data['bean'],
data['attributes'])
product = VENDORSEARCH[vendor]['name']
# try to get more details about the
if len(result.values()) > 0:
version = str(' '.join(result.values()))
if product not in version:
product = '%s %s' % (product, version)
else:
product = version
# return the generic name if we didn't find anything
return product
def beanExists(self, mbean=None):
'''return True if mbean exists'''
searchresult = self.proxy.request(type='search', mbean=mbean)
if 'value' in searchresult.keys() and len(searchresult['value']) > 0:
return True
return False
def getBeanAttributeValues(self, mbean=None, attributes=[]):
'''return list of attribute values for a given mbean'''
result = {}
for attr in attributes:
output = self.proxy.request(type='read',
mbean=mbean,
attribute=attr)
if 'value' in output.keys(): result[attr] = output['value']
return result
def getAttribute(self, mbean, attribute):
'''return single-valued mbean attribute'''
try:
return self.getBeanAttributeValues(mbean, [attribute])[attribute]
except:
return None
def search(self, mbean):
'''return parsed list of normalized mbean paths matching search'''
#log.debug("getting %s MBeans for %s" % (mbean, port))
output = []
result = self.get(mbean)
for r in result:
data = {
'fullpath': str(r['fullname']),
'mbean': '',
} #, 'enabled': True}
if 'name' in r.keys(): data['mbean'] = str(r['name'])
else: data['mbean'] = str(r['fullname'])
for k, v in r.items():
k = str(k)
if k in ['fullname', 'name']: continue
data[k] = str(v)
output.append(data)
return output
def get(self, mbean=None, attribute=None, task='search'):
'''perform task with mbean and attribute'''
try:
if attribute is not None:
result = self.proxy.request(type=task,
mbean=mbean,
attribute=attribute)
else:
result = self.proxy.request(type=task, mbean=mbean)
return self.parseResult(result)
except:
return []
def parseResult(self, result):
'''return a list of parsed result data dictionaries'''
output = []
if 'value' in result.keys():
if type(result['value']) is list:
for bean in result['value']:
output.append(self.parseData(bean))
return output
def parseData(self, data):
'''return dictionary of output'''
info = {'fullname': data}
start = data.find(':') + len(':')
data = data[start:]
keyvals = data.split(',')
for kv in keyvals:
k, v = kv.split('=')
info[k] = v
return info
def parseDictToList(self, result):
'''return list of dictionaries describing attributes of MBean path'''
output = []
if type(result) == dict:
for k, v in result.items():
info = self.parseData(k)
info['value'] = v
output.append(info)
return output
sys.stdout.write("\n")
if len(newattributes) > 0:
f.write(bean['mbean'] + ";" + ";".join(newattributes) + "\n")
f.close()
def JolokiaQueryList():
'''
This function queries a jolokia for the list of all available beans.
'''
# Enter the jolokia url
JolokiaURL = '%s/%s' % (
config['Weblogic']['AdminURL'],
config['Weblogic']['JolokiaPath']
)
#print(JolokiaURL)
j4p = Jolokia(JolokiaURL)
j4p.auth(httpusername=config['Weblogic']['User'],
httppassword=config['Weblogic']['Password'])
# Put in the type, the mbean, or other options. Check the jolokia users guide for more info
# This then will return back a python dictionary of what happend to the request
#data = j4p.request(type = 'read', mbean='java.lang:type=Threading', attribute='ThreadCount')
data = j4p.request(type = 'list', path='')
return data
def JolokiaParseList(data):
'''
This function parses the jolokia list-document and applies black/whitelists.
'''
TheValues = data['value']
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
author:Clay
Date:'15-11-20'
Description:Used to get jvm
"""
from __future__ import division
from pyjolokia import Jolokia
import time
import socket
import requests
import json
j4p = Jolokia('http://10.8.15.28:8778/jolokia/')
#------------------add request----------------------------
j4p.add_request(type='read',
mbean='java.lang:type=Threading',
attribute='ThreadCount')
j4p.add_request(type='read', mbean='java.lang:type=Memory')
hulb = j4p.getRequests()
threadcount = hulb[0]['value']
memhulb = hulb[1]['value']
#------------------------
#堆最大值
heapmem = memhulb['HeapMemoryUsage']
#堆当前分配值
Heapcommit = heapmem['committed']
def get_jolokia_inst(port):
"""Returns instance of jolokia"""
jolokia_url = "http://127.0.0.1:%s/jolokia/" % port
jolokiaclient = Jolokia(jolokia_url)
return jolokiaclient
class JolokiaCollector():
"""
Create an instance of JolokiaCollector for each instance of Jolokia you
will connect to.
"""
url = ''
monitors = []
j4p = None
def __init__(self, url, auth, tags, monitors):
"""
url - HTTP location of Jolokia
auth - Dict of username and password
tags - Dict of key value pairs
monitors - List of Dicts.
- mbean - String, eg 'java.lang:type=*'
- metric - String, prefix to generated metric name, eg 'java.lang'
- tags - (optional) Same as above, but specific for this mbean only
- not_tags - (optional) List of autogenerated tags to remove, eg ['type']
"""
self.url = url
if 'username' not in auth:
auth['username'] = ''
if 'password' not in auth:
auth['password'] = ''
self.auth = auth
self._set_monitors(monitors, tags)
def _set_monitors(self, monitors, tags):
self.j4p = Jolokia(self.url)
self.j4p.auth(httpusername=self.auth['username'], httppassword=self.auth['password'])
self.j4p.config(ignoreErrors=True)
self.monitors = copy.deepcopy(monitors)
for m in self.monitors:
if 'tags' in m:
m['tags'].update(tags)
else:
m['tags'] = tags
if 'tags' in m and len(m['tags']) > 0:
m['taglist'] = ["%s=%s" % (k, v) for k, v in m['tags'].items()]
else:
m['taglist'] = []
# set a default not_tags to '' to catch empty tag keys and append
# the value to metric name instead
m['not_tags'] += ['']
self.j4p.add_request(type='read', mbean=m['mbean'])
def print_metrics(self, d, metric_prefix, timestamp, tags, not_tags=[]):
""" Take a dict of attributes and print out numerical metric strings
Recurse if necessary
"""
for k, v in d.iteritems():
# Tack on the name of the attribute
attribute, more_tags = self.parse_attribute(k.lower(), not_tags)
metric_name = '.'.join([metric_prefix, attribute])
my_tags = tags + more_tags
# If numerical
if utils.is_numeric(v):
print "%s %d %s %s" % (metric_name, timestamp, str(v),
' '.join(my_tags))
# If a bool, True=1, False=0
elif type(v) is bool:
print "%s %d %s %s" % (metric_name, timestamp, str(int(v)),
' '.join(my_tags))
# Or a dict of more attributes, call ourselves again
elif type(v) is dict:
self.print_metrics(v, metric_name, timestamp, my_tags, not_tags)
else:
#lists, strings, etc
#print '# ', type(v), metric_name, str(v)
pass
def process_data(self):
""" Make request to Jolokia, make sure we have valid data, print out
the metrics for each mbean.
"""
data = []
try:
data = self.j4p.getRequests()
except JolokiaError:
utils.err('error: issue connecting to Jolokia ' + self.url)
if len(data) >= 1:
for mbean in data:
if 'error' in mbean:
utils.err("error: " + mbean['error'])
for monitor in self.monitors:
if monitor['mbean'] == mbean['request']['mbean']:
if mbean['status'] == 200:
self.print_metrics(mbean['value'], monitor['metric'], mbean['timestamp'],
monitor['taglist'], monitor['not_tags'])
break
else:
utils.err("error: mbean not found - " + monitor['mbean'])
def parse_attribute(self, attr, not_tags=[]):
""" Parse and order attribute text
eg from:
org.apache.cassandra.metrics:name=CurrentlyBlockedTasks,path=request,
scope=RequestResponseStage,type=ThreadPools
to: cassandra.metrics.threadpools.currentlyblockedtasks.count,
[path=request, scope=requestresponsestage]
"""
pruned = {}
parts = attr.split(',')
for p in parts:
# Take anything to the right of a =
tag_name, _, attrname = p.rpartition('=')
tag_name = tag_name.split(':')[-1]
# Swap out bad chars
attrname = attrname.replace('.', '_').replace('/', '').replace(' ', '_')
pruned[tag_name] = attrname
attr_list = []
for t in not_tags:
if t in pruned:
attr_list.append(pruned[t])
return '.'.join(attr_list), ["%s=%s" % (k, v) for k, v in pruned.items()
if k not in not_tags]
Port = Ports['PublicPort']
con_list.append({'{#NAME}': Name+"_java"})
con_dict = {}
con_dict['data'] = con_list
print(json.dumps(con_dict))
else:
for container in containerslist:
# Anda pelos containers, se nao for twemproxy (porque nao tem porta publica) pega a porta para conectar local do container/jolokia
Name = container['Names']
Ports = container['Ports'][0]
Name = re.sub('/|_[0-9].*','', Name[0])
Name = hostname+'_'+Name
if not Name in [hostname+'_twemproxy']:
Port = Ports['PublicPort']
j4p = Jolokia('http://localhost:'+ str(Port) + '/jolokia/')
j4p.add_request(type = 'read', mbean='java.lang:type=Memory')
j4p.add_request(type = 'read', mbean='java.lang:type=Threading', attribute='ThreadCount')
try:
bulkdata = j4p.getRequests()
# print bulkdata
# preparing key name
key_NonHeapMemoryUsageMax = 'user.docker.java[NonHeapMemoryUsageMax]'
key_NonHeapMemoryCommitted = 'user.docker.java[NonHeapMemoryCommitted]'
key_NonHeapMemoryInit = 'user.docker.java[NonHeapMemoryInit]'
key_NonHeapMemoryUsed = 'user.docker.java[NonHeapMemoryUsed]'
key_HeapMemoryUsageMax = 'user.docker.java[HeapMemoryUsageMax]'
mbean = MBean(object_name=object_name, domain=domain, description=json['desc'])
if 'attr' in json:
for attribute_name in json['attr'].keys():
mbean.add_attribute(create_attribute(attribute_name, json['attr'][attribute_name]))
return mbean
if __name__ == '__main__':
url_string = 'http://127.0.0.1:8161/api/jolokia'
url = urlparse(url_string)
j = Jolokia(url.geturl())
j.auth(httpusername='******', httppassword='******')
request = j.request(type='list')
domains = request['value'].keys()
# Multimap
mbeans_by_domain = defaultdict(list)
mbeans = []
for domain in domains:
object_names = request['value'][domain].keys()
for object_name in object_names:
mbean = create_mbean(object_name, domain, request['value'][domain][object_name])
mbeans.append(mbean)
class JolokiaCollector():
"""
Create an instance of JolokiaCollector for each instance of Jolokia you
will connect to.
"""
url = ''
monitors = []
j4p = None
def __init__(self, url, auth, tags, monitors):
"""
url - HTTP location of Jolokia
auth - Dict of username and password
tags - Dict of key value pairs
monitors - List of Dicts.
- mbean - String, eg 'java.lang:type=*'
- metric - String, prefix to generated metric name, eg 'java.lang'
- tags - (optional) Same as above, but specific for this mbean only
- not_tags - (optional) List of autogenerated tags to remove, eg ['type']
"""
self.url = url
if 'username' not in auth:
auth['username'] = ''
if 'password' not in auth:
auth['password'] = ''
self.auth = auth
self._set_monitors(monitors, tags)
def _set_monitors(self, monitors, tags):
self.j4p = Jolokia(self.url)
self.j4p.auth(httpusername=self.auth['username'], httppassword=self.auth['password'])
self.j4p.config(ignoreErrors=True)
self.monitors = copy.deepcopy(monitors)
for m in self.monitors:
if 'tags' in m:
m['tags'].update(tags)
else:
m['tags'] = tags
if 'tags' in m and len(m['tags']) > 0:
m['taglist'] = ["%s=%s" % (k, v) for k, v in m['tags'].items()]
else:
m['taglist'] = []
# set a default not_tags to '' to catch empty tag keys and append
# the value to metric name instead
m['not_tags'] += ['']
self.j4p.add_request(type='read', mbean=m['mbean'])
def print_metrics(self, d, metric_prefix, timestamp, tags, not_tags=[]):
""" Take a dict of attributes and print out numerical metric strings
Recurse if necessary
"""
for k, v in d.iteritems():
# Tack on the name of the attribute
attribute, more_tags = self.parse_attribute(k.lower(), not_tags)
metric_name = '.'.join([metric_prefix, attribute])
my_tags = tags + more_tags
# If numerical
if utils.is_numeric(v):
print "%s %d %s %s" % (metric_name, timestamp, str(v),
' '.join(my_tags))
# If a bool, True=1, False=0
elif type(v) is bool:
print "%s %d %s %s" % (metric_name, timestamp, str(int(v)),
' '.join(my_tags))
# Or a dict of more attributes, call ourselves again
elif type(v) is dict:
self.print_metrics(v, metric_name, timestamp, my_tags, not_tags)
else:
#lists, strings, etc
#print '# ', type(v), metric_name, str(v)
pass
def process_data(self):
""" Make request to Jolokia, make sure we have valid data, print out
the metrics for each mbean.
"""
data = []
try:
data = self.j4p.getRequests()
except JolokiaError:
utils.err('error: issue connecting to Jolokia ' + self.url)
if len(data) >= 1:
for mbean in data:
if 'error' in mbean:
utils.err("error: " + mbean['error'])
for monitor in self.monitors:
if monitor['mbean'] == mbean['request']['mbean']:
if mbean['status'] == 200:
self.print_metrics(mbean['value'], monitor['metric'], mbean['timestamp'],
monitor['taglist'], monitor['not_tags'])
break
else:
utils.err("error: mbean not found - " + monitor['mbean'])
def parse_attribute(self, attr, not_tags=[]):
""" Parse and order attribute text
eg from:
org.apache.cassandra.metrics:name=CurrentlyBlockedTasks,path=request,
scope=RequestResponseStage,type=ThreadPools
to: cassandra.metrics.threadpools.currentlyblockedtasks.count,
[path=request, scope=requestresponsestage]
"""
pruned = {}
parts = attr.split(',')
for p in parts:
# Take anything to the right of a =
tag_name, _, attrname = p.rpartition('=')
tag_name = tag_name.split(':')[-1]
# Swap out bad chars
attrname = attrname.replace('.', '_').replace('/', '').replace(' ', '_')
pruned[tag_name] = attrname
attr_list = []
for t in not_tags:
if t in pruned:
attr_list.append(pruned[t])
return '.'.join(attr_list), ["%s=%s" % (k, v) for k, v in pruned.items()
if k not in not_tags]
class JolokiaProxyHandler(object):
""" Handler for jmxterm session """
def __init__(self, proxyhost='localhost', proxyport=8888, timeout=5, debug=False):
''''''
self.proxyhost = proxyhost
self.proxyport = proxyport
self.debug = debug
self.proxyaddress = '%s:%s' % (self.proxyhost, self.proxyport)
self.proxy = Jolokia('http://%s/jolokia/' % self.proxyaddress)
self.proxy.timeout = timeout
self.order = ['RMI','REMOTING-JMX','JMXMP']
# these outputs indicate JMX available, but bad credentials
self.jmxReplies = [
#'Connection refused',
'Invalid credentials',
'Authentication failed',
'SecurityException',
'does not have administration access',
'com.sun.enterprise.security.LoginException',
]
self.jmx = None
def reset(self):
''' reset the proxy parameters '''
self.proxy.proxy(url='', user='', password='')
self.jmx = None
def connectEval(self, host=None, port=None, user=None, password=None, protocol=None):
'''poll the connected port for info'''
# establish the connection
self.connect(host, port, user, password, protocol)
# only return JMX port info
if self.jmx.protocol is not None:
# don't bother trying to get other info if we can't connect
if self.jmx.connected is True: self.getJVMDetails()
return self.jmx
return None
def connect(self, host=None, port=None, user=None, password=None, protocol=None):
''' connect to jmx via proxy'''
self.reset()
self.jmx = JMXConnect(host, port, user, password, protocol)
self._connect(protocol)
def _connect(self, protocol=None):
'''connect with or without known protocol'''
if protocol is not None:
self.jmx.setURL(protocol)
self.proxy.proxy(url=self.jmx.url, user=self.jmx.user, password=self.jmx.password)
self.jmx.connected = self.isConnected()
self.jmx.eval()
else:
protocol = self.findProtocol()
# if found, retry with the known protocol
if protocol is not None: self._connect(protocol)
# otherwise we couldn't find it so reset everything
else:
#self.reset()
self.jmx.url = ''
self.jmx.connected = False
self.jmx.protocol = None
self.proxy.proxy(url='', user='', password='')
def findProtocol(self):
'''try to determine what protocol a remote host uses'''
for protocol in self.order:
if self.usesProtocol(protocol) is True:
self.jmx.protocol = protocol
return protocol
return None
def usesProtocol(self, protocol=None):
'''decide if the remote jmx agent supports the given protocol'''
#print 'testing protocol: %s' % protocol
self._connect(protocol)
self.testConnection()
if self.jmx.result is not None and 'status' in self.jmx.result.keys():
if self.jmx.result['status'] == 200: return True
else:
if 'stacktrace' in self.jmx.result.keys():
for line in self.jmx.result['stacktrace'].split('\n'):
# any of these implies correct protocol even if wrong parameters were
for reply in self.jmxReplies:
if reply in str(line): return True
return False
def isConnected(self):
'''determine whether remote host is available'''
if self.jmx.result is None: self.testConnection()
return self.jmx.status()
def testConnection(self):
'''Test JMX connection by looking for generic mbean'''
self.jmx.result = None
try: self.jmx.result = self.proxy.request(type='search', mbean='java.lang:*')
except: pass
def checkConnect(self, host=None, port=None, user=None, password=None, protocol=None):
'''decide whether to start new connection'''
new = False
# completely new connection
if self.jmx is None: new = True
else:
if self.jmx.connected is False: new = True
else:
if self.jmx.host != host: new = True
if self.jmx.port != port: new = True
if new is True: self.connect(host, port, user, password, protocol)
def setJMX(self, jmx):
'''reuse a given JMXConnect object'''
self.jmx = jmx
self._connect(self.jmx.protocol)
def getJVMDetails(self):
''''''
self.jmx.javaversion = self.getJavaVersion()
self.jmx.vendorname = self.getJVMVendorName()
self.jmx.vendorproduct = self.getJVMVendorProduct(self.jmx.vendorname)
def getJavaVersion(self):
'''determine what version of Java is running'''
result = self.getBeanAttributeValues('java.lang:type=Runtime', ['VmName', 'VmVendor', 'VmVersion'])
if len(result.values()) > 0: return str(' '.join(result.values()))
else: return 'Unknown'
def getJVMVendorName(self):
'''determine who is the JVM vendor'''
vendor = "Unknown"
for v in VENDORSEARCH.keys():
# skip identifying as Sun unless there's no alternative
if v == 'Sun': continue
if self.beanExists(VENDORSEARCH[v]['search']) is True: vendor = v
if vendor == 'Unknown' and self.beanExists(VENDORSEARCH['Sun']['search']) is True: vendor = 'Sun'
return vendor
def getJVMVendorProduct(self, vendor='Unknown'):
'''determine what version of Java is running'''
product = "Java"
if vendor in VENDORSEARCH.keys():
for data in VENDORSEARCH[vendor]['jvmvendor']:
result = self.getBeanAttributeValues(data['bean'], data['attributes'])
product = VENDORSEARCH[vendor]['name']
# try to get more details about the
if len(result.values()) > 0:
version = str(' '.join(result.values()))
if product not in version: product = '%s %s' % (product, version)
else: product = version
# return the generic name if we didn't find anything
return product
def beanExists(self, mbean=None):
'''return True if mbean exists'''
searchresult = self.proxy.request(type='search', mbean=mbean)
if 'value' in searchresult.keys() and len(searchresult['value']) > 0: return True
return False
def getBeanAttributeValues(self, mbean=None, attributes=[]):
'''return list of attribute values for a given mbean'''
result = {}
for attr in attributes:
output = self.proxy.request(type='read', mbean=mbean, attribute=attr)
if 'value' in output.keys(): result[attr] = output['value']
return result
def getAttribute(self, mbean, attribute):
'''return single-valued mbean attribute'''
try: return self.getBeanAttributeValues(mbean, [attribute])[attribute]
except: return None
def search(self, mbean):
'''return parsed list of normalized mbean paths matching search'''
#log.debug("getting %s MBeans for %s" % (mbean, port))
output = []
result = self.get(mbean)
for r in result:
data = {
'fullpath' : str(r['fullname']),
'mbean': '',
} #, 'enabled': True}
if 'name' in r.keys(): data ['mbean'] = str(r['name'])
else: data ['mbean'] = str(r['fullname'])
for k,v in r.items():
k = str(k)
if k in ['fullname', 'name']: continue
data[k] = str(v)
output.append(data)
return output
def get(self, mbean=None, attribute=None, task='search'):
'''perform task with mbean and attribute'''
try:
if attribute is not None: result = self.proxy.request(type=task, mbean=mbean, attribute=attribute)
else: result = self.proxy.request(type=task, mbean=mbean)
return self.parseResult(result)
except: return []
def parseResult(self, result):
'''return a list of parsed result data dictionaries'''
output = []
if 'value' in result.keys():
if type(result['value']) is list:
for bean in result['value']: output.append(self.parseData(bean))
return output
def parseData(self, data):
'''return dictionary of output'''
info = {'fullname' : data}
start = data.find(':')+len(':')
data = data[start:]
keyvals = data.split(',')
for kv in keyvals:
k,v = kv.split('=')
info[k] = v
return info
def parseDictToList(self, result):
'''return list of dictionaries describing attributes of MBean path'''
output = []
if type(result) == dict:
for k, v in result.items():
info = self.parseData(k)
info['value'] = v
output.append(info)
return output
from pyjolokia import Jolokia
import json
JOLOKIA_URL = 'http://localhost:8778/jolokia/'
MBEAN = 'org.apache.activemq:type=Broker,brokerName=localhost,destinationType=Queue,destinationName=myQueue'
j4p = Jolokia(JOLOKIA_URL)
# Single requests
print(json.dumps(j4p.request(type='read', mbean='java.lang:type=Memory', attribute='HeapMemoryUsage'), indent=4))
# myQueue:QueueSize
# Number of messages in the destination which are yet to be consumed. Potentially dispatched but unacknowledged.
myQueueSize = j4p.request(type='read', mbean=MBEAN, attribute='QueueSize')
print(json.dumps(myQueueSize, indent=4))
print('myQueue:QueueSize - {0}'.format(myQueueSize["value"]))
# myQueue:AverageEnqueueTime
# Average time a message has been held this destination.
myQueueQueueTime = j4p.request(type='read', mbean=MBEAN, attribute='AverageEnqueueTime')
print('myQueue:AverageEnqueueTime - {0}'.format(myQueueQueueTime["value"]))
# myQueue:InFlightCount
# Number of messages that have been dispatched to, but not acknowledged by, consumers.
myQueueInFlight = j4p.request(type='read', mbean=MBEAN, attribute='InFlightCount')
print('myQueue:InFlightCount - {0}'.format(myQueueInFlight["value"]))
# Bulk requests
j4p.add_request(type='read', mbean=MBEAN, attribute='QueueSize')
j4p.add_request(type='read', mbean=MBEAN, attribute='AverageEnqueueTime')
j4p.add_request(type='read', mbean=MBEAN, attribute='InFlightCount')
def memoryUsageStatus():
# type: () -> object
serverAddress = config.get('RootContainerSection', 'serverAddress')
rootContainer = config.get('RootContainerSection', 'rootContainer')
appContainers = config.get('AppContainerSection', 'appContainers')
gatewayContainers = config.get('GatewayContainerSection',
'gatewayContainer')
jlkUserName = config.get('RootContainerSection', 'jlkUserName')
jlkUserPassword = config.get('RootContainerSection', 'jlkUserPassword')
memoryTestTotal = config.get('RootContainerSection', 'memoryTestTotal')
arrayContainersInstances = []
arrayContainersInstances.append(rootContainer)
arrayContainersInstances.append(gatewayContainers)
arrayContainersInstances.extend(appContainers.split(','))
for contSplit in arrayContainersInstances:
if 'root' in contSplit:
# Setting adminPort
admPort = config.get('RootContainerSection',
contSplit + '_adm_port')
elif 'SIL' in contSplit:
admPort = config.get('AppContainerSection',
contSplit + '_adm_port')
elif 'gateway' in contSplit:
admPort = config.get('GatewayContainerSection',
contSplit + '_adm_port')
# Checking if adminPort is ok to execute test if not ok the test will be interrupted
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
result = sock.connect_ex((serverAddress, int(admPort)))
# Looping to validate each admin port set in ConfigFile.properties
if result == 0:
# ###############################################
# ## Jolokia URL is based in serverAddress in ConfigFile.properties RootContainerSection
j4p = Jolokia('http://' + serverAddress + ':' + admPort +
'/jolokia/')
# # Using Jolokia Authentication
j4p.auth(httpusername=decode(jlkUserName),
httppassword=decode(jlkUserPassword))
# Request memory attributes
memory = j4p.request(type='read',
mbean='java.lang:type=Memory',
attribute='HeapMemoryUsage')
maxMemory = memory['value']['max']
usedMemory = memory['value']['used']
# Transform memory used in percentage of Max memory (Xmx)
memoryUsedPercentage = (usedMemory * 100) / maxMemory
fmemoControl = '/app/scripts/fuse-stats/control/.' + contSplit + '.txt'
if len((glob.glob(fmemoControl))) == 0:
with open(fmemoControl, 'wb') as csvfile:
csvWriter = csv.writer(csvfile)
csvWriter.writerow([
str(memoryUsedPercentage),
])
with open(fmemoControl, 'rb') as csvfile:
csvReader = csv.reader(csvfile)
calcMemAvg = 0
coutMemTest = 0
for rowMemValue in csvReader:
calcMemAvg = calcMemAvg + int(rowMemValue[0])
coutMemTest = coutMemTest + 1
memAvg = calcMemAvg / coutMemTest
print contSplit + ' ' + str(memAvg)
elif len((glob.glob(fmemoControl))) == 1:
with open(fmemoControl, 'a') as csvfile:
csvWriter = csv.writer(csvfile)
csvWriter.writerow([
str(memoryUsedPercentage),
])
teste = 0
with open(fmemoControl, 'rb') as csvfile:
csvReader = csv.reader(csvfile)
calcMemAvg = 0
countMemTest = 0
for rowMemValue in csvReader:
calcMemAvg = calcMemAvg + int(rowMemValue[0])
countMemTest = countMemTest + 1
if countMemTest >= int(memoryTestTotal):
os.remove(str(fmemoControl))
memAvg = calcMemAvg / coutMemTest
print contSplit + ' ' + str(memAvg)
else:
print contSplit + ' 0'
