def testUpdateServiceMetaFileLocal(self):
keys = ['hwPath', 'serverAddress']
values = manifestutil.readJsonServiceMeta('foo', keys)
assert values.get('hwPath') == values.get('serverAddress') == None
createManifest(self, manifest = 'bar', service = 'foo')
manifestutil.updateServiceMetaFile('foo', {'hwPath':'/env/pool/instance', 'serverAddress':'state.vip'})
assert os.path.exists(manifestutil.serviceMetadataPath('foo'))
values = manifestutil.readJsonServiceMeta('foo', keys)
assert values.get('hwPath') == '/env/pool/instance'
assert values.get('serverAddress') == 'state.vip'
manifestutil.updateServiceMetaFile('foo', {'hwPath':None})
values = manifestutil.readJsonServiceMeta('foo', keys)
assert values.get('hwPath') is None
assert values.get('serverAddress') == 'state.vip'
def testUpdateServiceMetaFileLocal(self):
keys = ['hwPath', 'serverAddress']
values = manifestutil.readJsonServiceMeta('foo', keys)
assert values.get('hwPath') == values.get('serverAddress') == None
createManifest(self, manifest='bar', service='foo')
manifestutil.updateServiceMetaFile('foo', {
'hwPath': '/env/pool/instance',
'serverAddress': 'state.vip'
})
assert os.path.exists(manifestutil.serviceMetadataPath('foo'))
values = manifestutil.readJsonServiceMeta('foo', keys)
assert values.get('hwPath') == '/env/pool/instance'
assert values.get('serverAddress') == 'state.vip'
manifestutil.updateServiceMetaFile('foo', {'hwPath': None})
values = manifestutil.readJsonServiceMeta('foo', keys)
assert values.get('hwPath') is None
assert values.get('serverAddress') == 'state.vip'
def getMetadata(self, service):
""" get .metadata file for a service """
try:
result = manifestutil.readJsonServiceMeta(service)
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error get service metadata (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller = self)
def getConfigOverrides():
"""
load configuration overrides from agent .metadata.json
"""
configsOR = {}
try:
configsMeta = manifestutil.readJsonServiceMeta('agent', ['configs'])
configsOR = configsMeta['configs'] if 'configs' in configsMeta else {}
except Exception as exc:
LOG.error(str(exc))
return configsOR
def getConfigOverrides():
"""
load configuration overrides from agent .metadata.json
"""
configsOR = {}
try:
configsMeta = manifestutil.readJsonServiceMeta('agent', ['configs'])
configsOR = configsMeta['configs'] if 'configs' in configsMeta else {}
except Exception as exc:
LOG.error(str(exc))
return configsOR
def getConfig(self):
""" get config overrides """
LOG.info('get config overrides for agent')
result = {}
try:
result = manifestutil.readJsonServiceMeta('agent', ['configs'])
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when clean agent configs %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
def getConfig(self):
""" get config overrides """
LOG.info('get config overrides for agent')
result = {}
try:
result = manifestutil.readJsonServiceMeta('agent', ['configs'])
return doneResult(request, response, result = result, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when clean agent configs %s - %s' %
(str(excep), traceback.format_exc(2)), controller = self)
def getMetadataValue(self, service, name):
""" get .metadata file for a service as property file,
kvp: name=value,
array: name=v1,v2,v3,
dict: name1.name2=value etc.
"""
try:
result = manifestutil.readJsonServiceMeta(service)
response.content_type = 'text/plain'
return result[name] if name in result else ''
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error get service metadata (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller = self)
def loadConfigOverrides():
""" load config override from .metadata """
try:
configsMeta = manifestutil.readJsonServiceMeta('agent', ['configs'])
CONFIGOR.clear()
if 'configs' in configsMeta:
CONFIGOR.update(configsMeta['configs'])
# apply to pylon configs
pylons_config = pylons.config.current_conf()
if pylons_config and CONFIGOR:
pylons_config.update(copy.deepcopy(CONFIGOR))
pylons.config.pop_process_config()
pylons.config.push_process_config(pylons_config)
except BaseException as exc:
LOG.error(str(exc))
def loadConfigOverrides():
""" load config override from .metadata """
try:
configsMeta = manifestutil.readJsonServiceMeta('agent', ['configs'])
CONFIGOR.clear()
if 'configs' in configsMeta:
CONFIGOR.update(configsMeta['configs'])
# apply to pylon configs
pylons_config = pylons.config.current_conf()
if pylons_config and CONFIGOR:
pylons_config.update(copy.deepcopy(CONFIGOR))
pylons.config.pop_process_config()
pylons.config.push_process_config(pylons_config)
except BaseException as exc:
LOG.error(str(exc))
def getMetadata(self, service):
""" get .metadata file for a service """
try:
result = manifestutil.readJsonServiceMeta(service)
return doneResult(request,
response,
result=result,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error get service metadata (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller=self)
def startAgentGlobals(startThreadMgr=True,
startPackageMgr=True,
startAgentMonitor=True):
"""
Create and start the global objects
"""
# cleanup incomplete manifests
from agent.controllers.service import ServiceController
ServiceController.cleanupServices()
# stop the existing agents
stopAgentGlobals()
appGlobal = pylons.config['pylons.app_globals']
# load config override from agent .metadata.json
configutil.loadPylonConfig(pylons.config)
configutil.loadConfigOverrides()
LOG.info("Complete loading config overrides")
# build in memory security token cache
appGlobal.authztoken = str(uuid.uuid4())
appGlobal.encryptedtokens = {}
agentauth.buildTokenCache(appGlobal.authztoken)
LOG.info("Complete building security token cache")
# agent health
appGlobal.agentHealth = 'True'
appGlobal.agentHealthFactor = None
from datetime import datetime
appGlobal.agentInfo = {
'version_wiri': agenthealth.loadVersion(),
'startup_time': str(datetime.now())
}
appGlobal.osInfo = agenthealth.getOsInfo()
appGlobal.diskOk = True
appGlobal.autoRestartEnabled = asbool(
pylons.config['auto_restart_enabled'])
LOG.info("Agent health initialized")
# start thread mgr
appGlobal.threadMgr = ThreadMgr(
garbageFreq=pylons.config['threadMgr_garbage_freq'],
maxThreadAge=pylons.config['threadMgr_thread_age'])
if startThreadMgr:
appGlobal.threadMgr.start()
LOG.info("Agent thread manager started")
# start package mgr
appGlobal.packageMgr = PackageMgr(
garbageFreq=pylons.config['packageMgr_garbage_freq'],
maxPackageAge=pylons.config['packageMgr_package_age'])
if startPackageMgr:
appGlobal.packageMgr.start()
LOG.info("Agent package manager started")
# now load saved .metadata for previously saved hwPath, serverAddress, and serverAuth
enrollMeta = manifestutil.readJsonServiceMeta('agent', ['hwPath'])
hwPath = enrollMeta.get('hwPath')
if hwPath is not None:
appGlobal.hwPath = hwPath
LOG.info("Agent hwPath loaded")
#start monitor manager
from agent.lib.monitors.monitor import AgentMonitor
appGlobal.agentMonitor = AgentMonitor()
if startAgentMonitor:
appGlobal.agentMonitor.start()
LOG.info("Agent monitor started")
# start graphite udp server
# from agent.lib.monitors.graphited import GraphiteDServer
# from multiprocessing import Pipe
# graphiteEnabled = configutil.getConfigAsBool('graphite_enabled')
# if startAgentMonitor and graphiteEnabled:
# output_p, input_p = Pipe()
# appGlobal.graphited = GraphiteDServer(input_p)
# appGlobal.graphited.start()
# udpMsgProcessThread = threading.Thread(target=GraphiteDServer.read_queue, args=(output_p,))
# udpMsgProcessThread.daemon = True
# udpMsgProcessThread.start()
# LOG.info("Agent graphited started")
# Declare dictionary for storing dynamic controllers
appGlobal.dynacontrollers = dict()
# metrix manager initialization
from agent.lib.monitors.metrix_manager import MetrixManager
from agent.lib.monitors.system_monitor import SystemMonitor
appGlobal.metrixManager = MetrixManager()
appGlobal.systemMonitor = SystemMonitor()
appGlobal.metrixManager.register('OS', appGlobal.systemMonitor.getOSinfo,
1)
appGlobal.metrixManager.register('Node Name',
appGlobal.systemMonitor.getNodeName, 1)
appGlobal.metrixManager.register('Free Memory(KB)',
appGlobal.systemMonitor.getFreeMemory, 2)
appGlobal.metrixManager.register('CPU Usage(%)',
appGlobal.systemMonitor.getCpuUsage, 2)
appGlobal.metrixManager.register('Version', agenthealth.loadVersion, 5)
appGlobal.metrixManager.register('Configuration',
validate_internals.getConfigFile, 6)
appGlobal.metrixManager.register('Configuration Overrides',
configutil.getConfigOverrides, 7)
appGlobal.metrixManager.register(
'hwPath', lambda: appGlobal.hwPath
if hasattr(appGlobal, 'hwPath') else None, 5)
appGlobal.metrixManager.register(AGENT_HEALTH_VI_KEY,
lambda: appGlobal.agentHealth, 1)
appGlobal.metrixManager.register(AGENT_HEALTH_FACTOR_VI_KEY,
lambda: appGlobal.agentHealthFactor, 1)
appGlobal.metrixManager.register(OS_INFO_KEY, lambda: appGlobal.osInfo, 2)
appGlobal.metrixManager.register(AGENT_INFO_KEY,
lambda: appGlobal.agentInfo, 2)
LOG.info("Agent health metrics registered")
# start all agent modules
modulestartthread = threading.Thread(
target=ModuleController.loadModuleOnAgentStartup)
modulestartthread.start()
LOG.info("Local modules started")
# start all services with active manifest, and load dynamic controllers
servicestartthread = threading.Thread(
target=ServiceController.startServicesOnAgentStartup)
servicestartthread.start()
LOG.info("Local services started")
appGlobal.sdutil = shutdownAgent
def _addMonitors(self, service, activeManifest):
''' add monitors for a service's activeManifest '''
monitorTasks = []
self.__monitorTasks[(service, activeManifest)] = monitorTasks
# metric tags
metricTags = {}
# add default metric tags
metricTags['fqdn'] = utils.fqdn
# read tags from agent
agtMetricTags = manifestutil.readJsonServiceMeta('agent', [METRIC_TAGS_KEY])
if agtMetricTags and METRIC_TAGS_KEY in agtMetricTags:
metricTags.update(agtMetricTags[METRIC_TAGS_KEY])
# read tags from service
if (service != 'agent'):
serviceMetricTags = manifestutil.readJsonServiceMeta(service, [METRIC_TAGS_KEY])
if serviceMetricTags or METRIC_TAGS_KEY in serviceMetricTags:
metricTags.update(serviceMetricTags[METRIC_TAGS_KEY])
# all packages in this manifest
packages = ManifestController.getPackages(service, os.path.basename(activeManifest))
# metric tags for externally passed in metrics
# monitorTags = {additionaltags_dict, datatype_dict, ctx(env.pool.host), appType(Monitor|custom), monitorType(Cronus|CronusApplication)}
self.__monitorTags[service] = {'default': (metricTags, {}, {})}
self.__monitorValues[(service, 'default')] = {}
self.__monitorMessages[(service, 'default')] = {}
# a unique index for all monitors in the service
for package in packages:
# load monitor setting from cronus.ini
pkgConfigs = manifestutil.getPackageInitConfig(service, activeManifest, package)
if pkgConfigs is not None and 'monitors' in pkgConfigs:
metricCtx = pkgConfigs[METRIC_CONTEXT] if METRIC_CONTEXT in pkgConfigs else {}
metricTypes = {}
if METRIC_TYPE_KEY in pkgConfigs and pkgConfigs[METRIC_TYPE_KEY]:
metricTypes = pkgConfigs[METRIC_TYPE_KEY]
if METRIC_TAGS_KEY in pkgConfigs and pkgConfigs[METRIC_TAGS_KEY]:
metricTags.update(pkgConfigs[METRIC_TAGS_KEY])
monitors = pkgConfigs['monitors']
for monitor in monitors:
try:
m_name = str(monitor['name'])
# metric tags for externally passed in metrics
# monitorTags = {additionaltags_dict, datatype_dict, ctx(env.pool.host), appType(Monitor|custom), monitorType(Cronus|CronusApplication)}
self.__monitorTags[service][m_name] = (metricTags, metricTypes, metricCtx)
# now normalize the interval to one allowable by EVPS (downcast)
if 'reportIntervalSec' not in monitor:
continue
m_interval = float(monitor['reportIntervalSec'])
m_interval = self._normalizeResolution(m_interval)
m_timeout = float(monitor['timeoutSec']) if 'timeoutSec' in monitor else m_interval
m_timeout = max(1, min(m_interval-1, m_timeout))
m_items = monitor['items'] if 'items' in monitor else []
# prime monitor value store
self.__monitorValues[(service, m_name)] = {}
self.__monitorMessages[(service, m_name)] = {}
mi_idx = 0
for m_item in m_items:
if 'type' not in m_item:
continue
mi_idx += 1
mi_type = m_item['type']
mi_cmd = None
if 'script' == mi_type:
mi_script = m_item['script']
mi_cmd = manifestutil.getMonitorScriptPath(service, activeManifest, package, mi_script)
else:
# unknown type, skip
LOG.error('Unknown monitor item type %s, can only be script' % mi_type)
continue
# schedule monitor
m_sch_name = '%s.%s' % (service, m_name)
try:
self.__monitorMessages[(service, m_name)][mi_cmd] = []
task = self.__monitorSch.add_interval_task(self._runMonitor, '%s.%s' % (m_sch_name, str(mi_idx)), 0, float(m_interval),
[mi_type,mi_cmd,service,m_name,m_timeout,(str(int(round(m_interval))),metricTags,metricTypes,metricCtx)],None)
monitorTasks.append(task)
except Exception as excep:
LOG.error('Cannot add monitor task for %s.%s - (%s) %s' % (m_sch_name, str(mi_idx), excep, traceback.format_exc(5)))
except Exception as excep1:
LOG.error('Cannot add report task for %s.%s - (%s) %s' % (service, m_name, excep1, traceback.format_exc(5)))
