def testUpdateServiceMetaFileLocal(self): keys = ['hwPath', 'serverAddress'] values = manifestutil.readJsonServiceMeta('foo', keys) assert values.get('hwPath') == values.get('serverAddress') == None createManifest(self, manifest = 'bar', service = 'foo') manifestutil.updateServiceMetaFile('foo', {'hwPath':'/env/pool/instance', 'serverAddress':'state.vip'}) assert os.path.exists(manifestutil.serviceMetadataPath('foo')) values = manifestutil.readJsonServiceMeta('foo', keys) assert values.get('hwPath') == '/env/pool/instance' assert values.get('serverAddress') == 'state.vip' manifestutil.updateServiceMetaFile('foo', {'hwPath':None}) values = manifestutil.readJsonServiceMeta('foo', keys) assert values.get('hwPath') is None assert values.get('serverAddress') == 'state.vip'
def testUpdateServiceMetaFileLocal(self): keys = ['hwPath', 'serverAddress'] values = manifestutil.readJsonServiceMeta('foo', keys) assert values.get('hwPath') == values.get('serverAddress') == None createManifest(self, manifest='bar', service='foo') manifestutil.updateServiceMetaFile('foo', { 'hwPath': '/env/pool/instance', 'serverAddress': 'state.vip' }) assert os.path.exists(manifestutil.serviceMetadataPath('foo')) values = manifestutil.readJsonServiceMeta('foo', keys) assert values.get('hwPath') == '/env/pool/instance' assert values.get('serverAddress') == 'state.vip' manifestutil.updateServiceMetaFile('foo', {'hwPath': None}) values = manifestutil.readJsonServiceMeta('foo', keys) assert values.get('hwPath') is None assert values.get('serverAddress') == 'state.vip'
def getMetadata(self, service): """ get .metadata file for a service """ try: result = manifestutil.readJsonServiceMeta(service) return doneResult(request, response, result = result, controller = self) except Exception as excep: return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = 'Unknown error get service metadata (%s) %s - %s' % (service, str(excep), traceback.format_exc(2)), controller = self)
def getConfigOverrides(): """ load configuration overrides from agent .metadata.json """ configsOR = {} try: configsMeta = manifestutil.readJsonServiceMeta('agent', ['configs']) configsOR = configsMeta['configs'] if 'configs' in configsMeta else {} except Exception as exc: LOG.error(str(exc)) return configsOR
def getConfigOverrides(): """ load configuration overrides from agent .metadata.json """ configsOR = {} try: configsMeta = manifestutil.readJsonServiceMeta('agent', ['configs']) configsOR = configsMeta['configs'] if 'configs' in configsMeta else {} except Exception as exc: LOG.error(str(exc)) return configsOR
def getConfig(self): """ get config overrides """ LOG.info('get config overrides for agent') result = {} try: result = manifestutil.readJsonServiceMeta('agent', ['configs']) return doneResult(request, response, result = result, controller = self) except Exception as excep: return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = 'Unknown error when clean agent configs %s - %s' % (str(excep), traceback.format_exc(2)), controller = self)
def getConfig(self): """ get config overrides """ LOG.info('get config overrides for agent') result = {} try: result = manifestutil.readJsonServiceMeta('agent', ['configs']) return doneResult(request, response, result = result, controller = self) except Exception as excep: return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = 'Unknown error when clean agent configs %s - %s' % (str(excep), traceback.format_exc(2)), controller = self)
def getMetadataValue(self, service, name): """ get .metadata file for a service as property file, kvp: name=value, array: name=v1,v2,v3, dict: name1.name2=value etc. """ try: result = manifestutil.readJsonServiceMeta(service) response.content_type = 'text/plain' return result[name] if name in result else '' except Exception as excep: return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = 'Unknown error get service metadata (%s) %s - %s' % (service, str(excep), traceback.format_exc(2)), controller = self)
def loadConfigOverrides(): """ load config override from .metadata """ try: configsMeta = manifestutil.readJsonServiceMeta('agent', ['configs']) CONFIGOR.clear() if 'configs' in configsMeta: CONFIGOR.update(configsMeta['configs']) # apply to pylon configs pylons_config = pylons.config.current_conf() if pylons_config and CONFIGOR: pylons_config.update(copy.deepcopy(CONFIGOR)) pylons.config.pop_process_config() pylons.config.push_process_config(pylons_config) except BaseException as exc: LOG.error(str(exc))
def loadConfigOverrides(): """ load config override from .metadata """ try: configsMeta = manifestutil.readJsonServiceMeta('agent', ['configs']) CONFIGOR.clear() if 'configs' in configsMeta: CONFIGOR.update(configsMeta['configs']) # apply to pylon configs pylons_config = pylons.config.current_conf() if pylons_config and CONFIGOR: pylons_config.update(copy.deepcopy(CONFIGOR)) pylons.config.pop_process_config() pylons.config.push_process_config(pylons_config) except BaseException as exc: LOG.error(str(exc))
def getMetadata(self, service): """ get .metadata file for a service """ try: result = manifestutil.readJsonServiceMeta(service) return doneResult(request, response, result=result, controller=self) except Exception as excep: return errorResult( request, response, error=Errors.UNKNOWN_ERROR, errorMsg='Unknown error get service metadata (%s) %s - %s' % (service, str(excep), traceback.format_exc(2)), controller=self)
def startAgentGlobals(startThreadMgr=True, startPackageMgr=True, startAgentMonitor=True): """ Create and start the global objects """ # cleanup incomplete manifests from agent.controllers.service import ServiceController ServiceController.cleanupServices() # stop the existing agents stopAgentGlobals() appGlobal = pylons.config['pylons.app_globals'] # load config override from agent .metadata.json configutil.loadPylonConfig(pylons.config) configutil.loadConfigOverrides() LOG.info("Complete loading config overrides") # build in memory security token cache appGlobal.authztoken = str(uuid.uuid4()) appGlobal.encryptedtokens = {} agentauth.buildTokenCache(appGlobal.authztoken) LOG.info("Complete building security token cache") # agent health appGlobal.agentHealth = 'True' appGlobal.agentHealthFactor = None from datetime import datetime appGlobal.agentInfo = { 'version_wiri': agenthealth.loadVersion(), 'startup_time': str(datetime.now()) } appGlobal.osInfo = agenthealth.getOsInfo() appGlobal.diskOk = True appGlobal.autoRestartEnabled = asbool( pylons.config['auto_restart_enabled']) LOG.info("Agent health initialized") # start thread mgr appGlobal.threadMgr = ThreadMgr( garbageFreq=pylons.config['threadMgr_garbage_freq'], maxThreadAge=pylons.config['threadMgr_thread_age']) if startThreadMgr: appGlobal.threadMgr.start() LOG.info("Agent thread manager started") # start package mgr appGlobal.packageMgr = PackageMgr( garbageFreq=pylons.config['packageMgr_garbage_freq'], maxPackageAge=pylons.config['packageMgr_package_age']) if startPackageMgr: appGlobal.packageMgr.start() LOG.info("Agent package manager started") # now load saved .metadata for previously saved hwPath, serverAddress, and serverAuth enrollMeta = manifestutil.readJsonServiceMeta('agent', ['hwPath']) hwPath = enrollMeta.get('hwPath') if hwPath is not None: appGlobal.hwPath = hwPath LOG.info("Agent hwPath loaded") #start monitor manager from agent.lib.monitors.monitor import AgentMonitor appGlobal.agentMonitor = AgentMonitor() if startAgentMonitor: appGlobal.agentMonitor.start() LOG.info("Agent monitor started") # start graphite udp server # from agent.lib.monitors.graphited import GraphiteDServer # from multiprocessing import Pipe # graphiteEnabled = configutil.getConfigAsBool('graphite_enabled') # if startAgentMonitor and graphiteEnabled: # output_p, input_p = Pipe() # appGlobal.graphited = GraphiteDServer(input_p) # appGlobal.graphited.start() # udpMsgProcessThread = threading.Thread(target=GraphiteDServer.read_queue, args=(output_p,)) # udpMsgProcessThread.daemon = True # udpMsgProcessThread.start() # LOG.info("Agent graphited started") # Declare dictionary for storing dynamic controllers appGlobal.dynacontrollers = dict() # metrix manager initialization from agent.lib.monitors.metrix_manager import MetrixManager from agent.lib.monitors.system_monitor import SystemMonitor appGlobal.metrixManager = MetrixManager() appGlobal.systemMonitor = SystemMonitor() appGlobal.metrixManager.register('OS', appGlobal.systemMonitor.getOSinfo, 1) appGlobal.metrixManager.register('Node Name', appGlobal.systemMonitor.getNodeName, 1) appGlobal.metrixManager.register('Free Memory(KB)', appGlobal.systemMonitor.getFreeMemory, 2) appGlobal.metrixManager.register('CPU Usage(%)', appGlobal.systemMonitor.getCpuUsage, 2) appGlobal.metrixManager.register('Version', agenthealth.loadVersion, 5) appGlobal.metrixManager.register('Configuration', validate_internals.getConfigFile, 6) appGlobal.metrixManager.register('Configuration Overrides', configutil.getConfigOverrides, 7) appGlobal.metrixManager.register( 'hwPath', lambda: appGlobal.hwPath if hasattr(appGlobal, 'hwPath') else None, 5) appGlobal.metrixManager.register(AGENT_HEALTH_VI_KEY, lambda: appGlobal.agentHealth, 1) appGlobal.metrixManager.register(AGENT_HEALTH_FACTOR_VI_KEY, lambda: appGlobal.agentHealthFactor, 1) appGlobal.metrixManager.register(OS_INFO_KEY, lambda: appGlobal.osInfo, 2) appGlobal.metrixManager.register(AGENT_INFO_KEY, lambda: appGlobal.agentInfo, 2) LOG.info("Agent health metrics registered") # start all agent modules modulestartthread = threading.Thread( target=ModuleController.loadModuleOnAgentStartup) modulestartthread.start() LOG.info("Local modules started") # start all services with active manifest, and load dynamic controllers servicestartthread = threading.Thread( target=ServiceController.startServicesOnAgentStartup) servicestartthread.start() LOG.info("Local services started") appGlobal.sdutil = shutdownAgent
def _addMonitors(self, service, activeManifest): ''' add monitors for a service's activeManifest ''' monitorTasks = [] self.__monitorTasks[(service, activeManifest)] = monitorTasks # metric tags metricTags = {} # add default metric tags metricTags['fqdn'] = utils.fqdn # read tags from agent agtMetricTags = manifestutil.readJsonServiceMeta('agent', [METRIC_TAGS_KEY]) if agtMetricTags and METRIC_TAGS_KEY in agtMetricTags: metricTags.update(agtMetricTags[METRIC_TAGS_KEY]) # read tags from service if (service != 'agent'): serviceMetricTags = manifestutil.readJsonServiceMeta(service, [METRIC_TAGS_KEY]) if serviceMetricTags or METRIC_TAGS_KEY in serviceMetricTags: metricTags.update(serviceMetricTags[METRIC_TAGS_KEY]) # all packages in this manifest packages = ManifestController.getPackages(service, os.path.basename(activeManifest)) # metric tags for externally passed in metrics # monitorTags = {additionaltags_dict, datatype_dict, ctx(env.pool.host), appType(Monitor|custom), monitorType(Cronus|CronusApplication)} self.__monitorTags[service] = {'default': (metricTags, {}, {})} self.__monitorValues[(service, 'default')] = {} self.__monitorMessages[(service, 'default')] = {} # a unique index for all monitors in the service for package in packages: # load monitor setting from cronus.ini pkgConfigs = manifestutil.getPackageInitConfig(service, activeManifest, package) if pkgConfigs is not None and 'monitors' in pkgConfigs: metricCtx = pkgConfigs[METRIC_CONTEXT] if METRIC_CONTEXT in pkgConfigs else {} metricTypes = {} if METRIC_TYPE_KEY in pkgConfigs and pkgConfigs[METRIC_TYPE_KEY]: metricTypes = pkgConfigs[METRIC_TYPE_KEY] if METRIC_TAGS_KEY in pkgConfigs and pkgConfigs[METRIC_TAGS_KEY]: metricTags.update(pkgConfigs[METRIC_TAGS_KEY]) monitors = pkgConfigs['monitors'] for monitor in monitors: try: m_name = str(monitor['name']) # metric tags for externally passed in metrics # monitorTags = {additionaltags_dict, datatype_dict, ctx(env.pool.host), appType(Monitor|custom), monitorType(Cronus|CronusApplication)} self.__monitorTags[service][m_name] = (metricTags, metricTypes, metricCtx) # now normalize the interval to one allowable by EVPS (downcast) if 'reportIntervalSec' not in monitor: continue m_interval = float(monitor['reportIntervalSec']) m_interval = self._normalizeResolution(m_interval) m_timeout = float(monitor['timeoutSec']) if 'timeoutSec' in monitor else m_interval m_timeout = max(1, min(m_interval-1, m_timeout)) m_items = monitor['items'] if 'items' in monitor else [] # prime monitor value store self.__monitorValues[(service, m_name)] = {} self.__monitorMessages[(service, m_name)] = {} mi_idx = 0 for m_item in m_items: if 'type' not in m_item: continue mi_idx += 1 mi_type = m_item['type'] mi_cmd = None if 'script' == mi_type: mi_script = m_item['script'] mi_cmd = manifestutil.getMonitorScriptPath(service, activeManifest, package, mi_script) else: # unknown type, skip LOG.error('Unknown monitor item type %s, can only be script' % mi_type) continue # schedule monitor m_sch_name = '%s.%s' % (service, m_name) try: self.__monitorMessages[(service, m_name)][mi_cmd] = [] task = self.__monitorSch.add_interval_task(self._runMonitor, '%s.%s' % (m_sch_name, str(mi_idx)), 0, float(m_interval), [mi_type,mi_cmd,service,m_name,m_timeout,(str(int(round(m_interval))),metricTags,metricTypes,metricCtx)],None) monitorTasks.append(task) except Exception as excep: LOG.error('Cannot add monitor task for %s.%s - (%s) %s' % (m_sch_name, str(mi_idx), excep, traceback.format_exc(5))) except Exception as excep1: LOG.error('Cannot add report task for %s.%s - (%s) %s' % (service, m_name, excep1, traceback.format_exc(5)))