def getApmonInstance(): global apmonInstance global apmonInit if apmonInstance is None and not apmonInit: apmonInit = True if apmonUseUrl: apm = None #print "Creating ApMon with dynamic configuration/url" try: apm = apmon.ApMon(apmonUrlList, apmonLoggingLevel) except Exception as e: pass if apm is not None and not apm.initializedOK(): #print "Setting ApMon to static configuration" try: apm.setDestinations(apmonConf) except Exception as e: apm = None apmonInstance = apm if apmonInstance is None: #print "Creating ApMon with static configuration" try: apmonInstance = apmon.ApMon(apmonConf, apmonLoggingLevel) except Exception as e: pass return apmonInstance
def getApmonObj(inDest): """ Checks if inDest is an object and if not tries to create one.""" if isinstance(inDest, (str, unicode, basestring)): apMonDestinations = tuple(inDest.split(',')) apMon = apmon.ApMon(apMonDestinations) apMon.enableBgMonitoring(True) return apMon else: return inDest
def connect(self): """ initialize via _connect_ """ self.apmon = apmon.ApMon(self.apmonConf, apmonLoggingLevel) self.apmon.setMaxMsgRate(self.setMaxMsgRate) self.apmon.sendParameter(self.clusterName, self.nodeName, 'InstanceId', self.instanceId) return
def doStuff(i): print("start", i) apm = apmon.ApMon('http://monalisa.cern.ch/apmon/destinations.conf') apm.sendParameter('cluster', 'node', 'param', 34 + i) time.sleep(0.01) apm.sendParameter('cluster', 'node', 'param', -34 + i) time.sleep(0.06) apm.sendParameter('cluster', 'node', 'param', i) # You absolutely have to call this function here! apm.free() print("end", i)
def __init__(self, configurationPath, socketPath): self._configurationPath = configurationPath self._socketPath = socketPath self._container = docker_utils.DockerUtils(socketPath) # Initialize ApMon specifying that it should not send information about the system. # Note that in this case the background monitoring process isn't stopped, in case you may # want later to monitor a job. self._apm = apmon.ApMon(self._configurationPath) self._apm.setLogLevel("INFO") self._apm.confCheck = False self._apm.enableBgMonitoring(False) self._apm.setMaxMsgRate(1000)
def __init__(self, config=None, max_rate=20, log_level="INFO"): super(Apmon, self).__init__() import apmon log_level = getattr(apmon.Logger, log_level.upper()) self._apmon = apmon.ApMon(config or self.default_config, log_level) self._apmon.maxMsgRate = int(max_rate * 1.5) # hotfix of a bug occurring in apmon for too large pids for key, value in self._apmon.senderRef.items(): value["INSTANCE_ID"] = value["INSTANCE_ID"] & 0x7fffffff self._max_rate = max_rate self._queue = six.moves.queue.Queue() self._stop_event = threading.Event()
def __init__(self, *destination): super(AliceApMonBackend, self).__init__() self._logger = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) # initialization self.destination = destination # initialize ApMon, reroute logging by replacing Logger at module level apmon_logger, apmon.Logger = apmon.Logger, ApMonLogger self._apmon = apmon.ApMon(destination) apmon.Logger = apmon_logger # BUGFIX: apmon can create an invalid identifier on systems with pids > 32767 if any(senderRef['INSTANCE_ID'] > 2147483647 for senderRef in self._apmon.senderRef.values()): raise RuntimeError( 'invalid ApMon INSTANCE_ID' ) # https://github.com/MonALISA-CIT/apmon_py/issues/4 # background monitoring self._background_monitor_sitename = None self._service_job_monitor = set()
# Also, note that: # - you can monitor multiple jobs # - the retrieved information is a sum for all processes forked from the given pid # - you can stop background processes with stopBgProcesses() and still do system and # job monitoring. You just have to call sendBgMonitoring() whenever you want to # send the data. However, note that also the config checker process is stopped, so # if you get the config from a URL, and the configuration changes, you will not get the # changes. """ from __future__ import print_function import apmon import time import os # Each 10 seconds send information about given jobs and no information about the system apm = apmon.ApMon(['dest_3.conf']) print("apmon was init") # Monitor this process and all its children # apm.addJobToMonitor(os.getpid(), os.getcwd(), "JobInfoTest", "job") apm.addJobToMonitor("1", os.getcwd(), "JobInfoTest", "job") t = 40 print('Sleeping for 40 seconds') time.sleep(t) print('Working for 40 seconds') start = time.time() while start + t > time.time(): pass
""" # This example shows how ApMon can be used to send parameters with a # given time, set by user to the MonALISA service # This feature can be useful, for example, if you generate the parameters by # parsing a log file and you want that they have the time from the log. """ from __future__ import print_function import apmon import time import random # Initialize ApMon by specifying a ref to a list of hosts having a MonALISA service running # with monXDRUDP module enabled. # Note that background monitor process sends information about the host. In apmon.py # is the list with default parameters. If you want to modify what is sent, you can # take the configuration from a file or URL apm = apmon.ApMon(('aliendb10.cern.ch:8884', )) for i in list(range(1, 20)): my_time = int(time.time() - 2 * 3600 - (20 - i) * 60) state = int(random.randint(0, 10)) # Note that also there's a sendTimedParams version apm.sendTimedParameters("LogParser", "SomeApp", my_time, {'state': state}) print("Set for i=", i)
#!/usr/bin/python """ This is a simple test program for the 'apmon' Python module. """ from __future__ import print_function import apmon import time from Logger import Logger # read destination hosts form file apmonUrls = ["http://lxgate35.cern.ch:40808/ApMonConf1", "http://monalisa.cacr.caltech.edu:40808/ApMonConf"] apm = apmon.ApMon(apmonUrls, Logger.DEBUG) if not apm.initializedOK(): print("It seems that ApMon cannot read its configuration. Setting the default destination") apm.setDestinations({'pcardaab.cern.ch:8884': {'sys_monitoring': 0, 'general_info': 0, 'job_monitoring': 1, 'job_interval': 300}}) # set the destinations as a tuple of strings # apm = apmon.ApMon (('141.85.99.136:8453', 'ui.rogrid.pub.ro')) # or # apm = apmon.ApMon(('ui.rogrid.pub.ro:8884', )) # trailing comma => tuple with a single value! # read the destinations from a URL # apm = apmon.ApMon("http://rb.rogrid.pub.ro/~catac/destinations.conf") # check for changes in the configuration files # apm.configRecheck = True
""" # This example shows how ApMon can be used to send the parameters # to the MonALISA service (given in the constructor) """ from __future__ import print_function import apmon import sys # Initialize ApMon specifying that it should not send information about the system. # Note that in this case the background monitoring process isn't stopped, in case you may # want later to monitor a job. apm = apmon.ApMon('dest_2.conf') apm.setLogLevel("INFO") apm.confCheck = False apm.enableBgMonitoring(False) apm.setMaxMsgRate(1000) maxV = 0 try: maxV = int(sys.argv[1]) except IndexError as er: print("Error in first call parameter: " + str(er)) print("sending to", maxV) for i in list(range(1, maxV)): # you can put as many pairs of parameter_name, parameter_value as you want # but be careful not to create packets longer than 8K. apm.sendParameters("SimpleCluster", "SimpleNode", {'var_i': i}) # time.sleep(0.1);
#print "Creating ApMon with dynamic configuration/url" try: apm = apmon.ApMon(apmonUrlList, apmonLoggingLevel) except Exception, e: pass if apm is not None and not apm.initializedOK(): #print "Setting ApMon to static configuration" try: apm.setDestinations(apmonConf) except Exception, e: apm = None apmonInstance = apm if apmonInstance is None: #print "Creating ApMon with static configuration" try: apmonInstance = apmon.ApMon(apmonConf, apmonLoggingLevel) except Exception, e: pass return apmonInstance # # Method to free the apmon instance # def apmonFree(): global apmonInstance global apmonInit if apmonInstance is not None: time.sleep(1) try: apmonInstance.free()
""" # This example shows how ApMon can be used to send parameters with a # given time, set by user to the MonALISA service # This feature can be useful, for example, if you generate the parameters by # parsing a log file and you want that they have the time from the log. """ from __future__ import print_function import apmon import time import random # Initialize ApMon by specifying a ref to a list of hosts having a MonALISA service running # with monXDRUDP module enabled. # Note that background monitor process sends information about the host. In apmon.py # is the list with default parameters. If you want to modify what is sent, you can # take the configuration from a file or URL apm = apmon.ApMon(('pcardaab.cern.ch:8884', )) for i in list(range(1, 20)): my_time = time.time() - 2 * 3600 - (20 - i) * 60 state = int(random.randint(0, 10)) # Note that also there's a sendTimedParams version apm.sendTimedParameters("LogParser", "SomeApp", my_time, {'state': state}) print("Set for i=", i)
from __future__ import print_function # example for testing / setting up ApMon MonALISA monitoring import apmon import time import random # read destination hosts from file #apm = apmon.ApMon("http://monalisa2.cern.ch/~catac/apmon/destinations.conf"); # set the destinations as a tuple of strings apm = apmon.ApMon(("monalisa.cern.ch:28884", "monalisa.caltech.edu:28884")) # check for changes in the configuration files #apm.configRecheck = True # apm.configRecheckInterval = 10 # (time in seconds) print("ApmTest: Destinations:", apm.destinations) # what happens if hostname is not specified? - puts just 'localhost' transferId = "some_id" print("sending stuff ...") for i in range(20): par = dict(id=transferId, initialisation=random.randint(10, 30)) apm.sendParameters("fdtcp", None, par) time.sleep(0.05)
# This is an example that shows how Log Levels can be used. # The logLevel can also be specified in the configuration file like: # xApMon_loglevel = INFO """ from __future__ import print_function from builtins import range import apmon import time # Also, we show how you can initialize using a hash of hashes. # By default, cpu_usr cpu_sys are not sent. This way you can enable them apm = apmon.ApMon( { 'pcardaab.cern.ch': { 'sys_cpu_usr': True, 'sys_cpu_sys': True, 'job_monitoring': False } }, defaultLogLevel=apmon.Logger.WARNING) # apm.setLogLevel("DEBUG") for i in range(1, 100): print('Sending i =', i) if i == 20: apm.setLogLevel("NOTICE") if i == 50: apm.setLogLevel("DEBUG") apm.sendParameters("MyCluster", "MyNode", {'val_i': i}) apm.sendTimedParameters("MyClusterOld", "MyNodeOld", time.time() - 5 * 3600, {'val_ii': i})
""" # This example shows how ApMon can be used as a simple host sensor to monitor and send # the desired parameters to the MonALISA service (configured in the given file/url/list) """ from __future__ import print_function import apmon import os import time # apm = apmon.ApMon('dest_3.conf'); apm = apmon.ApMon('http://monalisa2.cern.ch/~catac/apmon/destinations.conf') # apm = apmon.ApMon('http://gangamon.cern.ch:8080/apmon/ganga.conf'); # apm = apmon.ApMon({'lxgate35.cern.ch' : {'sys_interval' : 10}}); apm.setLogLevel('DEBUG') my_host = os.popen('hostname -f').readline().strip() # the background data about the system will be sent with this cluster and node apm.setMonitorClusterNode("MyTestSensor", my_host) print("host-sensor: sleeping forever; hit ctrl+c to stop") # sleep forever while True: time.sleep(1)
def main(): task = "TestTask4" # context['MonitorID'] job = "TestJob4" # context['MonitorJobID'] ppid = os.getppid() cdir = os.getcwd() apmonUrls = [ "http://lxgate35.cern.ch:40808/ApMonConf1", "http://monalisa.cacr.caltech.edu:40808/ApMonConf" ] # apmonUrls = ["http://monalisa.cacr.caltech.edu:40808/ApMonConf"] # apmonUrls = [] cdate = os.popen("date ").read()[:-1] print("Create apmon instance " + cdate) apm = apmon.ApMon(apmonUrls, apmon.Logger.ERROR) if not apm.initializedOK(): print( "It seems that ApMon cannot read its configuration. Setting the default destination" ) apm.setDestinations({ 'pcardaab.cern.ch:8884': { 'sys_monitoring': 0, 'general_info': 0, 'job_monitoring': 1, 'job_interval': 300 } }) print("ApMon Initialized\n===========================") print(apm.getConfig() + "===========================") # apm = apmon.ApMon({'192.91.245.5:58884': {'job_interval':60,'sys_info':False,'general_info': False}}) # apm.setLogLevel('DEBUG') apm.addJobToMonitor(ppid, cdir, task, job) apm.setMaxMsgRate(500) # print "ApmReport: Destinations: " + `apm.destinations` params = {} f = open("data.cms", "r") line = f.readline() while len(line) > 0: spc = line.find(" ") if spc == -1: print("ignoring", line) continue key = line[0:spc] val = line[spc + 1:-1] try: valn = float(val) params[key] = valn except ValueError: params[key] = val line = f.readline() f.close() cnt = 0 lastcnt = 0 lastsec = 0 # params = {"a": 1, "b": 3} print("params:", len(params), repr(params)) while 1: try: time.sleep(1) # print ".", if int(time.time()) != lastsec: lastsec = int(time.time()) print("last sec i sent", (cnt - lastcnt), "packets") lastcnt = cnt cnt += 1 apm.sendParams(params) print("params:", len(params), repr(params)) except: print("Before sendBgMonitoring") apm.sendBgMonitoring(True) print("After sendBgMonitoring") print("Now go out") apm.free() time.sleep(1) break
'echo $(cat gcc --version | grep gcc)', 'WLCG Metapackage': '[ -z $(echo $(rpm -qa | grep -w HEP_OSlibs) | cut -f1 -d$" ") ] && echo "Available: No" || echo "Available: Yes"' } if __name__ == "__main__": with zipfile.ZipFile('apmon.zip', 'r') as zip_ref: zip_ref.extractall('./') import apmon # Initialize ApMon specifying that it should not send information about the system. # Note that in this case the background monitoring process isn't stopped, in case you may # want later to monitor a job. apm = apmon.ApMon((url, )) apm.setLogLevel("NOTICE") apm.confCheck = False apm.enableBgMonitoring(False) apm.setMaxMsgRate(75) apm.setMaxMsgSize(500) if not apm.initializedOK(): print( "It seems that ApMon cannot read its configuration. Setting the default destination" ) else: print("ApMon initialized succesfully") # you can put as many pairs of parameter_name, parameter_value as you want # but be careful not to create packets longer than 8K.
# allows users to run jobs and each worker node where the job is run # you want to report the data to the closest MonALISA service without having # to know apriori on which site the job would run. # Generating the config based on parameters is useful for example to send the # the information from 2 or more different applications to different dedicated # MonALISA services. """ from __future__ import print_function import os import time import apmon # change the value for appName parameter to see the difference. # Also see the cgi script and/or the servlet that generates the configuration apm = apmon.ApMon("http://pcardaab.cern.ch:8888/cgi-bin/ApMonConf?appName=confgen_test") apm.setLogLevel("DEBUG") my_host = os.popen('hostname -f').readline().strip() # the background data about the system will be sent with this cluster and node apm.setMonitorClusterNode("MyStatus", my_host) # just send some data x = 0 while x < 20: print("Sending for x = ", x) apm.sendParameter("MyCluster", "MyNode", "x", x) time.sleep(1) x += 1