def __init__(self, job_info):
IMonitoringService.__init__(self, job_info)
from Ganga.Lib.MonitoringServices.MSGPeek.compatibility import uuid
self.ganga_job_uuid = uuid()
"""stdoutFile actualPos are needed to know until where the client has read"""
self.stdoutFile = None
self.actualPos = 0
"""The streaming is only check to regular intervals @NUM_SEC"""
self.start_time = datetime.datetime.now()
self.stream = False
def __init__(self, job_info):
IMonitoringService.__init__(self, job_info)
if isinstance(job_info, type({})):
print job_info
self.server = self.job_info['octopus_server']
self.port = self.job_info['octopus_port']
self.client = Octopus(self.server, self.port)
self.channel = self.job_info['channel']
self.stdoutpos = 0
self.stderrpos = 0
self.DEBUG = False
def __init__(self, monClasses, jobInfos, configInfos):
"""Create a new composite monitoring service based on the lists of
monitoring classes, jobs and configs (all the same length).
If this is called in the Ganga client, i.e. from Ganga/GPIDev/MonitoringServices,
then jobInfos is a list of Job (all the same), configInfos is a list of
Config (specific to each monitoring class).
If this is called on the worker node, i.e. from the text generated by
getWrapperScriptConstructorText(), the jobInfos are dictionaries (specific
to each monitoring class) and configInfos are dictionaries of effective
config options (specific to each monitoring class).
"""
if not (len(monClasses) == len(jobInfos) == len(configInfos)):
raise Exception(
"cannot create monitoring object, list of monitoring classes, jobs and configs are not the same length.")
IMonitoringService.__init__(self, jobInfos, configInfos)
# init the logger
try:
import Ganga.Utility.logging
self.logger = Ganga.Utility.logging.getLogger()
except ImportError:
# on the worker node we don't have access to Ganga logging facilities
# so we simple print out the log message
#@see self._log()
self.logger = None
# init the monitoring services
self.monMonServices = []
for i in range(len(monClasses)):
try:
monClass = monClasses[i]
# allow for existing monitoring classes which do not take
# config_info in constructor
if configInfos[i] is None:
monService = monClass(jobInfos[i])
else:
monService = monClass(jobInfos[i], configInfos[i])
self.monMonServices.append(monService)
except Exception as e:
# discard errors in initialization of monitoring services
self._log(
level="warning", msg="Failed to init %s monitoring service...discarding it" % str(monClass))
from Ganga.Utility.logging import log_user_exception
log_user_exception(self.logger)
def getSandboxModules(self):
"""Return list of MSGMS module dependencies."""
import Ganga.Lib.MonitoringServices.MSGMS
return IMonitoringService.getSandboxModules(self) + [
Ganga.Lib.MonitoringServices.MSGMS,
Ganga.Lib.MonitoringServices.MSGMS.MSGMS,
] + MSGUtil.getSandboxModules()
def getSandboxModules(self):
import Ganga.Lib.MonitoringServices.MSGPeek
import Ganga.Lib.MonitoringServices.MSGPeek.MSGPeek
return [
Ganga,
Ganga.Lib,
Ganga.Lib.MonitoringServices,
Ganga.Lib.MonitoringServices.MSGPeek,
Ganga.Lib.MonitoringServices.MSGPeek.MSGPeek,
Ganga.Lib.MonitoringServices.MSGPeek.MSGUtil,
Ganga.Lib.MonitoringServices.MSGPeek.compatibility,
Ganga.Lib.MonitoringServices.MSGPeek.stomp,
Ganga.Utility,
Ganga.Utility.logging,
Ganga.Utility.strings,
Ganga.Utility.files,
# Ganga.Utility.files.remove_prefix,
Ganga.Utility.ColourText,
Ganga.Utility.Config,
Ganga.Utility.Config.Config,
Ganga.GPIDev,
Ganga.GPIDev.Lib,
Ganga.GPIDev.Lib.Config,
Ganga.GPIDev.Lib.Config.Config,
Ganga.Core,
Ganga.Core.exceptions,
Ganga.Core.exceptions.GangaException
] + IMonitoringService.getSandboxModules(self)
def __init__(self, monClasses, jobInfos, configInfos):
"""Create a new composite monitoring service based on the lists of
monitoring classes, jobs and configs (all the same length).
If this is called in the Ganga client, i.e. from Ganga/GPIDev/MonitoringServices,
then jobInfos is a list of Job (all the same), configInfos is a list of
Config (specific to each monitoring class).
If this is called on the worker node, i.e. from the text generated by
getWrapperScriptConstructorText(), the jobInfos are dictionaries (specific
to each monitoring class) and configInfos are dictionaries of effective
config options (specific to each monitoring class).
"""
if not (len(monClasses) == len(jobInfos) == len(configInfos)):
raise Exception(
"cannot create monitoring object, list of monitoring classes, jobs and configs are not the same length.")
IMonitoringService.__init__(self, jobInfos, configInfos)
# init the logger
try:
import Ganga.Utility.logging
self.logger = Ganga.Utility.logging.getLogger()
except ImportError:
# on the worker node we don't have access to Ganga logging facilities
# so we simple print out the log message
self.logger = None
# init the monitoring services
self.monMonServices = []
for i in range(len(monClasses)):
try:
monClass = monClasses[i]
# allow for existing monitoring classes which do not take
# config_info in constructor
if configInfos[i] is None:
monService = monClass(jobInfos[i])
else:
monService = monClass(jobInfos[i], configInfos[i])
self.monMonServices.append(monService)
except Exception as e:
# discard errors in initialization of monitoring services
self.logger.warning("Failed to init %s monitoring service...discarding it" % str(monClass))
from Ganga.Utility.logging import log_user_exception
log_user_exception(self.logger)
def getSandboxModules(self):
import Ganga.Lib.MonitoringServices.JobExecutionMonitorMS
return IMonitoringService.getSandboxModules(self) + [
Ganga,
Ganga.Lib,
Ganga.Lib.MonitoringServices,
Ganga.Lib.MonitoringServices.JobExecutionMonitorMS,
Ganga.Lib.MonitoringServices.JobExecutionMonitorMS.JobExecutionMonitorMS
]
def getSandboxModules(self):
import Ganga.Lib.MonitoringServices.DummyMS
return IMonitoringService.getSandboxModules(self) + [
Ganga,
Ganga.Lib,
Ganga.Lib.MonitoringServices,
Ganga.Lib.MonitoringServices.DummyMS,
Ganga.Lib.MonitoringServices.DummyMS.DummyMS
]
def getSandboxModules(self):
""" Get the list of module dependencies of this monitoring module.
Called by: ganga client.
"""
#modules required by this container itself
import Ganga.Lib.MonitoringServices
modules = IMonitoringService.getSandboxModules(self) + \
[Ganga, Ganga.Lib, Ganga.Lib.MonitoringServices, Ganga.Lib.MonitoringServices.Composite]
for monService in self.monMonServices:
try:
monClass = str(monService.__class__)
#TODO:
# the list might contain duplicate elements.
# does this cause troubles on the upper levels?
modules.extend(monService.getSandboxModules())
except Exception,e:
#discard errors in initialization of monitoring services
self._log(level="warning",msg="%s monitoring service failed in *getSandboxModules* ... ignoring it." % monClass)
def getSandboxModules(self):
# it would be nice if this would be more readable.
import Ganga.Lib.MonitoringServices.ARDADashboard.LCG
import ApMon
import ApMon.apmon
import ApMon.Logger
return IMonitoringService.getSandboxModules(self) + [Ganga,
Ganga.Lib,
Ganga.Lib.MonitoringServices,
Ganga.Lib.MonitoringServices.ARDADashboard,
Ganga.Lib.MonitoringServices.ARDADashboard.DashboardAPI,
Ganga.GPIDev,
Ganga.GPIDev.Adapters,
Ganga.GPIDev.Adapters.IMonitoringService,
ApMon,
ApMon.apmon,
ApMon.Logger,
ApMon.ProcInfo,
Ganga.Lib.MonitoringServices.ARDADashboard.LCG,
Ganga.Lib.MonitoringServices.ARDADashboard.LCG.ARDADashboardLCG,
]
def __init__(self, job_info, config_info):
"""Construct the GangaMon monitoring service."""
IMonitoringService.__init__(self, job_info, config_info)
def __init__(self, job_info):
IMonitoringService.__init__(self, job_info)
def __init__(self, job_info):
IMonitoringService.__init__(self, job_info)
if isinstance(job_info, DictionaryType):
# we are on the worker node. We just need
# to get values from the dictionary
try:
self.gangaJobId = job_info['gangaJobId']
self.gangaTaskId = job_info['gangaTaskId']
self.gridBackend = job_info['gridBackend']
self.gridCertificate = job_info['gridCertificate']
self.VO = job_info['VO']
self._complete = True
except KeyError as msg:
# too bad, we will not monitor the job
return
# on the WN, we get the job ID from envar
self.gridJobId = safe_getenv('EDG_WL_JOBID')
if self.gridJobId == 'unknown':
self.gridJobId = safe_getenv('GLITE_WL_JOBID')
if self.gridJobId == 'unknown':
self._complete = False
else:
# we are in the client session. job_info is a Job()
from Ganga.Utility.logging import getLogger
self._logger = getLogger()
job = job_info
self.gridBackend = getattr(job, 'backend')._name
if self.gridBackend not in ['LCG']:
self._logger.debug('not sending monitoring because not in LCG')
return
self._logger.debug(job.backend)
self._logger.debug(job.backend.id)
self.gridJobId = job.backend.id
# we compute the "jobID" and "taskID"
# (which is the gangaJobId followed by the user@repository
# the repository unique ID:
from Ganga.Utility.Config import getConfig, ConfigError
config = getConfig('Configuration')
rep_type = config['repositorytype']
rep_login = config['user']
if 'Local' in rep_type:
from Ganga.Runtime import Repository_runtime
rep_dir = Repository_runtime.getLocalRoot()
sys_config = getConfig('System')
rep_hostname = sys_config['GANGA_HOSTNAME']
rep_location = rep_hostname + ':' + rep_dir
elif 'Remote' in rep_type:
remote_config = getConfig(rep_type + "_Repository")
rep_host = remote_config['host']
rep_port = remote_config['port']
rep_location = rep_host + ':' + rep_port
else:
return
repository_id = rep_login + '@' + rep_location
master_job = job.master
if master_job is not None:
master_id = master_job.id
self._logger.debug('found master: %d' % master_id)
self.gangaTaskId = self.taskPrefix + '_' + \
str(master_id) + '_' + repository_id
self.gangaJobId = str(job.id)
else:
self.gangaTaskId = self.taskPrefix + \
'_' + str(job.id) + '_' + repository_id
self.gangaJobId = '0'
self._logger.debug('task_id = %s' % self.gangaTaskId)
self._logger.debug('task_job_id = %s' % self.gangaJobId)
backendConfig = getConfig(self.gridBackend)
try:
self.VO = backendConfig['VirtualOrganisation']
except KeyError:
self._logger.debug('VirtualOrganisation not configured')
# we need it, it's too dangerous if we are not sure
return
from Ganga.GPIDev.Credentials import getCredential
proxy = getCredential('GridProxy')
self.gridCertificate = proxy.info('-subject')
if self.gridCertificate is None:
self._logger.debug('error: grid certificate not known')
return
if self.gridJobId is None:
self._logger.debug('normal: grid job ID is None')
self._logger.debug('job is complete')
self._complete = True
# we can now initialize the dashboard communication thing
if self.gridJobId is not None:
try:
self.dashboard = DashboardAPI(
self.gangaTaskId, self.gangaJobId + '_' + self.gridJobId)
except TypeError:
self.dashboard = DashboardAPI(
self.gangaTaskId, self.gangaJobId + '_' + '_'.join(self.gridJobId))
def getSandboxModules(self):
import Ganga.Lib.MonitoringServices.OutputServerMS.OutputServerMS
return [Ganga, Ganga.Lib, Ganga.Lib.MonitoringServices, Ganga.Lib.MonitoringServices.OutputServerMS, Ganga.Lib.MonitoringServices.OutputServerMS.OutputServerMS] + IMonitoringService.getSandboxModules(self)
def getSandboxModules(self):
print "Sending sandbox modules"
import Ganga.Lib.MonitoringServices.Octopus.OctopusMS
return [Ganga, Ganga.Lib, Ganga.Lib.MonitoringServices, Ganga.Lib.MonitoringServices.Octopus, Ganga.Lib.MonitoringServices.Octopus.OctopusMS, Ganga.Lib.MonitoringServices.Octopus.Octopus] + IMonitoringService.getSandboxModules(self)