Ejemplo n.º 1
0
 def __init__(self, job_info):
     IMonitoringService.__init__(self, job_info)
     from Ganga.Lib.MonitoringServices.MSGPeek.compatibility import uuid
     self.ganga_job_uuid = uuid()
     """stdoutFile actualPos are needed to know until where the client has read"""
     self.stdoutFile = None
     self.actualPos = 0
     """The streaming is only check to regular intervals @NUM_SEC"""
     self.start_time = datetime.datetime.now()
     self.stream = False
Ejemplo n.º 2
0
 def __init__(self, job_info):
     IMonitoringService.__init__(self, job_info)
     if isinstance(job_info, type({})):
         print job_info
         self.server = self.job_info['octopus_server']
         self.port = self.job_info['octopus_port']
         self.client = Octopus(self.server, self.port)
         self.channel = self.job_info['channel']
         self.stdoutpos = 0
         self.stderrpos = 0
         self.DEBUG = False
Ejemplo n.º 3
0
    def __init__(self, monClasses, jobInfos, configInfos):
        """Create a new composite monitoring service based on the lists of
        monitoring classes, jobs and configs (all the same length).

        If this is called in the Ganga client, i.e. from Ganga/GPIDev/MonitoringServices,
        then jobInfos is a list of Job (all the same), configInfos is a list of
        Config (specific to each monitoring class).

        If this is called on the worker node, i.e. from the text generated by
        getWrapperScriptConstructorText(), the jobInfos are dictionaries (specific
        to each monitoring class) and configInfos are dictionaries of effective
        config options (specific to each monitoring class).
        """

        if not (len(monClasses) == len(jobInfos) == len(configInfos)):
            raise Exception(
                "cannot create monitoring object, list of monitoring classes, jobs and configs are not the same length.")

        IMonitoringService.__init__(self, jobInfos, configInfos)

        # init the logger
        try:
            import Ganga.Utility.logging
            self.logger = Ganga.Utility.logging.getLogger()
        except ImportError:
            # on the worker node we don't have access to Ganga logging facilities
            # so we simple print out the log message
            #@see self._log()
            self.logger = None

        # init the monitoring services
        self.monMonServices = []
        for i in range(len(monClasses)):
            try:
                monClass = monClasses[i]
                # allow for existing monitoring classes which do not take
                # config_info in constructor
                if configInfos[i] is None:
                    monService = monClass(jobInfos[i])
                else:
                    monService = monClass(jobInfos[i], configInfos[i])
                self.monMonServices.append(monService)
            except Exception as e:
                # discard errors in initialization of monitoring services
                self._log(
                    level="warning", msg="Failed to init %s monitoring service...discarding it" % str(monClass))
                from Ganga.Utility.logging import log_user_exception
                log_user_exception(self.logger)
Ejemplo n.º 4
0
 def getSandboxModules(self):
     """Return list of MSGMS module dependencies."""
     import Ganga.Lib.MonitoringServices.MSGMS
     return IMonitoringService.getSandboxModules(self) + [
         Ganga.Lib.MonitoringServices.MSGMS,
         Ganga.Lib.MonitoringServices.MSGMS.MSGMS,
     ] + MSGUtil.getSandboxModules()
Ejemplo n.º 5
0
 def getSandboxModules(self):
     import Ganga.Lib.MonitoringServices.MSGPeek
     import Ganga.Lib.MonitoringServices.MSGPeek.MSGPeek
     return [
         Ganga,
         Ganga.Lib,
         Ganga.Lib.MonitoringServices,
         Ganga.Lib.MonitoringServices.MSGPeek,
         Ganga.Lib.MonitoringServices.MSGPeek.MSGPeek,
         Ganga.Lib.MonitoringServices.MSGPeek.MSGUtil,
         Ganga.Lib.MonitoringServices.MSGPeek.compatibility,
         Ganga.Lib.MonitoringServices.MSGPeek.stomp,
         Ganga.Utility,
         Ganga.Utility.logging,
         Ganga.Utility.strings,
         Ganga.Utility.files,
         # Ganga.Utility.files.remove_prefix,
         Ganga.Utility.ColourText,
         Ganga.Utility.Config,
         Ganga.Utility.Config.Config,
         Ganga.GPIDev,
         Ganga.GPIDev.Lib,
         Ganga.GPIDev.Lib.Config,
         Ganga.GPIDev.Lib.Config.Config,
         Ganga.Core,
         Ganga.Core.exceptions,
         Ganga.Core.exceptions.GangaException
     ] + IMonitoringService.getSandboxModules(self)
Ejemplo n.º 6
0
    def __init__(self, monClasses, jobInfos, configInfos):
        """Create a new composite monitoring service based on the lists of
        monitoring classes, jobs and configs (all the same length).

        If this is called in the Ganga client, i.e. from Ganga/GPIDev/MonitoringServices,
        then jobInfos is a list of Job (all the same), configInfos is a list of
        Config (specific to each monitoring class).

        If this is called on the worker node, i.e. from the text generated by
        getWrapperScriptConstructorText(), the jobInfos are dictionaries (specific
        to each monitoring class) and configInfos are dictionaries of effective
        config options (specific to each monitoring class).
        """

        if not (len(monClasses) == len(jobInfos) == len(configInfos)):
            raise Exception(
                "cannot create monitoring object, list of monitoring classes, jobs and configs are not the same length.")

        IMonitoringService.__init__(self, jobInfos, configInfos)

        # init the logger
        try:
            import Ganga.Utility.logging
            self.logger = Ganga.Utility.logging.getLogger()
        except ImportError:
            # on the worker node we don't have access to Ganga logging facilities
            # so we simple print out the log message
            self.logger = None

        # init the monitoring services
        self.monMonServices = []
        for i in range(len(monClasses)):
            try:
                monClass = monClasses[i]
                # allow for existing monitoring classes which do not take
                # config_info in constructor
                if configInfos[i] is None:
                    monService = monClass(jobInfos[i])
                else:
                    monService = monClass(jobInfos[i], configInfos[i])
                self.monMonServices.append(monService)
            except Exception as e:
                # discard errors in initialization of monitoring services
                self.logger.warning("Failed to init %s monitoring service...discarding it" % str(monClass))
                from Ganga.Utility.logging import log_user_exception
                log_user_exception(self.logger)
Ejemplo n.º 7
0
 def getSandboxModules(self):
     import Ganga.Lib.MonitoringServices.JobExecutionMonitorMS
     return IMonitoringService.getSandboxModules(self) + [
         Ganga,
         Ganga.Lib,
         Ganga.Lib.MonitoringServices,
         Ganga.Lib.MonitoringServices.JobExecutionMonitorMS,
         Ganga.Lib.MonitoringServices.JobExecutionMonitorMS.JobExecutionMonitorMS
         ]
Ejemplo n.º 8
0
 def getSandboxModules(self):
     import Ganga.Lib.MonitoringServices.DummyMS
     return IMonitoringService.getSandboxModules(self) + [
         Ganga,
         Ganga.Lib,
         Ganga.Lib.MonitoringServices,
         Ganga.Lib.MonitoringServices.DummyMS,
         Ganga.Lib.MonitoringServices.DummyMS.DummyMS
         ]
Ejemplo n.º 9
0
 def getSandboxModules(self):
    """ Get the list of module dependencies of this monitoring module.
    Called by: ganga client.
    """
    
    #modules required by this container itself
    import Ganga.Lib.MonitoringServices
    modules = IMonitoringService.getSandboxModules(self) + \
              [Ganga, Ganga.Lib, Ganga.Lib.MonitoringServices, Ganga.Lib.MonitoringServices.Composite]
    
    for monService in self.monMonServices:
       try:
          monClass = str(monService.__class__)
          #TODO: 
          # the list might contain duplicate elements.
          # does this cause troubles on the upper levels?            
          modules.extend(monService.getSandboxModules())
       except Exception,e:
          #discard errors in initialization of monitoring services
          self._log(level="warning",msg="%s monitoring service failed in *getSandboxModules* ... ignoring it." % monClass)
Ejemplo n.º 10
0
    def getSandboxModules(self):

        # it would be nice if this would be more readable.

        import Ganga.Lib.MonitoringServices.ARDADashboard.LCG
        import ApMon
        import ApMon.apmon
        import ApMon.Logger
        return IMonitoringService.getSandboxModules(self) + [Ganga,
                                                             Ganga.Lib,
                                                             Ganga.Lib.MonitoringServices,
                                                             Ganga.Lib.MonitoringServices.ARDADashboard,
                                                             Ganga.Lib.MonitoringServices.ARDADashboard.DashboardAPI,
                                                             Ganga.GPIDev,
                                                             Ganga.GPIDev.Adapters,
                                                             Ganga.GPIDev.Adapters.IMonitoringService,
                                                             ApMon,
                                                             ApMon.apmon,
                                                             ApMon.Logger,
                                                             ApMon.ProcInfo,
                                                             Ganga.Lib.MonitoringServices.ARDADashboard.LCG,
                                                             Ganga.Lib.MonitoringServices.ARDADashboard.LCG.ARDADashboardLCG,
                                                             ]
Ejemplo n.º 11
0
 def __init__(self, job_info, config_info):
     """Construct the GangaMon monitoring service."""
     IMonitoringService.__init__(self, job_info, config_info)
Ejemplo n.º 12
0
 def __init__(self, job_info):
     IMonitoringService.__init__(self, job_info)
Ejemplo n.º 13
0
    def __init__(self, job_info):
        IMonitoringService.__init__(self, job_info)

        if isinstance(job_info, DictionaryType):
            # we are on the worker node. We just need
            # to get values from the dictionary

            try:
                self.gangaJobId = job_info['gangaJobId']
                self.gangaTaskId = job_info['gangaTaskId']
                self.gridBackend = job_info['gridBackend']
                self.gridCertificate = job_info['gridCertificate']
                self.VO = job_info['VO']
                self._complete = True
            except KeyError as msg:
                # too bad, we will not monitor the job
                return
            # on the WN, we get the job ID from envar
            self.gridJobId = safe_getenv('EDG_WL_JOBID')
            if self.gridJobId == 'unknown':
                self.gridJobId = safe_getenv('GLITE_WL_JOBID')
                if self.gridJobId == 'unknown':
                    self._complete = False

        else:
            # we are in the client session. job_info is a Job()

            from Ganga.Utility.logging import getLogger
            self._logger = getLogger()

            job = job_info

            self.gridBackend = getattr(job, 'backend')._name
            if self.gridBackend not in ['LCG']:
                self._logger.debug('not sending monitoring because not in LCG')
                return

            self._logger.debug(job.backend)
            self._logger.debug(job.backend.id)
            self.gridJobId = job.backend.id

            # we compute the "jobID" and "taskID"
            # (which is the gangaJobId followed by the user@repository

            # the repository unique ID:
            from Ganga.Utility.Config import getConfig, ConfigError
            config = getConfig('Configuration')
            rep_type = config['repositorytype']
            rep_login = config['user']
            if 'Local' in rep_type:
                from Ganga.Runtime import Repository_runtime
                rep_dir = Repository_runtime.getLocalRoot()
                sys_config = getConfig('System')
                rep_hostname = sys_config['GANGA_HOSTNAME']
                rep_location = rep_hostname + ':' + rep_dir
            elif 'Remote' in rep_type:
                remote_config = getConfig(rep_type + "_Repository")
                rep_host = remote_config['host']
                rep_port = remote_config['port']
                rep_location = rep_host + ':' + rep_port
            else:
                return
            repository_id = rep_login + '@' + rep_location

            master_job = job.master

            if master_job is not None:
                master_id = master_job.id
                self._logger.debug('found master: %d' % master_id)
                self.gangaTaskId = self.taskPrefix + '_' + \
                    str(master_id) + '_' + repository_id
                self.gangaJobId = str(job.id)
            else:
                self.gangaTaskId = self.taskPrefix + \
                    '_' + str(job.id) + '_' + repository_id
                self.gangaJobId = '0'

            self._logger.debug('task_id = %s' % self.gangaTaskId)
            self._logger.debug('task_job_id = %s' % self.gangaJobId)

            backendConfig = getConfig(self.gridBackend)
            try:
                self.VO = backendConfig['VirtualOrganisation']
            except KeyError:
                self._logger.debug('VirtualOrganisation not configured')
                # we need it, it's too dangerous if we are not sure
                return

            from Ganga.GPIDev.Credentials import getCredential

            proxy = getCredential('GridProxy')
            self.gridCertificate = proxy.info('-subject')
            if self.gridCertificate is None:
                self._logger.debug('error: grid certificate not known')
                return

            if self.gridJobId is None:
                self._logger.debug('normal: grid job ID is None')

            self._logger.debug('job is complete')
            self._complete = True

        # we can now initialize the dashboard communication thing
        if self.gridJobId is not None:
            try:
                self.dashboard = DashboardAPI(
                    self.gangaTaskId, self.gangaJobId + '_' + self.gridJobId)
            except TypeError:
                self.dashboard = DashboardAPI(
                    self.gangaTaskId, self.gangaJobId + '_' + '_'.join(self.gridJobId))
Ejemplo n.º 14
0
    def getSandboxModules(self):
        import Ganga.Lib.MonitoringServices.OutputServerMS.OutputServerMS

        return [Ganga, Ganga.Lib, Ganga.Lib.MonitoringServices, Ganga.Lib.MonitoringServices.OutputServerMS, Ganga.Lib.MonitoringServices.OutputServerMS.OutputServerMS] + IMonitoringService.getSandboxModules(self)
Ejemplo n.º 15
0
    def getSandboxModules(self):
        print "Sending sandbox modules"
        import Ganga.Lib.MonitoringServices.Octopus.OctopusMS

        return [Ganga, Ganga.Lib, Ganga.Lib.MonitoringServices, Ganga.Lib.MonitoringServices.Octopus, Ganga.Lib.MonitoringServices.Octopus.OctopusMS, Ganga.Lib.MonitoringServices.Octopus.Octopus] + IMonitoringService.getSandboxModules(self)