Exemple #1
0
    def __init__(self):
        logger.info("crabbackend init")
        super(CRABBackend, self).__init__()
        config = Config.getConfig('CMSSW')
        shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'))
        #shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
        #              [config['CMSSW_VERSION'], config['CRAB_VERSION']])
        self.crab_env = shell.env

        config = Config.getConfig('CRAB_CFG')
        self.server_name = config['server_name']
        self.apiresource = config['apiresource']
        self.userproxy = config['userproxy']
        self.asyncdest = config['asyncdest']
        logger.info("asyncdest %s" % self.asyncdest)
Exemple #2
0
    def __init__(self):
        logger.info("crabbackend init")
        super(CRABBackend, self).__init__()
        config = Config.getConfig('CMSSW')
        shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh')) 
        #shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
        #              [config['CMSSW_VERSION'], config['CRAB_VERSION']])
        self.crab_env = shell.env

        config = Config.getConfig('CRAB_CFG')
        self.server_name = config['server_name']
        self.apiresource = config['apiresource']
        self.userproxy = config['userproxy']
        self.asyncdest = config['asyncdest']
        logger.info("asyncdest %s" % self.asyncdest )
	def run(self):
		# create connection
		self.log.info('Connecting to LGI project %s server %s'%(self.res._project, self.res._url))
		self.res.connect()
		self.queued = None
		config = Config.getConfig('LGI')

		# LGI update loop
		self.log.debug('Starting LGIResourceThread main loop')
		while not self.should_stop():
			now = time.time()

			try:
				work = [self._workForApp(app) for app in self.res.getApplications()]
				totalwork = sum(work)
				if self.log.isEnabledFor('DEBUG'):
					self.log.debug('LGI pending work: %s'%(dict(zip([str(x) for x in self.res.getApplications()], work))))
				if self.queued != totalwork:
					self.log.info('LGI jobs: %d waiting'%totalwork)
				self.queued = totalwork
			except Exception, e:
				self.log.warn(e)
		
			# and wait for next iteration
			while not self.should_stop() and time.time()-now < config['Poll']:
				time.sleep(1)
Exemple #4
0
    def submit(self, **opts):  # called on client, so job_info is Job object
        """Log submit event on client."""
        # if this job has a master and it is the first subjob then sent
        # submitted for master job
        if self.job_info.master is not None:
            if self.job_info.id == 0:
                masterjob_msg = self.getMessage('submitted')
                masterjob_msg['subjobs'] = len(self.job_info.master.subjobs)
                masterjob_msg['ganga_job_id'] = str(
                    masterjob_msg['ganga_job_id']).split('.')[0]
                # override ganga_job_uuid as the message 'from the master' is
                # really sent from the subjob
                masterjob_msg['ganga_job_uuid'] = masterjob_msg[
                    'ganga_master_uuid']
                masterjob_msg['ganga_master_uuid'] = 0
                self.send(masterjob_msg)

        from Ganga.Utility import Config
        gangausername = Config.getConfig('Configuration')['user']
        self.job_info.info.monitoring_links.append(
            ('http://gangamon.cern.ch/ganga/#user=%s' % gangausername, 'dashboard'))

        # send submitted for this job
        msg = self.getMessage('submitted')
        from Ganga.GPI import queues
        queues.add(self.send, (msg))
Exemple #5
0
def _initconfig():
    """Initialize MSGMS configuration."""
    try:
        from Ganga.Utility import Config
        # create configuration
        config = Config.makeConfig(
            'MSGMS', 'Settings for the MSGMS monitoring plugin. Cannot be changed ruding the interactive Ganga session.')
        config.addOption(
            'server', 'dashb-mb.cern.ch', 'The server to connect to')
        config.addOption('port', 61113, 'The port to connect to')
        config.addOption('username', 'ganga', '')
        config.addOption('password', 'analysis', '')
        config.addOption('message_destination', '/queue/ganga.status', '')
        config.addOption('usage_message_destination', "/queue/ganga.usage", '')
        config.addOption(
            'job_submission_message_destination', "/queue/ganga.jobsubmission", '')

        # prevent modification during the interactive ganga session
        def deny_modification(name, x):
            raise Config.ConfigError(
                'Cannot modify [MSGMS] settings (attempted %s=%s)' % (name, x))
        config.attachUserHandler(deny_modification, None)
    except ImportError:
        # on worker node so Config is not needed since it is copied to MSGMS
        # constructor
        pass
Exemple #6
0
def _initconfig():
    """Initialize DashboardMS configuration."""
    try:
        from Ganga.Utility import Config
        # create configuration
        config = Config.makeConfig(
            'DashboardMS', 'Settings for Dashboard Messaging Service.')
        config.addOption('server', 'dashb-mb.cern.ch', 'The MSG server name.')
        config.addOption('port', 61113, 'The MSG server port.')
        config.addOption('user', 'ganga-atlas', '')
        config.addOption('password', 'analysis', '')
        config.addOption('destination_job_status', '/topic/dashboard.atlas.jobStatus',
                         'The MSG destination (topic or queue) for job status messages.')
        config.addOption('destination_job_processing_attributes', '/topic/dashboard.atlas.jobProcessingAttributes',
                         'The MSG destination (topic or queue) for job processing attributes messages.')
        config.addOption('destination_job_meta', '/topic/dashboard.atlas.jobMeta',
                         'The MSG destination (topic or queue) for job meta messages.')
        config.addOption('destination_task_meta', '/topic/dashboard.atlas.taskMeta',
                         'The MSG destination (topic or queue) for task meta messages.')
        config.addOption(
            'task_type', 'analysis', 'The type of task. e.g. analysis, production, hammercloud,...')
        # prevent modification during the interactive ganga session

        def deny_modification(name, value):
            raise Config.ConfigError(
                'Cannot modify [DashboardMS] settings (attempted %s=%s)' % (name, value))
        config.attachUserHandler(deny_modification, None)
    except ImportError:
        # on worker node so Config is not needed since it is copied to
        # DashboardMS constructor
        pass
	def __init__(self):
		GangaThread.__init__(self, 'LGI_Resource')
		self.log = getLogger('LGI.Resource.Thread')
		config = Config.getConfig('LGI')
		if not os.path.exists(config['PilotDist']):
			self.log.error('cannot connect to LGI server: pilotjob tarball not found: '+config['PilotDist'])
		self.res = LGI.Resource(config['PilotDist'])
		# number of queued LGI jobs
		self.queued = None
Exemple #8
0
 def getJobInfo(self):  # called on client, so job_info is Job object
     """Create job_info from Job object."""
     if self.job_info.master is None:  # no master job; this job is not splitjob
         ganga_job_id = str(self.job_info.id)
         ganga_job_uuid = self.job_info.info.uuid
         ganga_master_uuid = 0
     else:  # there is a master job; we are in a subjob
         ganga_job_id = str(self.job_info.master.id) + \
             '.' + str(self.job_info.id)
         ganga_job_uuid = self.job_info.info.uuid
         ganga_master_uuid = self.job_info.master.info.uuid
     from Ganga.Utility import Config
     return {'ganga_job_uuid': ganga_job_uuid, 'ganga_master_uuid': ganga_master_uuid, 'ganga_user_repository': Config.getConfig('Configuration')['user']
             + '@' + Config.getConfig('System')['GANGA_HOSTNAME']
             # place-holder updated in getMessage
             # place-holder updated in getMessage
             + ':' + Config.getConfig('Configuration')['gangadir'], 'ganga_job_id': ganga_job_id, 'subjobs': len(self.job_info.subjobs), 'backend': self.job_info.backend.__class__.__name__, 'application': self.job_info.application.__class__.__name__, 'job_name': self.job_info.name, 'hostname': '', 'event': ''
             }
Exemple #9
0
 def start(self):
     config = Config.getConfig("LGI")
     if config["StatsInterval"] == 0:
         self.log.debug("Not starting LGI stats thread because [LGI]StatsInterval is zero")
         return
     if not config["StatsFile"]:
         self.log.debug("Not starting LGI stats thread because [LGI]StatsFile is empty")
         return
     if config["Enable"] is False:
         self.log.debug("Not starting LGI stats thread because [LGI]Enable is False")
         return False
     return GangaThread.start(self)
Exemple #10
0
    def prepare_job_config(self, job):
        """ Generates a CRAB config object from the Ganga job configuration. """

        from WMCore.Configuration import Configuration
        job_config = Configuration()

        for section in job.backend.CRABConfig._schema.datadict.keys():

            section_config = getattr(job.backend.CRABConfig, section)
            ganga_section_config = Config.getConfig('CRABConfig_%s' % section)
            task_section_config = job_config.section_(section)

            for parameter_name, parameter_type in section_config._schema.allItems():
                parameter_value = getattr(section_config, parameter_name)

                if parameter_value not in (None, [None]):
                    
                    # CRAB Config doesn't like Ganga sequence type instead of Lists
                    if parameter_type._meta['sequence']:
                        parameter_value = list(parameter_value)

                    task_section_config.__setattr__(parameter_name, parameter_value)

                # Updating configuration in case of Ganga inline options specified                
                ganga_option = ganga_section_config[parameter_name]
                if ganga_option:

                    # CRAB Config doesn't like Ganga sequence (or tuples) type instead of Lists 
                    # Passing sequance with Ganga inline options makes it a tuple.                   
                    if parameter_type._meta['sequence']:
                        # Ugly but we need this because otherwise tuple of 1 element with a string would
                        # be transformed in a list of chars ( ('ab') --> ['a', 'b'] )
                        import json
                        ganga_option = json.loads(json.dumps(ganga_option))
                        if type(ganga_option) != list:
                            ganga_option = [ganga_option]
                        # loads makes strings 'utf' type, CRAB3 Server wants 'str' type
                        ganga_option = map(lambda x: str(x), ganga_option)

                    task_section_config.__setattr__(parameter_name, ganga_option)

        # Some internal configuration
        job_config.General.workArea = job.outputdir 

        return job_config
Exemple #11
0
def _initconfigFeed():
    """Initialize Feedback configuration."""
    try:
        from Ganga.Utility import Config
        # create configuration
        config = Config.makeConfig(
            'Feedback', 'Settings for the Feedback plugin. Cannot be changed ruding the interactive Ganga session.')
        config.addOption(
            'uploadServer', 'http://gangamon.cern.ch/django/errorreports', 'The server to connect to')

        def deny_modification(name, x):
            raise Config.ConfigError(
                'Cannot modify [Feedback] settings (attempted %s=%s)' % (name, x))
        config.attachUserHandler(deny_modification, None)
    except ImportError:
        # on worker node so Config is not needed since it is copied to Feedback
        # constructor
        pass
Exemple #12
0
    def run(self):
        config = Config.getConfig("LGI")

        # wait for GPI to become ready (taken from GangaJEM)
        while not self.should_stop():
            try:
                from Ganga.GPI import jobs

                break
            except:
                pass
            time.sleep(1)
        from Ganga.GPI import LGI

        # LGI update loop
        self.log.debug("Starting LGI StatsThread main loop")
        self.data = []
        self._writeStats(self.data, config["StatsFile"])
        while not self.should_stop():
            now = time.time()

            try:
                # add new line of data
                lgiQueued = LGI.resource.queued
                if lgiQueued is None:
                    lgiQueued = 0
                lgiRunning = 0  # TODO
                pilotQueued = sum([len(jobs.select(status=s)) for s in ["submitted", "submitting"]])
                pilotRunning = len(jobs.select(status="running"))
                self.data.append(
                    [int(now), lgiQueued + lgiRunning, lgiRunning, pilotQueued + pilotRunning, pilotRunning]
                )
                # trash too old lines
                self.data = filter(lambda x: now - x[0] < config["StatsHistory"], self.data)
                # write data
                self._writeStats(self.data, config["StatsFile"])
            except Exception, e:
                self.log.warn(e)

                # and wait for next iteration
            while not self.should_stop() and time.time() - now < config["StatsInterval"]:
                time.sleep(1)
Exemple #13
0
    def checkReport(self,jobDoc):

        job = self.getJobObject()     

        config = Config.getConfig('Metrics')
        location = config['location']
        if not os.path.exists(location):
            raise BackendError(0,'Location %s file doesnt exist.'%(location))

        config = ConfigParser()
        config.read(location)      

        PARAMS = [('status','status')]

        if config.has_section('report'):
            PARAMS += config.items('report')
        else:
            logger.warning('No report in metrics')

        for n,v in PARAMS:
            if v:
                job.backend.report[v] = jobDoc.getAttribute(v)
Exemple #14
0
    def checkReport(self, jobDoc):

        job = self.getJobObject()

        config = Config.getConfig('Metrics')
        location = config['location']
        if not os.path.exists(location):
            raise BackendError('CRAB',
                               'Location %s file doesnt exist.' % (location))

        config = ConfigParser()
        config.read(location)

        PARAMS = [('status', 'status')]

        if config.has_section('report'):
            PARAMS += config.items('report')
        else:
            logger.warning('No report in metrics')

        for n, v in PARAMS:
            if v:
                job.backend.report[v] = jobDoc.getAttribute(v)
Exemple #15
0
    def _getDefaultValueInternal(self, attr, val=None, check=False):
        """ Get the default value of a schema item, both simple and component.
        If check is True then val is used instead of default value: this is used to check if the val may be used as a default value (e.g. if it is OK to use it as a value in the config file)
        """
        def_name = defaultConfigSectionName(self.name)

        item = self.getItem(attr)

        stored_attr_key = def_name + ':' + str(attr)

        from Ganga.Utility.Config import Config
        is_finalized = Config._after_bootstrap

        useDefVal = False

        try:
            # Attempt to get the relevant config section
            config = Config.getConfig(def_name)

            if is_finalized and stored_attr_key in _found_attrs and not config.hasModified:
                defvalue = _found_attrs[stored_attr_key]
            else:
                if attr in config.getEffectiveOptions():
                    defvalue = config[attr]
                    from Ganga.GPIDev.Base.Proxy import isProxy
                    if isProxy(defvalue):
                        raise GangaException("(1)Proxy found where it shouldn't be in the Config: %s" % stored_attr_key)
                    ## Just in case a developer puts the proxied object into the default value!
                    _found_attrs[stored_attr_key] = defvalue
                else:
                    useDefVal = True
        except (KeyError, Config.ConfigError):
            useDefVal = True

        if useDefVal:
            # hidden, protected and sequence values are not represented in config
            defvalue = item['defvalue']
            from Ganga.GPIDev.Base.Proxy import isProxy
            if isProxy(defvalue):
                raise GangaException("(2)Proxy found where is shouldn't be in the Config" % stored_attr_key)
            ## Just in case a developer puts the proxied object into the default value!
            _found_attrs[stored_attr_key] = defvalue

        # in the checking mode, use the provided value instead
        if check is True:
            defvalue = val

        if isinstance(item, ComponentItem):

            # FIXME: limited support for initializing non-empty sequences (i.e.
            # apps => ['DaVinci','Executable'] is NOT correctly initialized)

            if not item['sequence']:
                if defvalue is None:
                    if not item['load_default']:
                        assert(item['optional'])
                        return None

                # if a defvalue of a component item is an object (not string) just process it as for SimpleItems (useful for FileItems)
                # otherwise do a lookup via plugin registry

                category = item['category']

                if isinstance(defvalue, str) or defvalue is None:
                    try:
                        config = Config.getConfig(def_name)
                        has_modified = config.hasModified
                    except KeyError:
                        has_modified = False

                    if category not in _found_components or has_modified:
                        _found_components[category] = allPlugins.find(category, defvalue)
                    return _found_components[category]()

        # If needed/requested make a copy of the function elsewhwre
        return defvalue
Exemple #16
0
    def report_inner(job=None, isJob=False, isTask=False):

        userInfoDirName = "userreport"
        tempDirName = "reportsRepository"
        # job relevant info
        jobSummaryFileName = "jobsummary.txt"
        jobFullPrintFileName = "jobfullprint.txt"
        repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx"
        # task relevant info
        taskSummaryFileName = "tasksummary.txt"
        taskFullPrintFileName = "taskfullprint.txt"
        tasksRepositoryPath = "repository/$usr/LocalXML/6.0/tasks/$thousandsNumxxx"
        # user's info
        environFileName = "environ.txt"
        userConfigFileName = "userconfig.txt"
        defaultConfigFileName = "gangarc.txt"
        ipythonHistoryFileName = "ipythonhistory.txt"
        gangaLogFileName = "gangalog.txt"
        jobsListFileName = "jobslist.txt"
        tasksListFileName = "taskslist.txt"
        from Ganga.Utility import Config
        uploadFileServer = Config.getConfig('Feedback')['uploadServer']
        #uploadFileServer= "http://gangamon.cern.ch/django/errorreports/"
        #uploadFileServer= "http://ganga-ai-02.cern.ch/django/errorreports/"
        #uploadFileServer= "http://127.0.0.1:8000/errorreports"

        def printDictionary(dictionary, file=sys.stdout):
            for k, v in dictionary.iteritems():
                print('%s: %s' % (k, v), file=file)

                if k == 'PYTHONPATH':
                    global PYTHON_PATH
                    PYTHON_PATH = v

        def extractFileObjects(fileName, targetDirectoryName):
            try:
                fileToRead = open(fileName, 'r')
                try:
                    fileText = fileToRead.read()
                    import re
                    pattern = "File\(name=\'(.+?)\'"
                    matches = re.findall(pattern, fileText)

                    for fileName in matches:
                        fileName = os.path.expanduser(fileName)
                        targetFileName = os.path.join(
                            targetDirectoryName, os.path.basename(fileName))
                        shutil.copyfile(fileName, targetFileName)

                finally:
                    fileToRead.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        def writeErrorLog(errorMessage):
            try:
                fileToWrite = open(errorLogPath, 'a')
                try:
                    fileToWrite.write(errorMessage)
                    fileToWrite.write("\n")
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise
                finally:
                    fileToWrite.close()
            except Exception as err2:
                logger.debug("Err: %s" % str(err2))
                pass

        def writeStringToFile(fileName, stringToWrite):

            try:
                # uncomment this to try the error logger
                #fileName = '~/' + fileName
                fileToWrite = open(fileName, 'w')
                try:
                    fileToWrite.write(stringToWrite)
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise err
                finally:
                    fileToWrite.close()
            # except IOError:
            except Exception as err:
                logger.debug("Err2: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        def renameDataFiles(directory):

            for fileName in os.listdir(directory):
                fullFileName = os.path.join(directory, fileName)
                if os.path.isfile(fullFileName):
                    if fileName == 'data':
                        os.rename(fullFileName, fullFileName + '.txt')
                else:
                    renameDataFiles(fullFileName)

        import shutil
        import tarfile
        import tempfile
        import os

        userHomeDir = os.getenv("HOME")
        tempDir = tempfile.mkdtemp()

        errorLogPath = os.path.join(tempDir, 'reportErrorLog.txt')

        fullPathTempDir = os.path.join(tempDir, tempDirName)
        fullLogDirName = ''
        # create temp dir and specific dir for the job/user

        try:
            if not os.path.exists(fullPathTempDir):
                os.mkdir(fullPathTempDir)

            import datetime
            now = datetime.datetime.now()
            userInfoDirName = userInfoDirName + \
                now.strftime("%Y-%m-%d-%H:%M:%S")
            fullLogDirName = os.path.join(fullPathTempDir, userInfoDirName)

            # if report directory exists -> delete it's content(we would like
            # last version of the report)
            if os.path.exists(fullLogDirName):
                shutil.rmtree(fullLogDirName)

            os.mkdir(fullLogDirName)
        # except OSError:
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import os.environ in a file
        fullEnvironFileName = os.path.join(fullLogDirName, environFileName)

        try:
            inputFile = open(fullEnvironFileName, 'w')
            try:
                printDictionary(os.environ, file=inputFile)

                print('OS VERSION : ' + platform.platform(), file=inputFile)

            finally:
                inputFile.close()
        # except IOError
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import user config in a file
        userConfigFullFileName = os.path.join(
            fullLogDirName, userConfigFileName)

        try:
            inputFile = open(userConfigFullFileName, 'w')
            try:

                print("#GANGA_VERSION = %s" %
                      config.System.GANGA_VERSION, file=inputFile)

                global GANGA_VERSION
                GANGA_VERSION = config.System.GANGA_VERSION

                # this gets the default values
                # Ganga.GPIDev.Lib.Config.Config.print_config_file()

                # this should get the changed values
                for c in config:
                    print(config[c], file=inputFile)

            finally:
                inputFile.close()
        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # write gangarc - default configuration
        defaultConfigFullFileName = os.path.join(
            fullLogDirName, defaultConfigFileName)

        try:
            outputFile = open(os.path.join(userHomeDir, '.gangarc'), 'r')

            try:
                writeStringToFile(defaultConfigFullFileName, outputFile.read())
            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import ipython history in a file
        try:
            ipythonFile = open(
                os.path.join(os.environ['IPYTHONDIR'], 'history'), 'r')

            try:
                lastIPythonCommands = ipythonFile.readlines()[-20:]
                writeStringToFile(os.path.join(
                    fullLogDirName, ipythonHistoryFileName), '\n'.join(lastIPythonCommands))
                #writeStringToFile(os.path.join(fullLogDirName, ipythonHistoryFileName), ipythonFile.read())
            finally:
                ipythonFile.close()
        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import gangalog in a file
        userLogFileLocation = config["Logging"]._logfile
        userLogFileLocation = os.path.expanduser(userLogFileLocation)

        try:
            gangaLogFile = open(userLogFileLocation, 'r')
            try:
                writeStringToFile(
                    os.path.join(fullLogDirName, gangaLogFileName), gangaLogFile.read())
            finally:
                gangaLogFile.close()
        # except IOError:
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import the result of jobs command in the report
        jobsListFullFileName = os.path.join(fullLogDirName, jobsListFileName)

        try:
            outputFile = open(jobsListFullFileName, 'w')
            try:

                from Ganga.GPI import jobs
                print(jobs, file=outputFile)

            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import the result of tasks command in the report
        tasksListFullFileName = os.path.join(fullLogDirName, tasksListFileName)

        try:
            outputFile = open(tasksListFullFileName, 'w')
            try:

                from Ganga.GPI import tasks
                print(tasks, file=outputFile)

            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # save it here because we will change fullLogDirName, but we want this
        # to be the archive and to be deleted
        folderToArchive = fullLogDirName

        # import job relevant info
        if (job is not None and isJob):

            global JOB_REPORT, APPLICATION_NAME, BACKEND_NAME

            JOB_REPORT = True
            APPLICATION_NAME = getName(job.application)
            BACKEND_NAME = getName(job.backend)

            # create job folder
            jobFolder = 'job_%s' % str(job.fqid)
            fullLogDirName = os.path.join(fullLogDirName, jobFolder)
            os.mkdir(fullLogDirName)

            # import job summary in a file
            fullJobSummaryFileName = os.path.join(
                fullLogDirName, jobSummaryFileName)
            writeStringToFile(fullJobSummaryFileName, str(job))

            # import job full print in a file
            fullJobPrintFileName = os.path.join(
                fullLogDirName, jobFullPrintFileName)

            try:
                inputFile = open(fullJobPrintFileName, 'w')
                try:
                    full_print(job, inputFile)
                finally:
                    inputFile.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # extract file objects
            try:
                fileObjectsPath = os.path.join(fullLogDirName, 'fileobjects')
                os.mkdir(fileObjectsPath)
                extractFileObjects(fullJobSummaryFileName, fileObjectsPath)
            # except OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy dir of the job ->input/output and subjobs
            try:
                parentDir, currentDir = os.path.split(job.inputdir[:-1])
                workspaceDir = os.path.join(fullLogDirName, 'workspace')
                shutil.copytree(parentDir, workspaceDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy shared area of the job
            try:

                if hasattr(job.application, 'is_prepared'):
                    if job.application.is_prepared is not None and job.application.is_prepared is not True:
                        import os
                        from Ganga.Utility.Config import getConfig
                        from Ganga.Utility.files import expandfilename
                        shared_path = os.path.join(expandfilename(getConfig(
                            'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'])
                        shareddir = os.path.join(
                            shared_path, job.application.is_prepared.name)
                        if os.path.isdir(shareddir):

                            sharedAreaDir = os.path.join(
                                fullLogDirName, 'sharedarea')
                            shutil.copytree(shareddir, sharedAreaDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy repository job file
            try:
                indexFileName = str(job.id) + '.index'

                repositoryPath = repositoryPath.replace(
                    '$usr', os.getenv("USER"))

                # check if the job is subjob -> different way of forming the
                # path to the repository
                is_subjob = job.fqid.find('.') > -1

                if is_subjob:

                    jobid, subjobid = job.fqid.split(
                        '.')[0], job.fqid.split('.')[1]
                    repositoryPath = repositoryPath.replace(
                        '$thousandsNum', str(int(jobid) / 1000))
                    repositoryPath = os.path.join(repositoryPath, jobid)

                else:
                    repositoryPath = repositoryPath.replace(
                        '$thousandsNum', str(job.id / 1000))

                repositoryFullPath = os.path.join(
                    config.Configuration.gangadir, repositoryPath)
                indexFileSourcePath = os.path.join(
                    repositoryFullPath, indexFileName)
                repositoryFullPath = os.path.join(
                    repositoryFullPath, str(job.id))

                repositoryTargetPath = os.path.join(
                    fullLogDirName, 'repository', str(job.id))

                os.mkdir(os.path.join(fullLogDirName, 'repository'))

                shutil.copytree(repositoryFullPath, repositoryTargetPath)
                # data files are copied but can not be opened -> add .txt to
                # their file names
                renameDataFiles(repositoryTargetPath)

                if not is_subjob:
                    # copy .index file
                    indexFileTargetPath = os.path.join(
                        fullLogDirName, 'repository', indexFileName)
                    shutil.copyfile(indexFileSourcePath, indexFileTargetPath)

            # except OSError, IOError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        # import task relevant info
        if (job is not None and isTask):
            # job is actually a task object
            task = job
            # create task folder
            taskFolder = 'task_%s' % str(task.id)
            fullLogDirName = os.path.join(fullLogDirName, taskFolder)
            os.mkdir(fullLogDirName)

            # import task summary in a file
            fullTaskSummaryFileName = os.path.join(
                fullLogDirName, taskSummaryFileName)
            writeStringToFile(fullTaskSummaryFileName, str(task))

            # import task full print in a file
            fullTaskPrintFileName = os.path.join(
                fullLogDirName, taskFullPrintFileName)

            try:
                inputFile = open(fullTaskPrintFileName, 'w')
                try:
                    full_print(task, inputFile)
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise err
                finally:
                    inputFile.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err2: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy shared area of the task
            try:
                if len(task.transforms) > 0:
                    if hasattr(task.transforms[0], 'application') and hasattr(task.transforms[0].application, 'is_prepared'):
                        if task.transforms[0].application.is_prepared is not None and task.transforms[0].application.is_prepared is not True:
                            import os
                            from Ganga.Utility.Config import getConfig
                            from Ganga.Utility.files import expandfilename
                            shared_path = os.path.join(expandfilename(getConfig(
                                'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'])
                            shareddir = os.path.join(
                                shared_path, task.transforms[0].application.is_prepared.name)
                            if os.path.isdir(shareddir):

                                sharedAreaDir = os.path.join(
                                    fullLogDirName, 'sharedarea')
                                shutil.copytree(shareddir, sharedAreaDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy repository task file
            try:
                indexFileName = str(task.id) + '.index'

                tasksRepositoryPath = tasksRepositoryPath.replace(
                    '$usr', os.getenv("USER"))
                tasksRepositoryPath = tasksRepositoryPath.replace(
                    '$thousandsNum', str(task.id / 1000))

                repositoryFullPath = os.path.join(
                    config.Configuration.gangadir, tasksRepositoryPath)
                indexFileSourcePath = os.path.join(
                    repositoryFullPath, indexFileName)
                repositoryFullPath = os.path.join(
                    repositoryFullPath, str(task.id))

                repositoryTargetPath = os.path.join(
                    fullLogDirName, 'repository', str(task.id))

                os.mkdir(os.path.join(fullLogDirName, 'repository'))

                shutil.copytree(repositoryFullPath, repositoryTargetPath)
                # data files are copied but can not be opened -> add .txt to
                # their file names
                renameDataFiles(repositoryTargetPath)

                # copy .index file
                indexFileTargetPath = os.path.join(
                    fullLogDirName, 'repository', indexFileName)
                shutil.copyfile(indexFileSourcePath, indexFileTargetPath)

            # except OSError, IOError:
            except Exception as err:
                logger.debug("Err %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        resultArchive = '%s.tar.gz' % folderToArchive

        try:
            resultFile = tarfile.TarFile.open(resultArchive, 'w:gz')
            try:
                resultFile.add(
                    folderToArchive, arcname=os.path.basename(folderToArchive))
                # put the error log in the archive
                if(os.path.exists(errorLogPath)):
                    resultFile.add(
                        errorLogPath, arcname=os.path.basename(errorLogPath))
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                raise
            finally:
                resultFile.close()
        except Exception as err:
            logger.debug("Err2: %s" % str(err))
            raise  # pass

        # remove temp dir
        if(os.path.exists(folderToArchive)):
            shutil.rmtree(folderToArchive)

        # print the error if there is something
        if os.path.exists(errorLogPath):
            logger.error('')
            logger.error('An error occured while collecting report information : ' + open(errorLogPath, 'r').read())
            logger.error('')

        # delete the errorfile from user's pc
        if(os.path.exists(errorLogPath)):
            os.remove(errorLogPath)

        # return the path to the archive and the path to the upload server
        return (resultArchive, uploadFileServer, tempDir)
Exemple #17
0
    def _getDefaultValueInternal(self, attr, val=None, check=False):
        """ Get the default value of a schema item, both simple and component.
        If check is True then val is used instead of default value: this is used to check if the val may be used as a default value (e.g. if it is OK to use it as a value in the config file)
        """
        def_name = defaultConfigSectionName(self.name)

        item = self.getItem(attr)

        stored_option_key = def_name
        stored_attr_key = def_name + ':' + attr

        is_finalized = Config._after_bootstrap

        useDefVal = False

        try:

            # Attempt to get the relevant config section
            if is_finalized:
                if stored_option_key in _stored_configs:
                    config = _stored_configs[stored_option_key]
                else:
                    config = Config.getConfig(def_name)
                    _stored_configs[stored_option_key] = config
            else:
                config = Config.getConfig(def_name)

            if is_finalized and not config.hasModified and stored_attr_key in _found_attrs:
                defvalue = _found_attrs[stored_attr_key]
            else:
                if is_finalized:
                    if stored_option_key in _stored_options and not config.hasModified:
                        eff_ops = _stored_options[stored_option_key]
                    else:
                        eff_ops = config.getEffectiveOptions()
                        _stored_options[stored_option_key] = eff_ops
                else:
                    eff_ops = config.getEffectiveOptions()

                if attr in eff_ops:
                    defvalue = config[attr]
                    from Ganga.GPIDev.Base.Proxy import isProxy
                    if isProxy(defvalue):
                        raise GangaException("(1)Proxy found where it shouldn't be in the Config: %s" % stored_attr_key)
                    ## Just in case a developer puts the proxied object into the default value!
                    _found_attrs[stored_attr_key] = defvalue
                else:
                    useDefVal = True
        except (KeyError, Config.ConfigError):
            useDefVal = True

        if useDefVal:
            # hidden, protected and sequence values are not represented in config
            defvalue = item['defvalue']
            from Ganga.GPIDev.Base.Proxy import isProxy
            if isProxy(defvalue):
                raise GangaException("(2)Proxy found where is shouldn't be in the Config" % stored_attr_key)
            ## Just in case a developer puts the proxied object into the default value!
            _found_attrs[stored_attr_key] = defvalue

        # in the checking mode, use the provided value instead
        if check is True:
            defvalue = val

        if isinstance(item, ComponentItem):

            # FIXME: limited support for initializing non-empty sequences (i.e.
            # apps => ['DaVinci','Executable'] is NOT correctly initialized)

            if not item['sequence']:
                if defvalue is None:
                    if not item['load_default']:
                        try:
                            assert(item['optional'])
                        except AssertionError:
                            raise SchemaError("This item '%s' is not a sequence, doesn't have a load_default and is not optional. This is unsupported!" % type(item))
                        return None

                # if a defvalue of a component item is an object (not string) just process it as for SimpleItems (useful for FileItems)
                # otherwise do a lookup via plugin registry

                category = item['category']

                if isinstance(defvalue, str) or defvalue is None:
                    try:
                        config = Config.getConfig(def_name)
                        has_modified = config.hasModified
                    except KeyError:
                        has_modified = False

                    if category not in _found_components or has_modified:
                        _found_components[category] = allPlugins.find(category, defvalue)
                    return _found_components[category]()
                if isclass(defvalue):
                    return defvalue()

        # If needed/requested make a copy of the function elsewhwre
        return defvalue
Exemple #18
0
 def getConfig():
     """Return MSGMS Config object."""
     from Ganga.Utility import Config
     return Config.getConfig('MSGMS')
Exemple #19
0
#
# This is free software; you can redistribute it and/or modify it under the terms of
# the GNU General Public License as published by the Free Software Foundation.
#
# http://www.gnu.org/licenses/gpl.txt

import os
import time
import datetime
from InterpoList import InterpoList
from Ganga.Utility import Config
from Ganga.Utility.logging import getLogger
from Ganga.Core.GangaThread import GangaThread

# Module configuration
config = Config.makeConfig('LGI', 'Leiden Grid Initiative Pilot job framework settings')
config.addOption('PilotDist', 'pilotdist/pilotjob.tar.gz',
	'Pilot job resource daemon tarball, fully configured for your applications and project server')
config.addOption('PilotScript', 'pilotdist/pilotrun.sh',
	'Script to run inside pilotjob, which unpacks the tarball and executes the resource daemon')

config.addOption('SchedMin',  1, 'Minimum number of pilotjobs at all times')
config.addOption('SchedMax', 10, 'Maximum number of pilotjobs')

config.addOption('Poll', 30, 'LGI thread polling time')
config.addOption('Update', 10, 'Pilot thread update time')
config.addOption('WaitNew', 60, 'If after this many seconds there are (still) more LGI jobs than pilotjobs, spawn new pilotjobs.')
config.addOption('WaitTerm', 300, 'Terminate pilotjob after seconds of idle time')
config.addOption('MaxRuntime', None, 'Maximum run-time of pilotjobs in seconds. Leave empty to run indefinitely until batch system terminates it.')

config.addOption('StatsInterval', 0, 'Statistics logging interval, or 0 for no statistics')
Exemple #20
0
	def start(self):
		if Config.getConfig('LGI')['Enable'] is False:
			self.log.debug('Not starting LGI pilot thread because [LGI]Enable is False')
			return False
		return GangaThread.start(self)
Exemple #21
0
    def parseResults(self):

        job = self.getJobObject()   

        server = CRABServer()
        try:
            server.status(job)
            server.getOutput(job) 
        except:
            logger.error('Could not get the output of the job.')
            # Let's not raise this yet (in case of a double call).
            # raise CRABServerError('Impossible to get the output of the job')

        workdir = job.inputdata.ui_working_dir
        index = int(job.id) + 1
        doc_path = '%s/res/crab_fjr_%d.xml'%(workdir,index)

        if not os.path.exists(doc_path):
            logger.error('FJR %s not found.'%(doc_path))
            return

        try:
            doc = parse(doc_path)   
        except:
            logger.error("Could not parse document. File not present?")
            return
        status = doc.firstChild.getAttribute("Status")

        if status in ["Failed"]:
            self.postMortem(job)
            job.updateStatus('failed')
        elif status in ["Success"]:
            if job.status == 'submitting':
                job.updateStatus('submitted')
            job.updateStatus('completed')
        else:
            logger.warning("UNKNOWN PARSE STATUS: "+str(status))

        config = Config.getConfig('Metrics')
        location = config['location']
        if not os.path.exists(location):
            raise BackendError(0,'Location %s file doesnt exist.'%(location))

        config = ConfigParser()
        config.read(location)      

        #Iterate over all them
        SECTIONS = config.sections()
        if 'report' in SECTIONS:
            SECTIONS.remove('report')

        # Only five sections work here...
        for section in SECTIONS:

            if not job.backend.fjr.has_key(section):
                job.backend.fjr[section] = {}

            performancereport = doc.getElementsByTagName("PerformanceReport")[0]
            performancesummary = performancereport.getElementsByTagName("PerformanceSummary")
            for pfs in performancesummary:
                if pfs.getAttribute("Metric") == section:
                    metrics = pfs.getElementsByTagName("Metric")
                    for metric in metrics:
                        name = metric.getAttribute("Name")
                        if config.has_option(section,name):
                            # Due to the names with minus intead of underscore, we have to do thiw walkarround
                            # to send them to the DB.
                            name = config.get(section,name)
                            if name:
                                job.backend.fjr[section][name] = metric.getAttribute("Value")
Exemple #22
0
                      'saveoutput',
                      'faillimit',
                      'ignorelocality',
                      'inputdata',
                      'publishdbsurl',
                      'activity',
                      'extrajdl',
                      'scheddname',
                      'collector'
                     ]
        spec = {}
        for field in specFields:
            if getattr(job.inputdata, field) not in [None, [None]]:
                spec[field] = getattr(job.inputdata, field)

        config = Config.getConfig('TASK_CFG')
        for field in config:
            if field in specFields and config[field]:
                logger.info(field)
                logger.info(config[field])
                spec[field] = config[field]

        spec['cachefilename'] = cachefilename
        spec['cacheurl'] = 'https://cmsweb.cern.ch/crabcache'         

        if spec.has_key('userfiles'):
            # open usefiles source file and add userfiles list
            uf = open(spec['userfiles'], 'r')
            spec['userfiles'] = []
            for line in uf:
                spec['userfiles'].append(line.replace('\n', ''))
Exemple #23
0
"""Utilities for using MSG within Ganga.
"""

# default stomp.py logging to CRITICAL

import Ganga.Utility.Config as Config

config = Config.getConfig("Logging")
# test if stomp.py logging is already set
if "stomp.py" in config:
    pass  # config['stomp.py']
else:
    from Ganga.Utility.logging import getLogger, _get_logging

    # set stomp.py logger to CRITICAL
    getLogger("stomp.py").setLevel(_get_logging().CRITICAL)


def createPublisher(server, port, user="******", password="******", idle_timeout=None, exit_timeout=None):
    """Create a new publisher thread which extends GangaThread where available
    (i.e. on the client) or Thread otherwise (i.e. on the worker node).

    N.B. If GangaThread is not available then an exit handler is added, with the
    given timeout.

    @param server: The server host name.
    @param user: The user name.
    @param password: The password.
    @param logger: The logger instance.
    @param idle_timeout: Maximum seconds to idle before closing connection.
            Negative value indicates never close connection.
Exemple #24
0
 def __init__(self):
     super(CRABBackend, self).__init__()
     config = Config.getConfig('CMSSW')
     shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
                   [config['CMSSW_VERSION'], config['CRAB_VERSION']])
     self.crab_env = shell.env
Exemple #25
0
    def _getDefaultValueInternal(self, attr, val=None, check=False):
        """ Get the default value of a schema item, both simple and component.
        If check is True then val is used instead of default value: this is used to check if the val may be used as a default value (e.g. if it is OK to use it as a value in the config file)
        """
        def_name = defaultConfigSectionName(self.name)

        item = self.getItem(attr)

        stored_attr_key = def_name + ':' + str(attr)

        from Ganga.Utility.Config import Config
        is_finalized = Config._after_bootstrap

        useDefVal = False

        try:
            # Attempt to get the relevant config section
            config = Config.getConfig(def_name, create=False)

            if is_finalized and stored_attr_key in _found_attrs and not config.hasModified():
                defvalue = _found_attrs[stored_attr_key]
            else:
                if attr in config.getEffectiveOptions():
                    defvalue = config[attr]
                    from Ganga.GPIDev.Base.Proxy import isProxy
                    if isProxy(defvalue):
                        raise GangaException("(1)Proxy found where it shouldn't be in the Config: %s" % stored_attr_key)
                    ## Just in case a developer puts the proxied object into the default value!
                    _found_attrs[stored_attr_key] = defvalue
                else:
                    useDefVal = True
        except (KeyError, Config.ConfigError):
            useDefVal = True

        if useDefVal:
            # hidden, protected and sequence values are not represented in config
            defvalue = item['defvalue']
            from Ganga.GPIDev.Base.Proxy import isProxy
            if isProxy(defvalue):
                raise GangaException("(2)Proxy found where is shouldn't be in the Config" % stored_attr_key)
            ## Just in case a developer puts the proxied object into the default value!
            _found_attrs[stored_attr_key] = defvalue

        # in the checking mode, use the provided value instead
        if check is True:
            defvalue = val

        if isinstance(item, ComponentItem):

            # FIXME: limited support for initializing non-empty sequences (i.e.
            # apps => ['DaVinci','Executable'] is NOT correctly initialized)

            if not item['sequence']:
                if defvalue is None:
                    if not item['load_default']:
                        assert(item['optional'])
                        return None

                # if a defvalue of a component item is an object (not string) just process it as for SimpleItems (useful for FileItems)
                # otherwise do a lookup via plugin registry

                category = item['category']

                if isinstance(defvalue, str) or defvalue is None:
                    try:
                        config = Config.getConfig(def_name, create=False)
                        has_modified = config.hasModified()
                    except KeyError:
                        has_modified = False

                    if category not in _found_components or has_modified:
                        _found_components[category] = allPlugins.find(category, defvalue)
                    return _found_components[category]()

        # make a copy of the default value (to avoid strange effects if the
        # original modified)
        try:
            from Ganga.GPIDev.Base.Proxy import isType, getRuntimeGPIObject, stripProxy, getName
            from Ganga.GPIDev.Base.Objects import Node
            if isinstance(defvalue, Node):
                return stripProxy(getRuntimeGPIObject(getName(defvalue)))
            else:
                return copy.deepcopy(defvalue)
        except ImportError:
            return copy.deepcopy(defvalue)
Exemple #26
0
 def __init__(self):
     super(CRABBackend, self).__init__()
     config = Config.getConfig('CMSSW')
     shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
                   [config['CMSSW_VERSION'], config['CRAB_VERSION']])
     self.crab_env = shell.env
Exemple #27
0
    def report_inner(job=None, isJob=False, isTask=False):

        userInfoDirName = "userreport"
        tempDirName = "reportsRepository"
        # job relevant info
        jobSummaryFileName = "jobsummary.txt"
        jobFullPrintFileName = "jobfullprint.txt"
        repositoryPath = "repository/$usr/LocalXML/6.0/jobs/$thousandsNumxxx"
        # task relevant info
        taskSummaryFileName = "tasksummary.txt"
        taskFullPrintFileName = "taskfullprint.txt"
        tasksRepositoryPath = "repository/$usr/LocalXML/6.0/tasks/$thousandsNumxxx"
        # user's info
        environFileName = "environ.txt"
        userConfigFileName = "userconfig.txt"
        defaultConfigFileName = "gangarc.txt"
        ipythonHistoryFileName = "ipythonhistory.txt"
        gangaLogFileName = "gangalog.txt"
        jobsListFileName = "jobslist.txt"
        tasksListFileName = "taskslist.txt"
        from Ganga.Utility import Config
        uploadFileServer = Config.getConfig('Feedback')['uploadServer']
        #uploadFileServer= "http://gangamon.cern.ch/django/errorreports/"
        #uploadFileServer= "http://ganga-ai-02.cern.ch/django/errorreports/"
        #uploadFileServer= "http://127.0.0.1:8000/errorreports"

        def printDictionary(dictionary, file=sys.stdout):
            for k, v in dictionary.iteritems():
                print('%s: %s' % (k, v), file=file)

                if k == 'PYTHONPATH':
                    global PYTHON_PATH
                    PYTHON_PATH = v

        def extractFileObjects(fileName, targetDirectoryName):
            try:
                fileToRead = open(fileName, 'r')
                try:
                    fileText = fileToRead.read()
                    import re
                    pattern = "File\(name=\'(.+?)\'"
                    matches = re.findall(pattern, fileText)

                    for fileName in matches:
                        fileName = os.path.expanduser(fileName)
                        targetFileName = os.path.join(
                            targetDirectoryName, os.path.basename(fileName))
                        shutil.copyfile(fileName, targetFileName)

                finally:
                    fileToRead.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        def writeErrorLog(errorMessage):
            try:
                fileToWrite = open(errorLogPath, 'a')
                try:
                    fileToWrite.write(errorMessage)
                    fileToWrite.write("\n")
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise
                finally:
                    fileToWrite.close()
            except Exception as err2:
                logger.debug("Err: %s" % str(err2))
                pass

        def writeStringToFile(fileName, stringToWrite):

            try:
                # uncomment this to try the error logger
                #fileName = '~/' + fileName
                fileToWrite = open(fileName, 'w')
                try:
                    fileToWrite.write(stringToWrite)
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise err
                finally:
                    fileToWrite.close()
            # except IOError:
            except Exception as err:
                logger.debug("Err2: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        def renameDataFiles(directory):

            for fileName in os.listdir(directory):
                fullFileName = os.path.join(directory, fileName)
                if os.path.isfile(fullFileName):
                    if fileName == 'data':
                        os.rename(fullFileName, fullFileName + '.txt')
                else:
                    renameDataFiles(fullFileName)

        import shutil
        import tarfile
        import tempfile
        import os

        userHomeDir = os.getenv("HOME")
        tempDir = tempfile.mkdtemp()

        errorLogPath = os.path.join(tempDir, 'reportErrorLog.txt')

        fullPathTempDir = os.path.join(tempDir, tempDirName)
        fullLogDirName = ''
        # create temp dir and specific dir for the job/user

        try:
            if not os.path.exists(fullPathTempDir):
                os.mkdir(fullPathTempDir)

            import datetime
            now = datetime.datetime.now()
            userInfoDirName = userInfoDirName + \
                now.strftime("%Y-%m-%d-%H:%M:%S")
            fullLogDirName = os.path.join(fullPathTempDir, userInfoDirName)

            # if report directory exists -> delete it's content(we would like
            # last version of the report)
            if os.path.exists(fullLogDirName):
                shutil.rmtree(fullLogDirName)

            os.mkdir(fullLogDirName)
        # except OSError:
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import os.environ in a file
        fullEnvironFileName = os.path.join(fullLogDirName, environFileName)

        try:
            inputFile = open(fullEnvironFileName, 'w')
            try:
                printDictionary(os.environ, file=inputFile)

                print('OS VERSION : ' + platform.platform(), file=inputFile)

            finally:
                inputFile.close()
        # except IOError
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import user config in a file
        userConfigFullFileName = os.path.join(
            fullLogDirName, userConfigFileName)

        try:
            inputFile = open(userConfigFullFileName, 'w')
            try:

                print("#GANGA_VERSION = %s" %
                      config.System.GANGA_VERSION, file=inputFile)

                global GANGA_VERSION
                GANGA_VERSION = config.System.GANGA_VERSION

                # this gets the default values
                # Ganga.GPIDev.Lib.Config.Config.print_config_file()

                # this should get the changed values
                for c in config:
                    print(config[c], file=inputFile)

            finally:
                inputFile.close()
        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # write gangarc - default configuration
        defaultConfigFullFileName = os.path.join(
            fullLogDirName, defaultConfigFileName)

        try:
            outputFile = open(os.path.join(userHomeDir, '.gangarc'), 'r')

            try:
                writeStringToFile(defaultConfigFullFileName, outputFile.read())
            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import ipython history in a file
        try:
            ipythonFile = open(
                os.path.join(os.environ['IPYTHONDIR'], 'history'), 'r')

            try:
                lastIPythonCommands = ipythonFile.readlines()[-20:]
                writeStringToFile(os.path.join(
                    fullLogDirName, ipythonHistoryFileName), '\n'.join(lastIPythonCommands))
                #writeStringToFile(os.path.join(fullLogDirName, ipythonHistoryFileName), ipythonFile.read())
            finally:
                ipythonFile.close()
        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import gangalog in a file
        userLogFileLocation = config["Logging"]._logfile
        userLogFileLocation = os.path.expanduser(userLogFileLocation)

        try:
            gangaLogFile = open(userLogFileLocation, 'r')
            try:
                writeStringToFile(
                    os.path.join(fullLogDirName, gangaLogFileName), gangaLogFile.read())
            finally:
                gangaLogFile.close()
        # except IOError:
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import the result of jobs command in the report
        jobsListFullFileName = os.path.join(fullLogDirName, jobsListFileName)

        try:
            outputFile = open(jobsListFullFileName, 'w')
            try:

                from Ganga.GPI import jobs
                print(jobs, file=outputFile)

            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # import the result of tasks command in the report
        tasksListFullFileName = os.path.join(fullLogDirName, tasksListFileName)

        try:
            outputFile = open(tasksListFullFileName, 'w')
            try:

                from Ganga.GPI import tasks
                print(tasks, file=outputFile)

            finally:
                outputFile.close()

        # except IOError does not catch the exception ???
        except Exception as err:
            logger.debug("Err: %s" % str(err))
            writeErrorLog(str(sys.exc_info()[1]))

        # save it here because we will change fullLogDirName, but we want this
        # to be the archive and to be deleted
        folderToArchive = fullLogDirName

        # import job relevant info
        if (job is not None and isJob):

            global JOB_REPORT, APPLICATION_NAME, BACKEND_NAME

            JOB_REPORT = True
            APPLICATION_NAME = job.application.__class__.__name__
            BACKEND_NAME = job.backend.__class__.__name__

            # create job folder
            jobFolder = 'job_%s' % str(job.fqid)
            fullLogDirName = os.path.join(fullLogDirName, jobFolder)
            os.mkdir(fullLogDirName)

            # import job summary in a file
            fullJobSummaryFileName = os.path.join(
                fullLogDirName, jobSummaryFileName)
            writeStringToFile(fullJobSummaryFileName, str(job))

            # import job full print in a file
            fullJobPrintFileName = os.path.join(
                fullLogDirName, jobFullPrintFileName)

            try:
                inputFile = open(fullJobPrintFileName, 'w')
                try:
                    full_print(job, inputFile)
                finally:
                    inputFile.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # extract file objects
            try:
                fileObjectsPath = os.path.join(fullLogDirName, 'fileobjects')
                os.mkdir(fileObjectsPath)
                extractFileObjects(fullJobSummaryFileName, fileObjectsPath)
            # except OSError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy dir of the job ->input/output and subjobs
            try:
                parentDir, currentDir = os.path.split(job.inputdir[:-1])
                workspaceDir = os.path.join(fullLogDirName, 'workspace')
                shutil.copytree(parentDir, workspaceDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy shared area of the job
            try:

                if hasattr(job.application, 'is_prepared'):
                    if job.application.is_prepared is not None and job.application.is_prepared is not True:
                        import os
                        from Ganga.Utility.Config import getConfig
                        from Ganga.Utility.files import expandfilename
                        shared_path = os.path.join(expandfilename(getConfig(
                            'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'])
                        shareddir = os.path.join(
                            shared_path, job.application.is_prepared.name)
                        if os.path.isdir(shareddir):

                            sharedAreaDir = os.path.join(
                                fullLogDirName, 'sharedarea')
                            shutil.copytree(shareddir, sharedAreaDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy repository job file
            try:
                indexFileName = str(job.id) + '.index'

                repositoryPath = repositoryPath.replace(
                    '$usr', os.getenv("USER"))

                # check if the job is subjob -> different way of forming the
                # path to the repository
                is_subjob = job.fqid.find('.') > -1

                if is_subjob:

                    jobid, subjobid = job.fqid.split(
                        '.')[0], job.fqid.split('.')[1]
                    repositoryPath = repositoryPath.replace(
                        '$thousandsNum', str(int(jobid) / 1000))
                    repositoryPath = os.path.join(repositoryPath, jobid)

                else:
                    repositoryPath = repositoryPath.replace(
                        '$thousandsNum', str(job.id / 1000))

                repositoryFullPath = os.path.join(
                    config.Configuration.gangadir, repositoryPath)
                indexFileSourcePath = os.path.join(
                    repositoryFullPath, indexFileName)
                repositoryFullPath = os.path.join(
                    repositoryFullPath, str(job.id))

                repositoryTargetPath = os.path.join(
                    fullLogDirName, 'repository', str(job.id))

                os.mkdir(os.path.join(fullLogDirName, 'repository'))

                shutil.copytree(repositoryFullPath, repositoryTargetPath)
                # data files are copied but can not be opened -> add .txt to
                # their file names
                renameDataFiles(repositoryTargetPath)

                if not is_subjob:
                    # copy .index file
                    indexFileTargetPath = os.path.join(
                        fullLogDirName, 'repository', indexFileName)
                    shutil.copyfile(indexFileSourcePath, indexFileTargetPath)

            # except OSError, IOError:
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        # import task relevant info
        if (job is not None and isTask):
            # job is actually a task object
            task = job
            # create task folder
            taskFolder = 'task_%s' % str(task.id)
            fullLogDirName = os.path.join(fullLogDirName, taskFolder)
            os.mkdir(fullLogDirName)

            # import task summary in a file
            fullTaskSummaryFileName = os.path.join(
                fullLogDirName, taskSummaryFileName)
            writeStringToFile(fullTaskSummaryFileName, str(task))

            # import task full print in a file
            fullTaskPrintFileName = os.path.join(
                fullLogDirName, taskFullPrintFileName)

            try:
                inputFile = open(fullTaskPrintFileName, 'w')
                try:
                    full_print(task, inputFile)
                except Exception as err:
                    logger.debug("Err: %s" % str(err))
                    raise err
                finally:
                    inputFile.close()
            # except IOError, OSError:
            except Exception as err:
                logger.debug("Err2: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy shared area of the task
            try:
                if len(task.transforms) > 0:
                    if hasattr(task.transforms[0], 'application') and hasattr(task.transforms[0].application, 'is_prepared'):
                        if task.transforms[0].application.is_prepared is not None and task.transforms[0].application.is_prepared is not True:
                            import os
                            from Ganga.Utility.Config import getConfig
                            from Ganga.Utility.files import expandfilename
                            shared_path = os.path.join(expandfilename(getConfig(
                                'Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'])
                            shareddir = os.path.join(
                                shared_path, task.transforms[0].application.is_prepared.name)
                            if os.path.isdir(shareddir):

                                sharedAreaDir = os.path.join(
                                    fullLogDirName, 'sharedarea')
                                shutil.copytree(shareddir, sharedAreaDir)
            # except IOError, OSError
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

            # copy repository task file
            try:
                indexFileName = str(task.id) + '.index'

                tasksRepositoryPath = tasksRepositoryPath.replace(
                    '$usr', os.getenv("USER"))
                tasksRepositoryPath = tasksRepositoryPath.replace(
                    '$thousandsNum', str(task.id / 1000))

                repositoryFullPath = os.path.join(
                    config.Configuration.gangadir, tasksRepositoryPath)
                indexFileSourcePath = os.path.join(
                    repositoryFullPath, indexFileName)
                repositoryFullPath = os.path.join(
                    repositoryFullPath, str(task.id))

                repositoryTargetPath = os.path.join(
                    fullLogDirName, 'repository', str(task.id))

                os.mkdir(os.path.join(fullLogDirName, 'repository'))

                shutil.copytree(repositoryFullPath, repositoryTargetPath)
                # data files are copied but can not be opened -> add .txt to
                # their file names
                renameDataFiles(repositoryTargetPath)

                # copy .index file
                indexFileTargetPath = os.path.join(
                    fullLogDirName, 'repository', indexFileName)
                shutil.copyfile(indexFileSourcePath, indexFileTargetPath)

            # except OSError, IOError:
            except Exception as err:
                logger.debug("Err %s" % str(err))
                writeErrorLog(str(sys.exc_info()[1]))

        resultArchive = '%s.tar.gz' % folderToArchive

        try:
            resultFile = tarfile.TarFile.open(resultArchive, 'w:gz')
            try:
                resultFile.add(
                    folderToArchive, arcname=os.path.basename(folderToArchive))
                # put the error log in the archive
                if(os.path.exists(errorLogPath)):
                    resultFile.add(
                        errorLogPath, arcname=os.path.basename(errorLogPath))
            except Exception as err:
                logger.debug("Err: %s" % str(err))
                raise
            finally:
                resultFile.close()
        except Exception as err:
            logger.debug("Err2: %s" % str(err))
            raise  # pass

        # remove temp dir
        if(os.path.exists(folderToArchive)):
            shutil.rmtree(folderToArchive)

        # print the error if there is something
        if os.path.exists(errorLogPath):
            logger.error('')
            logger.error('An error occured while collecting report information : ' + open(errorLogPath, 'r').read())
            logger.error('')

        # delete the errorfile from user's pc
        if(os.path.exists(errorLogPath)):
            os.remove(errorLogPath)

        # return the path to the archive and the path to the upload server
        return (resultArchive, uploadFileServer, tempDir)
Exemple #28
0
"""Utilities for using MSG within Ganga.
"""

# default stomp.py logging to CRITICAL

import Ganga.Utility.Config as Config

config = Config.getConfig('Logging')
# test if stomp.py logging is already set
if 'stomp.py' in config:
    pass  # config['stomp.py']
else:
    from Ganga.Utility.logging import getLogger
    import logging
    # set stomp.py logger to CRITICAL
    getLogger('stomp.py').setLevel(logging.CRITICAL)


def createPublisher(server, port, user='******', password='******', idle_timeout=None, exit_timeout=None):
    """Create a new publisher thread which extends GangaThread where available
    (i.e. on the client) or Thread otherwise (i.e. on the worker node).

    N.B. If GangaThread is not available then an exit handler is added, with the
    given timeout.

    @param server: The server host name.
    @param user: The user name.
    @param password: The password.
    @param logger: The logger instance.
    @param idle_timeout: Maximum seconds to idle before closing connection.
            Negative value indicates never close connection.
Exemple #29
0
    def parseResults(self):

        job = self.getJobObject()

        server = CRABServer()
        try:
            server.status(job)
            server.getOutput(job)
        except:
            logger.error('Could not get the output of the job.')
            # Let's not raise this yet (in case of a double call).
            # raise CRABServerError('Impossible to get the output of the job')

        workdir = job.inputdata.ui_working_dir
        index = int(job.id) + 1
        doc_path = '%s/res/crab_fjr_%d.xml' % (workdir, index)

        if not os.path.exists(doc_path):
            logger.error('FJR %s not found.' % (doc_path))
            return

        try:
            doc = parse(doc_path)
        except:
            logger.error("Could not parse document. File not present?")
            return
        status = doc.firstChild.getAttribute("Status")

        if status in ["Failed"]:
            self.postMortem(job)
            job.updateStatus('failed')
        elif status in ["Success"]:
            if job.status == 'submitting':
                job.updateStatus('submitted')
            job.updateStatus('completed')
        else:
            logger.warning("UNKNOWN PARSE STATUS: " + str(status))

        config = Config.getConfig('Metrics')
        location = config['location']
        if not os.path.exists(location):
            raise BackendError(0,
                               'Location %s file doesnt exist.' % (location))

        config = ConfigParser()
        config.read(location)

        #Iterate over all them
        SECTIONS = config.sections()
        if 'report' in SECTIONS:
            SECTIONS.remove('report')

        # Only five sections work here...
        for section in SECTIONS:

            if not job.backend.fjr.has_key(section):
                job.backend.fjr[section] = {}

            performancereport = doc.getElementsByTagName(
                "PerformanceReport")[0]
            performancesummary = performancereport.getElementsByTagName(
                "PerformanceSummary")
            for pfs in performancesummary:
                if pfs.getAttribute("Metric") == section:
                    metrics = pfs.getElementsByTagName("Metric")
                    for metric in metrics:
                        name = metric.getAttribute("Name")
                        if config.has_option(section, name):
                            # Due to the names with minus intead of underscore, we have to do thiw walkarround
                            # to send them to the DB.
                            name = config.get(section, name)
                            if name:
                                job.backend.fjr[section][
                                    name] = metric.getAttribute("Value")
Exemple #30
0
 def getConfig():
     """Return DashboardMS Config object."""
     from Ganga.Utility import Config
     return Config.getConfig('DashboardMS')