Ejemplo n.º 1
0
    def __init__(self, msg):
        """
        __init__

        Initialize the error, just accepts a message without error code
        """
        WMException.__init__(self, msg)
        return
Ejemplo n.º 2
0
    def __init__(self, msg):
        """
        __init__

        Initialize the error, just accepts a message without error code
        """
        WMException.__init__(self, msg)
        return
Ejemplo n.º 3
0
    def testExceptionUnicode0(self):
        """
        create an exception with non-ascii chars in message and test WMException.addInfo().
        """

        exception = WMException(
            "an exception message with nr. 100 and some non-ascii characters: ₩♏ℭ☺яε",
            100)
        exception.addInfo(**self.test_data)
        self.logger.debug("XML version of exception: %s", exception.xml())
        self.logger.debug("String version of exception: %s", str(exception))
Ejemplo n.º 4
0
    def testException(self):
        """
        create an exception and do some tests.
        """

        exception = WMException("an exception message with nr. 100", 100)
        self.logger.debug("String version of exception: " + str(exception))
        self.logger.debug("XML version of exception: " + exception.xml())
        self.logger.debug("Adding data")
        data = {}
        data['key1'] = 'value1'
        data['key2'] = 'data2'
        exception.addInfo(**data)
        self.logger.debug("String version of exception: " + str(exception))
Ejemplo n.º 5
0
    def testException(self):
        """
        create an exception and do some tests (only ascii chars)
        """

        exception = WMException("an exception message with nr. 100", 100)
        self.logger.debug("String version of exception: %s", str(exception))
        self.logger.debug("XML version of exception: %s", exception.xml())
        self.logger.debug("Adding data")
        data = {}
        data['key1'] = 'value1'
        data['key2'] = 3.14159
        exception.addInfo(**data)
        self.logger.debug("String version of exception: %s", str(exception))
Ejemplo n.º 6
0
    def testExceptionUnicode1(self):
        """
        create an exception with non-ascii chars in message and test WMException constructor
        """

        exception = WMException(
            "an exception message with nr. 100 and some non-ascii characters: ₩♏ℭ☺яε",
            100, **self.test_data)
        self.logger.debug("XML version of exception: %s", exception.xml())
        self.logger.debug("String version of exception: %s", str(exception))
        self.logger.debug("exception.__str__(): %s", type(exception.__str__(
        )))  # from py2 interpreter: <class 'future.types.newbytes.newbytes'>
        self.logger.debug("str(exception): %s", type(
            str(exception)))  # <class 'future.types.newstr.newstr'>
Ejemplo n.º 7
0
    def testException(self):
        """
        create an exception and do some tests.
        """

        exception = WMException("an exception message with nr. 100", 100)
        self.logger.debug("String version of exception: " + str(exception))
        self.logger.debug("XML version of exception: " + exception.xml())
        self.logger.debug("Adding data")
        data = {}
        data['key1'] = 'value1'
        data['key2'] = 'data2'
        exception.addInfo(**data)
        self.logger.debug("String version of exception: "+ str(exception))
Ejemplo n.º 8
0
    def testExceptionUnicode(self):
        """
        create an exception with non-ascii characters and do some tests.
        """

        exception = WMException(
            "an exception message with nr. 100 and some non-ascii characters: ₩♏ℭ☺яε",
            100)
        self.logger.debug("String version of exception: " + str(exception))
        self.logger.debug("XML version of exception: " + exception.xml())
        self.logger.debug("Adding data")
        data = {}
        data['key1'] = 'value1'
        data['key2'] = 'data2'
        exception.addInfo(**data)
        self.logger.debug("String version of exception: " + str(exception))
Ejemplo n.º 9
0
    def execute(self, conn=None, transaction=False):
        """
        _execute_
        
        Generic method to create tables and constraints by execute
        sql statements in the create, and constraints dictionaries.
        
        Before execution the keys assigned to the tables in the self.create
        dictionary are sorted, to offer the possibilitiy of executing 
        table creation in a certain order.
        """
        # get the keys for the table mapping:
        tableKeys = self.create.keys()
        tableKeys.sort()

        for i in tableKeys:
            try:
                self.dbi.processData(self.create[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception, e:

                msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
                                  str(self.create[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCore-2')

            keys = self.constraints.keys()
Ejemplo n.º 10
0
def parseLFN(candidate):
    """
    _parseLFN_

    Take an LFN, return the component parts
    """
    separator = "/"

    # First, make sure what we've gotten is a real LFN
    lfn(candidate)

    parts = candidate.split('/')
    final = {}

    if parts[0] == '':
        parts.remove('')
    if 'user' in parts[1:3] or 'group' in parts[1:3]:
        if parts[1] in ['user', 'group']:
            final['baseLocation'] = '/%s' % separator.join(parts[:2])
            parts = parts[2:]
        else:
            final['baseLocation'] = '/%s' % separator.join(parts[:3])
            parts = parts[3:]

        final['hnName'] = parts[0]
        final['primaryDataset'] = parts[1]
        final['secondaryDataset'] = parts[2]
        final['processingVersion'] = parts[3]
        final['lfnCounter'] = parts[4]
        final['filename'] = parts[5]

        return final

    if len(parts) == 8:
        # Then we have only two locations
        final['baseLocation'] = '/%s' % separator.join(parts[:2])
        parts = parts[2:]
    elif len(parts) == 9:
        final['baseLocation'] = '/%s' % separator.join(parts[:3])
        parts = parts[3:]
    else:
        # How did we end up here?
        # Something just went wrong
        msg = """CRITICAL!  This machine has experienced a complete logic failure while parsing LFNs.\n
        If you are a developer this indicates that you have changed the Lexicon LFN regexp functions without changing the parsing.\n
        If you are an operator, this indicates that this machine is likely unstable.\n
        All data should be backed up and the machine removed from production for examination.\n"""
        msg += "Candidate: %s" % candidate
        raise WMException(msg)

    final['acquisitionEra'] = parts[0]
    final['primaryDataset'] = parts[1]
    final['dataTier'] = parts[2]
    final['processingVersion'] = parts[3]
    final['lfnCounter'] = parts[4]
    final['filename'] = parts[5]

    return final
Ejemplo n.º 11
0
    def __init__(self, dict={}):
        try:
            Service.__init__(self, dict)
            self["requests"] = SSLRequests(self["requests"]["host"])

        except WMException, ex:
            msg = str(ex)
            self["logger"].exception(msg)
            raise WMException(msg)
Ejemplo n.º 12
0
class DBCreator(DBFormatter):
    """
    _DBCreator_

    Generic class for creating database tables.

    """
    def __init__(self, logger, dbinterface):
        """
        _init_

        Call the constructor of the parent class and create empty dictionaries
        to hold table create statements, constraint statements and insert
        statements.
        """
        DBFormatter.__init__(self, logger, dbinterface)
        self.create = {}
        self.constraints = {}
        self.inserts = {}
        self.indexes = {}

    def execute(self, conn=None, transaction=False):
        """
        _execute_

        Generic method to create tables and constraints by execute
        sql statements in the create, and constraints dictionaries.

        Before execution the keys assigned to the tables in the self.create
        dictionary are sorted, to offer the possibilitiy of executing
        table creation in a certain order.

        """
        # create tables
        for i in sorted(self.create.keys()):
            try:
                self.dbi.processData(self.create[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception, e:
                msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
                                  str(self.create[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCore-2')

        # create indexes
        for i in self.indexes.keys():
            try:
                self.dbi.processData(self.indexes[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception, e:
                msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
                                  str(self.indexes[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCore-2')
Ejemplo n.º 13
0
    def __init__(self, config, compName = None):
        """
        init

        The constructor is empty as we have an initalization method
        that can be called inside new threads (we use thread local attributes
        at startup.

        Default intialization of the harness including setting some diagnostic
        messages
        """
        self.config = config

        # component name is always the class name of child class
        if not compName:
            compName = self.__class__.__name__

        if not compName in (self.config.listComponents_() + self.config.listWebapps_()):
            raise WMException(WMEXCEPTION['WMCORE-8']+compName, 'WMCORE-8')
        if not hasattr(self.config, "Agent"):
            self.config.section_("Agent")

        self.config.Agent.componentName = compName
        compSect = getattr(self.config, compName, None)
        if compSect == None:
            # Then we have a major problem - there's no section with this name
            logging.error("Could not find section %s in config" % compName)
            logging.error("We are returning, and hoping you know what you're doing!")
            logging.debug("Config: %s" % self.config)
            return
        # check if componentDir is set if not assign.
        if getattr(compSect, 'componentDir', None) == None:
            if not hasattr(self.config, "General"):
                # Don't do anything.  Assume the user knows what they are doing.
                logging.error("Missing componentDir and General section in config")
                logging.error("Going to trust you to know what you're doing.")
                return

            compSect.componentDir =  os.path.join(self.config.General.workDir,
                                                  'Components',
                                                  self.config.Agent.componentName)
        # we have name and location of the log files. Now make sure there
        # is a directory.
        try:
            if not os.path.isdir(compSect.componentDir):
                os.makedirs(compSect.componentDir)
        except Exception as ex:
            logging.error("Encountered exception while making componentDirs: %s" % str(ex))
            logging.error("Ignoring")

        self.threadManagerName = ''
        self.heartbeatAPI      = None
        self.messages          = {}
        self.logMsg            = {}

        return
Ejemplo n.º 14
0
    def getKeyCert(self):
        """
       _getKeyCert_

       Get the user credentials if they exist, otherwise throw an exception.
       This code was modified from DBSAPI/dbsHttpService.py
        """
        cert = None
        key = None
        # Zeroth case is if the class has over ridden the key/cert and has it
        # stored in self
        if self.has_key('cert') and self.has_key('key' ) \
             and self['cert'] and self['key']:
            key = self['key']
            cert = self['cert']

        # Now we're trying to guess what the right cert/key combo is...
        # First preference to HOST Certificate, This is how it set in Tier0
        elif os.environ.has_key('X509_HOST_CERT'):
            cert = os.environ['X509_HOST_CERT']
            key = os.environ['X509_HOST_KEY']
        # Second preference to User Proxy, very common
        elif (os.environ.has_key('X509_USER_PROXY')) and \
                (os.path.exists( os.environ['X509_USER_PROXY'])):
            cert = os.environ['X509_USER_PROXY']
            key = cert

        # Third preference to User Cert/Proxy combinition
        elif os.environ.has_key('X509_USER_CERT'):
            cert = os.environ['X509_USER_CERT']
            key = os.environ['X509_USER_KEY']

        # TODO: only in linux, unix case, add other os case
        # look for proxy at default location /tmp/x509up_u$uid
        elif os.path.exists('/tmp/x509up_u' + str(os.getuid())):
            cert = '/tmp/x509up_u' + str(os.getuid())
            key = cert

        # if interactive we can use an encrypted certificate
        elif sys.stdin.isatty():
            if os.path.exists(os.environ['HOME'] + '/.globus/usercert.pem'):
                cert = os.environ['HOME'] + '/.globus/usercert.pem'
                if os.path.exists(os.environ['HOME'] + '/.globus/userkey.pem'):
                    key = os.environ['HOME'] + '/.globus/userkey.pem'
                else:
                    key = cert

        #Set but not found
        if key and cert:
            if not os.path.exists(cert) or not os.path.exists(key):
                raise WMException(
                    'Request requires a host certificate and key', "WMCORE-11")

        # All looks OK, still doesn't guarantee proxy's validity etc.
        return key, cert
Ejemplo n.º 15
0
    def __init__(self, dict={}):
        #The following should read the configuration class
        for a in ['logger', 'endpoint']:
            assert a in dict.keys(), "Can't have a service without a %s" % a

        scheme = ''
        netloc = ''
        path = ''

        # then split the endpoint into netloc and basepath
        endpoint = urlparse(dict['endpoint'])

        try:
            #Only works on python 2.5 or above
            scheme = endpoint.scheme
            netloc = endpoint.netloc
            path = endpoint.path
        except AttributeError:
            scheme, netloc, path = endpoint[:3]

        #set up defaults
        self.setdefault("inputdata", {})
        self.setdefault("cachepath", '/tmp')
        self.setdefault("cacheduration", 0.5)
        self.setdefault("maxcachereuse", 24.0)
        self.supportVerbList = ('GET', 'POST', 'PUT', 'DELETE')
        # this value should be only set when whole service class uses
        # the same verb ('GET', 'POST', 'PUT', 'DELETE')
        self.setdefault("method", None)

        #Set a timeout for the socket
        self.setdefault("timeout", 30)

        # then update with the incoming dict
        self.update(dict)

        # Get the request class, to instatiate later
        # Is this a secure service - add other schemes as we need them
        if self.get('secure', False) or scheme in SECURE_SERVICES:
            # only bring in ssl stuff if we really need it
            from WMCore.Services.Requests import SecureRequests
            requests = SecureRequests
        else:
            requests = Requests

        try:
            if path and not path.endswith('/'):
                path += '/'
            self.setdefault("basepath", path)
            # Instantiate a Request
            self.setdefault("requests", requests(netloc, dict))
        except WMException, ex:
            msg = str(ex)
            self["logger"].exception(msg)
            raise WMException(msg)
Ejemplo n.º 16
0
    def execute(self, conn=None, transaction=None):
        """
        _execute_

        Check to make sure that all required tables have been defined.  If
        everything is in place have the DBCreator make everything.
        """
        for requiredTable in self.requiredTables:
            if requiredTable not in self.create.keys():
                raise WMException("The table '%s' is not defined." % \
                                  requiredTable, "WMCORE-2")

        DBCreator.execute(self, conn, transaction)
        return True
Ejemplo n.º 17
0
    def getKeyCert(self):
        """
       _getKeyCert_

       Get the user credentials if they exist, otherwise throw an exception.
       This code was modified from DBSAPI/dbsHttpService.py
        """

        # Zeroth case is if the class has over ridden the key/cert and has it
        # stored in self
        if self['cert'] and self['key']:
            key = self['key']
            cert = self['cert']
        else:
            key, cert = getKeyCertFromEnv()

        # Set but not found
        if key is None or cert is None:
            raise WMException('Request requires a host certificate and key',
                              "WMCORE-11")

        # All looks OK, still doesn't guarantee proxy's validity etc.
        return key, cert
Ejemplo n.º 18
0
 def __init__(self):
     WMException.__init__(self, "McM responded correctly but has no data")
Ejemplo n.º 19
0
    def __init__(self, message, **data):
        WMException.__init__(self, message, **data)

        self.data.setdefault("ErrorCode", 60311)
        self.data.setdefault("ErrorType", ErrorDefinitions[60311])
Ejemplo n.º 20
0
 def __init__(self, message):
     WMException.__init__(self, message, 'WMCORE-13')
Ejemplo n.º 21
0
 def __init__(self, message):
     WMException.__init__(self, message, 'WMCORE-13')
Ejemplo n.º 22
0
    def __init__(self, message, **data):
        WMException.__init__(self, message, **data)

        self.data.setdefault("ErrorCode", 60311)
        self.data.setdefault("ErrorType", ErrorDefinitions[60311])
Ejemplo n.º 23
0
 def __init__(self):
     WMException.__init__(self, 'McM responded correctly but has no data')
Ejemplo n.º 24
0
 def __init__(self, code, name, detail):
     WMException.__init__(self, detail, code)
     self.code = code
     self.name = name
     self.detail = detail
Ejemplo n.º 25
0
                msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
                                  str(self.indexes[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCore-2')

        # set constraints
        for i in self.constraints.keys():
            try:
                self.dbi.processData(self.constraints[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception, e:
                msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
                                  str(self.constraints[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCore-2')

        # insert permanent data
        for i in self.inserts.keys():
            try:
                self.dbi.processData(self.inserts[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception, e:
                msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
                                  str(self.inserts[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCore-2')

        return True
Ejemplo n.º 26
0
    def __init__(self, config):

        self.config = config

        BasePlugin.__init__(self, config)

        self.locationDict = {}

        myThread = threading.currentThread()
        daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
                                dbinterface = myThread.dbi)
        self.locationAction = daoFactory(classname = "Locations.GetSiteInfo")


        self.packageDir = None

        if os.path.exists(os.path.join(getWMBASE(),
                                       'src/python/WMCore/WMRuntime/Unpacker.py')):
            self.unpacker = os.path.join(getWMBASE(),
                                         'src/python/WMCore/WMRuntime/Unpacker.py')
        else:
            self.unpacker = os.path.join(getWMBASE(),
                                         'WMCore/WMRuntime/Unpacker.py')

        self.agent         = getattr(config.Agent, 'agentName', 'WMAgent')
        self.sandbox       = None
        self.scriptFile    = None
        self.submitDir     = None
        self.removeTime    = getattr(config.BossAir, 'removeTime', 60)
        self.useGSite      = getattr(config.BossAir, 'useGLIDEINSites', False)
        self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False)
        self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10)
        self.errorCount    = 0
        self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
        self.maxTaskPriority     = getattr(config.BossAir, 'maxTaskPriority', 1e7)

        # Required for global pool accounting
        self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
        self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops")

        # Build ourselves a pool
        self.pool     = []
        self.input    = None
        self.result   = None
        self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4)

        # Set up my proxy and glexec stuff
        self.setupScript = getattr(config.BossAir, 'UISetupScript', None)
        self.proxy       = None
        self.serverCert  = getattr(config.BossAir, 'delegatedServerCert', None)
        self.serverKey   = getattr(config.BossAir, 'delegatedServerKey', None)
        self.myproxySrv  = getattr(config.BossAir, 'myproxyServer', None)
        self.proxyDir    = getattr(config.BossAir, 'proxyDir', '/tmp/')
        self.serverHash  = getattr(config.BossAir, 'delegatedServerHash', None)
        self.glexecPath  = getattr(config.BossAir, 'glexecPath', None)
        self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None)
        self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None)
        self.jdlProxyFile    = None # Proxy name to put in JDL (owned by submit user)
        self.glexecProxyFile = None # Copy of same file owned by submit user

        if self.glexecPath:
            if not (self.myproxySrv and self.proxyDir):
                raise WMException('glexec requires myproxyServer and proxyDir to be set.')
        if self.myproxySrv:
            if not (self.serverCert and self.serverKey):
                raise WMException('MyProxy server requires serverCert and serverKey to be set.')

        # Make the directory for the proxies
        if self.proxyDir and not os.path.exists(self.proxyDir):
            logging.debug("proxyDir not found: creating it.")
            try:
                os.makedirs(self.proxyDir, 0o1777)
            except Exception as ex:
                msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex)
                raise BossAirPluginException(msg)
        elif not os.path.isdir(self.proxyDir):
            msg = "Error: proxyDir '%s' is not a directory" % self.proxyDir
            raise BossAirPluginException(msg)

        if self.serverCert and self.serverKey and self.myproxySrv:
            self.proxy = self.setupMyProxy()

        # Build a request string
        self.reqStr = "(Memory >= 1 && OpSys == \"LINUX\" ) && (Arch == \"INTEL\" || Arch == \"X86_64\") && stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) && ((REQUIRED_OS==\"any\") || (GLIDEIN_REQUIRED_OS==REQUIRED_OS))"
        if hasattr(config.BossAir, 'condorRequirementsString'):
            self.reqStr = config.BossAir.condorRequirementsString

        return
Ejemplo n.º 27
0
 def __init__(self, code, name, detail):
     WMException.__init__(self, detail, code)
     self.code = code
     self.name = name
     self.detail = detail
Ejemplo n.º 28
0
class LogArchive(Executor):
    """
    _LogArchive_

    Execute a LogArchive Step

    """

    def pre(self, emulator = None):
        """
        _pre_

        Pre execution checks
        """

        #Are we using an emulator?
        if (emulator != None):
            return emulator.emulatePre( self.step )

        logging.info("Steps.Executors.LogArchive.pre called")
        return None

    def execute(self, emulator = None, **overrides):
        """
        _execute_


        """
        # Are we using emulators again?
        if (emulator != None):
            return emulator.emulate( self.step, self.job )

        overrides = {}
        if hasattr(self.step, 'override'):
            overrides = self.step.override.dictionary_()

        # Find alternate stageout location
        self.altLFN = overrides.get('altLFN', None)

        logging.info("Beginning Steps.Executors.LogArchive.Execute")
        logging.info("Using the following overrides: %s " % overrides)
        logging.info("Step is: %s" % self.step)
        # Wait timout for stageOut
        waitTime = overrides.get('waitTime', 3600 + (self.step.retryDelay * self.step.retryCount))

        matchFiles = [
            ".log$",
            "FrameworkJobReport",
            "Report.pkl",
            "Report.pcl",
            "^PSet.py$",
            "^PSet.pkl$"
            ]

        #Okay, we need a stageOut Manager
        useNewStageOutCode = False
        if getattr(self.step, 'newStageout', False) or \
            (overrides.has_key('newStageOut') and overrides.get('newStageOut')):
            useNewStageOutCode = True
        if not useNewStageOutCode:
            # old style
            manager = StageOutMgr.StageOutMgr(**overrides)
            manager.numberOfRetries = self.step.retryCount
            manager.retryPauseTime  = self.step.retryDelay
        else:
            # new style
            logging.info("LOGARCHIVE IS USING NEW STAGEOUT CODE")
            manager = WMCore.Storage.FileManager.StageOutMgr(
                                retryPauseTime  = self.step.retryDelay,
                                numberOfRetries = self.step.retryCount,
                                **overrides)

        #Now we need to find all the reports
        logFilesForTransfer = []
        #Look in the taskSpace first
        logFilesForTransfer.extend(self.findFilesInDirectory(self.stepSpace.taskSpace.location, matchFiles))

        #What if it's empty?
        if len(logFilesForTransfer) == 0:
            msg = "Could find no log files in job"
            logging.error(msg)
            return logFilesForTransfer

        #Now that we've gone through all the steps, we have to tar it out
        tarName         = 'logArchive.tar.gz'
        tarBallLocation = os.path.join(self.stepSpace.location, tarName)
        tarBall         = tarfile.open(tarBallLocation, 'w:gz')
        for f in logFilesForTransfer:
            tarBall.add(name  = f,
                        arcname = f.replace(self.stepSpace.taskSpace.location, '', 1).lstrip('/'))
        tarBall.close()


        fileInfo = {'LFN': self.getLFN(tarName),
            'PFN' : tarBallLocation,
            'SEName' : None,
            'GUID' : None
            }

        signal.signal(signal.SIGALRM, alarmHandler)
        signal.alarm(waitTime)
        try:
            manager(fileInfo)
            self.report.addOutputModule(moduleName = "logArchive")
            reportFile = {"lfn": fileInfo["LFN"], "pfn": fileInfo["PFN"],
                          "location": fileInfo["SEName"], "module_label": "logArchive",
                          "events": 0, "size": 0, "merged": False,
                          "checksums": {'md5': BasicAlgos.getMD5(tarBallLocation),
                                        'adler32': readAdler32(tarBallLocation),
                                        'cksum': readCksum(tarBallLocation)}}
            self.report.addOutputFile(outputModule = "logArchive", file = reportFile)
        except Alarm:
            msg = "Indefinite hang during stageOut of logArchive"
            logging.error(msg)
            self.report.addError(self.stepName, 60404, "LogArchiveTimeout", msg)
            self.report.persist("Report.pkl")
            raise WMExecutionFailure(60404, "LogArchiveTimeout", msg)
        except WMException, ex:
            self.report.addError(self.stepName, 60307, "LogArchiveFailure", str(ex))
            self.report.setStepStatus(self.stepName, 0)
            self.report.persist("Report.pkl")
            raise ex
        except Exception, ex:
            self.report.addError(self.stepName, 60405, "LogArchiveFailure", str(ex))
            self.report.setStepStatus(self.stepName, 0)
            self.report.persist("Report.pkl")
            msg = "Failure in transferring logArchive tarball\n"
            msg += str(ex) + "\n"
            msg += traceback.format_exc()
            logging.error(msg)
            raise WMException("LogArchiveFailure", message = str(ex))
Ejemplo n.º 29
0
            #try reconnecting
            self.begin()
            try:
                for sql, binds in self.sqlBuffer:
                    result = self.dbi.processData(sql, binds, \
                        conn = self.conn, transaction = True)
                tries = self.retries
                connectionProblem = False
            except Exception, ex:
                logging.warning("Again error. " + str(ex))
                logging.warning(str(self.retries - tries) + " tries left")
                reportedError = str(ex)
                #if connection problem:
                tries += 1
                if tries > self.retries:
                    fatalError = True
                ##else:
                ##connectionProblem = False
                ##tries = self.retries
            # wait a few seconds if connection failed again:
            # multiply wait time.
            if connectionProblem:
                waitTime = waitTime * 3
                logging.warning("Waiting: " + str(waitTime) +
                                " seconds before retry")
                time.sleep(waitTime)
        if connectionProblem or fatalError:
            raise WMException(WMEXCEPTION['WMCORE-12']+str(reportedError), \
                'WMCORE-12')
        return result
Ejemplo n.º 30
0
    def execute(self, emulator=None):
        """
        _execute_

        """
        # Are we using emulators again?
        if emulator is not None:
            return emulator.emulate(self.step, self.job)

        logging.info("Steps.Executors.%s.execute called", self.__class__.__name__)

        overrides = {}
        if hasattr(self.step, 'override'):
            overrides = self.step.override.dictionary_()
        logging.info("Using the following overrides: %s ", overrides)
        # Find alternate stageout location
        self.altLFN = overrides.get('altLFN', None)
        self.failedPreviousStep = overrides.get('previousCmsRunFailure', False)

        logging.info("Step configuration is: %s", self.step)
        # Wait timeout for stageOut
        waitTime = overrides.get('waitTime', 3600 + (self.step.retryDelay * self.step.retryCount))

        matchFiles = [
            ".log$",  # matches the scram, wmagent and cmsRun logs
            "FrameworkJobReport.xml",
            "Report.pkl",
            "^PSet.py$",
            "^PSet.pkl$",
            "_condor_std*",  # condor wrapper logs at the pilot top level
        ]

        ignoredDirs = ['Utils', 'WMCore', 'WMSandbox']

        # Okay, we need a stageOut Manager
        useNewStageOutCode = False
        if getattr(self.step, 'newStageout', False) or \
                ('newStageOut' in overrides and overrides.get('newStageOut')):
            useNewStageOutCode = True
        if not useNewStageOutCode:
            # old style
            manager = StageOutMgr.StageOutMgr(**overrides)
            manager.numberOfRetries = self.step.retryCount
            manager.retryPauseTime = self.step.retryDelay
        else:
            # new style
            logging.info("LOGARCHIVE IS USING NEW STAGEOUT CODE")
            manager = WMCore.Storage.FileManager.StageOutMgr(retryPauseTime=self.step.retryDelay,
                                                             numberOfRetries=self.step.retryCount,
                                                             **overrides)

        # Now we need to find all the reports
        # The log search follows this structure: ~pilotArea/jobArea/WMTaskSpaceArea/StepsArea
        # Start looking at the pilot scratch area first, such that we find the condor logs
        # Then look at the job area in order to find the wmagentJob log
        # Finally, at the taskspace area to find the cmsRun/FWJR/PSet files
        pilotScratchDir = os.path.join(self.stepSpace.taskSpace.location, '../../')
        logFilesToArchive = self.findFilesInDirectory(pilotScratchDir, matchFiles, ignoredDirs)

        # What if it's empty?
        if len(logFilesToArchive) == 0:
            msg = "Couldn't find any log files in the job"
            logging.error(msg)
            return logFilesToArchive

        # Now that we've gone through all the steps, we have to tar it out
        tarName = 'logArchive.tar.gz'
        tarBallLocation = os.path.join(self.stepSpace.location, tarName)
        with tarfile.open(tarBallLocation, 'w:gz') as tarBall:
            for fName in logFilesToArchive:
                altName = fName.replace(pilotScratchDir, '', 1)
                tarBall.add(name=fName, arcname=altName)

        fileInfo = {'LFN': self.getLFN(tarName),
                    'PFN': tarBallLocation,
                    'PNN': None,
                    'GUID': None
                    }
        signal.signal(signal.SIGALRM, alarmHandler)
        signal.alarm(waitTime)

        try:
            manager(fileInfo)
            self.report.addOutputModule(moduleName="logArchive")
            (adler32, cksum) = calculateChecksums(tarBallLocation)
            reportFile = {"lfn": fileInfo["LFN"], "pfn": fileInfo["PFN"],
                          "location": fileInfo["PNN"], "module_label": "logArchive",
                          "events": 0, "size": 0, "merged": False,
                          "checksums": {'adler32': adler32, 'cksum': cksum}}
            self.report.addOutputFile(outputModule="logArchive", aFile=reportFile)
        except Alarm:
            msg = "Indefinite hang during stageOut of logArchive"
            logging.error(msg)
            self.report.addError(self.stepName, 60404, "LogArchiveTimeout", msg)
            self.saveReport()
            raise WMExecutionFailure(60404, "LogArchiveTimeout", msg)
        except WMException as ex:
            self.report.addError(self.stepName, 60307, "LogArchiveFailure", str(ex))
            self.saveReport()
            raise ex
        except Exception as ex:
            self.report.addError(self.stepName, 60405, "LogArchiveFailure", str(ex))
            self.saveReport()
            msg = "Failure in transferring logArchive tarball\n"
            logging.exception(msg)
            raise WMException("LogArchiveFailure", message=str(ex))
        signal.alarm(0)

        signal.alarm(waitTime)
        self.sendLogToEOS(overrides, tarBallLocation, useNewStageOutCode)
        signal.alarm(0)

        return
Ejemplo n.º 31
0
 def __init__(self, message, **data):
     WMException.__init__(self, message, **data)
     self.data.setdefault("ErrorCode", 60313)
     self.data.setdefault("ErrorType", self.__class__.__name__)
Ejemplo n.º 32
0
    def execute(self, conn=None, transaction=False):
        """
        _execute_

        Generic method to create tables and constraints by execute
        sql statements in the create, and constraints dictionaries.

        Before execution the keys assigned to the tables in the self.create
        dictionary are sorted, to offer the possibilitiy of executing
        table creation in a certain order.

        """
        # create tables
        for i in sorted(self.create.keys()):
            try:
                self.dbi.processData(self.create[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception as e:
                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
                                  str(self.create[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCORE-2')

        # create indexes
        for i in self.indexes:
            try:
                self.dbi.processData(self.indexes[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception as e:
                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
                                  str(self.indexes[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCORE-2')

        # set constraints
        for i in self.constraints:
            try:
                self.dbi.processData(self.constraints[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception as e:
                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
                                  str(self.constraints[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCORE-2')

        # insert permanent data
        for i in self.inserts:
            try:
                self.dbi.processData(self.inserts[i],
                                     conn=conn,
                                     transaction=transaction)
            except Exception as e:
                msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
                                  str(self.inserts[i]) +'\n\n' +str(e)
                self.logger.debug(msg)
                raise WMException(msg, 'WMCORE-2')

        return True
Ejemplo n.º 33
0
    def __init__(self, config):

        self.config = config

        BasePlugin.__init__(self, config)

        self.locationDict = {}

        myThread = threading.currentThread()
        daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
                                dbinterface = myThread.dbi)
        self.locationAction = daoFactory(classname = "Locations.GetSiteInfo")


        self.packageDir = None

        if os.path.exists(os.path.join(getWMBASE(),
                                       'src/python/WMCore/WMRuntime/Unpacker.py')):
            self.unpacker = os.path.join(getWMBASE(),
                                         'src/python/WMCore/WMRuntime/Unpacker.py')
        else:
            self.unpacker = os.path.join(getWMBASE(),
                                         'WMCore/WMRuntime/Unpacker.py')

        self.agent         = getattr(config.Agent, 'agentName', 'WMAgent')
        self.sandbox       = None
        self.scriptFile    = None
        self.submitDir     = None
        self.removeTime    = getattr(config.BossAir, 'removeTime', 60)
        self.multiTasks    = getattr(config.BossAir, 'multicoreTaskTypes', [])
        self.useGSite      = getattr(config.BossAir, 'useGLIDEINSites', False)
        self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False)
        self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10)
        self.errorCount    = 0
        self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
        self.maxTaskPriority     = getattr(config.BossAir, 'maxTaskPriority', 1e7)

        # Build ourselves a pool
        self.pool     = []
        self.input    = None
        self.result   = None
        self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4)

        # Set up my proxy and glexec stuff
        self.setupScript = getattr(config.BossAir, 'UISetupScript', None)
        self.proxy       = None
        self.serverCert  = getattr(config.BossAir, 'delegatedServerCert', None)
        self.serverKey   = getattr(config.BossAir, 'delegatedServerKey', None)
        self.myproxySrv  = getattr(config.BossAir, 'myproxyServer', None)
        self.proxyDir    = getattr(config.BossAir, 'proxyDir', '/tmp/')
        self.serverHash  = getattr(config.BossAir, 'delegatedServerHash', None)
        self.glexecPath  = getattr(config.BossAir, 'glexecPath', None)
        self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None)
        self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None)
        self.jdlProxyFile    = None # Proxy name to put in JDL (owned by submit user)
        self.glexecProxyFile = None # Copy of same file owned by submit user

        if self.glexecPath:
            if not (self.myproxySrv and self.proxyDir):
                raise WMException('glexec requires myproxyServer and proxyDir to be set.')
        if self.myproxySrv:
            if not (self.serverCert and self.serverKey):
                raise WMException('MyProxy server requires serverCert and serverKey to be set.')

        # Make the directory for the proxies
        if self.proxyDir and not os.path.exists(self.proxyDir):
            logging.debug("proxyDir not found: creating it.")
            try:
                os.makedirs(self.proxyDir, 01777)
            except Exception, ex:
                msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex)
                raise BossAirPluginException(msg)