def __init__(self, msg):
"""
__init__
Initialize the error, just accepts a message without error code
"""
WMException.__init__(self, msg)
return
def __init__(self, msg):
"""
__init__
Initialize the error, just accepts a message without error code
"""
WMException.__init__(self, msg)
return
def testExceptionUnicode0(self):
"""
create an exception with non-ascii chars in message and test WMException.addInfo().
"""
exception = WMException(
"an exception message with nr. 100 and some non-ascii characters: ₩♏ℭ☺яε",
100)
exception.addInfo(**self.test_data)
self.logger.debug("XML version of exception: %s", exception.xml())
self.logger.debug("String version of exception: %s", str(exception))
def testException(self):
"""
create an exception and do some tests.
"""
exception = WMException("an exception message with nr. 100", 100)
self.logger.debug("String version of exception: " + str(exception))
self.logger.debug("XML version of exception: " + exception.xml())
self.logger.debug("Adding data")
data = {}
data['key1'] = 'value1'
data['key2'] = 'data2'
exception.addInfo(**data)
self.logger.debug("String version of exception: " + str(exception))
def testException(self):
"""
create an exception and do some tests (only ascii chars)
"""
exception = WMException("an exception message with nr. 100", 100)
self.logger.debug("String version of exception: %s", str(exception))
self.logger.debug("XML version of exception: %s", exception.xml())
self.logger.debug("Adding data")
data = {}
data['key1'] = 'value1'
data['key2'] = 3.14159
exception.addInfo(**data)
self.logger.debug("String version of exception: %s", str(exception))
def testExceptionUnicode1(self):
"""
create an exception with non-ascii chars in message and test WMException constructor
"""
exception = WMException(
"an exception message with nr. 100 and some non-ascii characters: ₩♏ℭ☺яε",
100, **self.test_data)
self.logger.debug("XML version of exception: %s", exception.xml())
self.logger.debug("String version of exception: %s", str(exception))
self.logger.debug("exception.__str__(): %s", type(exception.__str__(
))) # from py2 interpreter: <class 'future.types.newbytes.newbytes'>
self.logger.debug("str(exception): %s", type(
str(exception))) # <class 'future.types.newstr.newstr'>
def testException(self):
"""
create an exception and do some tests.
"""
exception = WMException("an exception message with nr. 100", 100)
self.logger.debug("String version of exception: " + str(exception))
self.logger.debug("XML version of exception: " + exception.xml())
self.logger.debug("Adding data")
data = {}
data['key1'] = 'value1'
data['key2'] = 'data2'
exception.addInfo(**data)
self.logger.debug("String version of exception: "+ str(exception))
def testExceptionUnicode(self):
"""
create an exception with non-ascii characters and do some tests.
"""
exception = WMException(
"an exception message with nr. 100 and some non-ascii characters: ₩♏ℭ☺яε",
100)
self.logger.debug("String version of exception: " + str(exception))
self.logger.debug("XML version of exception: " + exception.xml())
self.logger.debug("Adding data")
data = {}
data['key1'] = 'value1'
data['key2'] = 'data2'
exception.addInfo(**data)
self.logger.debug("String version of exception: " + str(exception))
def execute(self, conn=None, transaction=False):
"""
_execute_
Generic method to create tables and constraints by execute
sql statements in the create, and constraints dictionaries.
Before execution the keys assigned to the tables in the self.create
dictionary are sorted, to offer the possibilitiy of executing
table creation in a certain order.
"""
# get the keys for the table mapping:
tableKeys = self.create.keys()
tableKeys.sort()
for i in tableKeys:
try:
self.dbi.processData(self.create[i],
conn=conn,
transaction=transaction)
except Exception, e:
msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
str(self.create[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCore-2')
keys = self.constraints.keys()
def parseLFN(candidate):
"""
_parseLFN_
Take an LFN, return the component parts
"""
separator = "/"
# First, make sure what we've gotten is a real LFN
lfn(candidate)
parts = candidate.split('/')
final = {}
if parts[0] == '':
parts.remove('')
if 'user' in parts[1:3] or 'group' in parts[1:3]:
if parts[1] in ['user', 'group']:
final['baseLocation'] = '/%s' % separator.join(parts[:2])
parts = parts[2:]
else:
final['baseLocation'] = '/%s' % separator.join(parts[:3])
parts = parts[3:]
final['hnName'] = parts[0]
final['primaryDataset'] = parts[1]
final['secondaryDataset'] = parts[2]
final['processingVersion'] = parts[3]
final['lfnCounter'] = parts[4]
final['filename'] = parts[5]
return final
if len(parts) == 8:
# Then we have only two locations
final['baseLocation'] = '/%s' % separator.join(parts[:2])
parts = parts[2:]
elif len(parts) == 9:
final['baseLocation'] = '/%s' % separator.join(parts[:3])
parts = parts[3:]
else:
# How did we end up here?
# Something just went wrong
msg = """CRITICAL! This machine has experienced a complete logic failure while parsing LFNs.\n
If you are a developer this indicates that you have changed the Lexicon LFN regexp functions without changing the parsing.\n
If you are an operator, this indicates that this machine is likely unstable.\n
All data should be backed up and the machine removed from production for examination.\n"""
msg += "Candidate: %s" % candidate
raise WMException(msg)
final['acquisitionEra'] = parts[0]
final['primaryDataset'] = parts[1]
final['dataTier'] = parts[2]
final['processingVersion'] = parts[3]
final['lfnCounter'] = parts[4]
final['filename'] = parts[5]
return final
def __init__(self, dict={}):
try:
Service.__init__(self, dict)
self["requests"] = SSLRequests(self["requests"]["host"])
except WMException, ex:
msg = str(ex)
self["logger"].exception(msg)
raise WMException(msg)
class DBCreator(DBFormatter):
"""
_DBCreator_
Generic class for creating database tables.
"""
def __init__(self, logger, dbinterface):
"""
_init_
Call the constructor of the parent class and create empty dictionaries
to hold table create statements, constraint statements and insert
statements.
"""
DBFormatter.__init__(self, logger, dbinterface)
self.create = {}
self.constraints = {}
self.inserts = {}
self.indexes = {}
def execute(self, conn=None, transaction=False):
"""
_execute_
Generic method to create tables and constraints by execute
sql statements in the create, and constraints dictionaries.
Before execution the keys assigned to the tables in the self.create
dictionary are sorted, to offer the possibilitiy of executing
table creation in a certain order.
"""
# create tables
for i in sorted(self.create.keys()):
try:
self.dbi.processData(self.create[i],
conn=conn,
transaction=transaction)
except Exception, e:
msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
str(self.create[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCore-2')
# create indexes
for i in self.indexes.keys():
try:
self.dbi.processData(self.indexes[i],
conn=conn,
transaction=transaction)
except Exception, e:
msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
str(self.indexes[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCore-2')
def __init__(self, config, compName = None):
"""
init
The constructor is empty as we have an initalization method
that can be called inside new threads (we use thread local attributes
at startup.
Default intialization of the harness including setting some diagnostic
messages
"""
self.config = config
# component name is always the class name of child class
if not compName:
compName = self.__class__.__name__
if not compName in (self.config.listComponents_() + self.config.listWebapps_()):
raise WMException(WMEXCEPTION['WMCORE-8']+compName, 'WMCORE-8')
if not hasattr(self.config, "Agent"):
self.config.section_("Agent")
self.config.Agent.componentName = compName
compSect = getattr(self.config, compName, None)
if compSect == None:
# Then we have a major problem - there's no section with this name
logging.error("Could not find section %s in config" % compName)
logging.error("We are returning, and hoping you know what you're doing!")
logging.debug("Config: %s" % self.config)
return
# check if componentDir is set if not assign.
if getattr(compSect, 'componentDir', None) == None:
if not hasattr(self.config, "General"):
# Don't do anything. Assume the user knows what they are doing.
logging.error("Missing componentDir and General section in config")
logging.error("Going to trust you to know what you're doing.")
return
compSect.componentDir = os.path.join(self.config.General.workDir,
'Components',
self.config.Agent.componentName)
# we have name and location of the log files. Now make sure there
# is a directory.
try:
if not os.path.isdir(compSect.componentDir):
os.makedirs(compSect.componentDir)
except Exception as ex:
logging.error("Encountered exception while making componentDirs: %s" % str(ex))
logging.error("Ignoring")
self.threadManagerName = ''
self.heartbeatAPI = None
self.messages = {}
self.logMsg = {}
return
def getKeyCert(self):
"""
_getKeyCert_
Get the user credentials if they exist, otherwise throw an exception.
This code was modified from DBSAPI/dbsHttpService.py
"""
cert = None
key = None
# Zeroth case is if the class has over ridden the key/cert and has it
# stored in self
if self.has_key('cert') and self.has_key('key' ) \
and self['cert'] and self['key']:
key = self['key']
cert = self['cert']
# Now we're trying to guess what the right cert/key combo is...
# First preference to HOST Certificate, This is how it set in Tier0
elif os.environ.has_key('X509_HOST_CERT'):
cert = os.environ['X509_HOST_CERT']
key = os.environ['X509_HOST_KEY']
# Second preference to User Proxy, very common
elif (os.environ.has_key('X509_USER_PROXY')) and \
(os.path.exists( os.environ['X509_USER_PROXY'])):
cert = os.environ['X509_USER_PROXY']
key = cert
# Third preference to User Cert/Proxy combinition
elif os.environ.has_key('X509_USER_CERT'):
cert = os.environ['X509_USER_CERT']
key = os.environ['X509_USER_KEY']
# TODO: only in linux, unix case, add other os case
# look for proxy at default location /tmp/x509up_u$uid
elif os.path.exists('/tmp/x509up_u' + str(os.getuid())):
cert = '/tmp/x509up_u' + str(os.getuid())
key = cert
# if interactive we can use an encrypted certificate
elif sys.stdin.isatty():
if os.path.exists(os.environ['HOME'] + '/.globus/usercert.pem'):
cert = os.environ['HOME'] + '/.globus/usercert.pem'
if os.path.exists(os.environ['HOME'] + '/.globus/userkey.pem'):
key = os.environ['HOME'] + '/.globus/userkey.pem'
else:
key = cert
#Set but not found
if key and cert:
if not os.path.exists(cert) or not os.path.exists(key):
raise WMException(
'Request requires a host certificate and key', "WMCORE-11")
# All looks OK, still doesn't guarantee proxy's validity etc.
return key, cert
def __init__(self, dict={}):
#The following should read the configuration class
for a in ['logger', 'endpoint']:
assert a in dict.keys(), "Can't have a service without a %s" % a
scheme = ''
netloc = ''
path = ''
# then split the endpoint into netloc and basepath
endpoint = urlparse(dict['endpoint'])
try:
#Only works on python 2.5 or above
scheme = endpoint.scheme
netloc = endpoint.netloc
path = endpoint.path
except AttributeError:
scheme, netloc, path = endpoint[:3]
#set up defaults
self.setdefault("inputdata", {})
self.setdefault("cachepath", '/tmp')
self.setdefault("cacheduration", 0.5)
self.setdefault("maxcachereuse", 24.0)
self.supportVerbList = ('GET', 'POST', 'PUT', 'DELETE')
# this value should be only set when whole service class uses
# the same verb ('GET', 'POST', 'PUT', 'DELETE')
self.setdefault("method", None)
#Set a timeout for the socket
self.setdefault("timeout", 30)
# then update with the incoming dict
self.update(dict)
# Get the request class, to instatiate later
# Is this a secure service - add other schemes as we need them
if self.get('secure', False) or scheme in SECURE_SERVICES:
# only bring in ssl stuff if we really need it
from WMCore.Services.Requests import SecureRequests
requests = SecureRequests
else:
requests = Requests
try:
if path and not path.endswith('/'):
path += '/'
self.setdefault("basepath", path)
# Instantiate a Request
self.setdefault("requests", requests(netloc, dict))
except WMException, ex:
msg = str(ex)
self["logger"].exception(msg)
raise WMException(msg)
def execute(self, conn=None, transaction=None):
"""
_execute_
Check to make sure that all required tables have been defined. If
everything is in place have the DBCreator make everything.
"""
for requiredTable in self.requiredTables:
if requiredTable not in self.create.keys():
raise WMException("The table '%s' is not defined." % \
requiredTable, "WMCORE-2")
DBCreator.execute(self, conn, transaction)
return True
def getKeyCert(self):
"""
_getKeyCert_
Get the user credentials if they exist, otherwise throw an exception.
This code was modified from DBSAPI/dbsHttpService.py
"""
# Zeroth case is if the class has over ridden the key/cert and has it
# stored in self
if self['cert'] and self['key']:
key = self['key']
cert = self['cert']
else:
key, cert = getKeyCertFromEnv()
# Set but not found
if key is None or cert is None:
raise WMException('Request requires a host certificate and key',
"WMCORE-11")
# All looks OK, still doesn't guarantee proxy's validity etc.
return key, cert
def __init__(self):
WMException.__init__(self, "McM responded correctly but has no data")
def __init__(self, message, **data):
WMException.__init__(self, message, **data)
self.data.setdefault("ErrorCode", 60311)
self.data.setdefault("ErrorType", ErrorDefinitions[60311])
def __init__(self, message):
WMException.__init__(self, message, 'WMCORE-13')
def __init__(self, message):
WMException.__init__(self, message, 'WMCORE-13')
def __init__(self, message, **data):
WMException.__init__(self, message, **data)
self.data.setdefault("ErrorCode", 60311)
self.data.setdefault("ErrorType", ErrorDefinitions[60311])
def __init__(self):
WMException.__init__(self, 'McM responded correctly but has no data')
def __init__(self, code, name, detail):
WMException.__init__(self, detail, code)
self.code = code
self.name = name
self.detail = detail
msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
str(self.indexes[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCore-2')
# set constraints
for i in self.constraints.keys():
try:
self.dbi.processData(self.constraints[i],
conn=conn,
transaction=transaction)
except Exception, e:
msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
str(self.constraints[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCore-2')
# insert permanent data
for i in self.inserts.keys():
try:
self.dbi.processData(self.inserts[i],
conn=conn,
transaction=transaction)
except Exception, e:
msg = WMEXCEPTION['WMCore-2'] + '\n\n' +\
str(self.inserts[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCore-2')
return True
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
dbinterface = myThread.dbi)
self.locationAction = daoFactory(classname = "Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = None
self.submitDir = None
self.removeTime = getattr(config.BossAir, 'removeTime', 60)
self.useGSite = getattr(config.BossAir, 'useGLIDEINSites', False)
self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False)
self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10)
self.errorCount = 0
self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
# Required for global pool accounting
self.acctGroup = getattr(config.BossAir, 'acctGroup', "production")
self.acctGroupUser = getattr(config.BossAir, 'acctGroupUser', "cmsdataops")
# Build ourselves a pool
self.pool = []
self.input = None
self.result = None
self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4)
# Set up my proxy and glexec stuff
self.setupScript = getattr(config.BossAir, 'UISetupScript', None)
self.proxy = None
self.serverCert = getattr(config.BossAir, 'delegatedServerCert', None)
self.serverKey = getattr(config.BossAir, 'delegatedServerKey', None)
self.myproxySrv = getattr(config.BossAir, 'myproxyServer', None)
self.proxyDir = getattr(config.BossAir, 'proxyDir', '/tmp/')
self.serverHash = getattr(config.BossAir, 'delegatedServerHash', None)
self.glexecPath = getattr(config.BossAir, 'glexecPath', None)
self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None)
self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None)
self.jdlProxyFile = None # Proxy name to put in JDL (owned by submit user)
self.glexecProxyFile = None # Copy of same file owned by submit user
if self.glexecPath:
if not (self.myproxySrv and self.proxyDir):
raise WMException('glexec requires myproxyServer and proxyDir to be set.')
if self.myproxySrv:
if not (self.serverCert and self.serverKey):
raise WMException('MyProxy server requires serverCert and serverKey to be set.')
# Make the directory for the proxies
if self.proxyDir and not os.path.exists(self.proxyDir):
logging.debug("proxyDir not found: creating it.")
try:
os.makedirs(self.proxyDir, 0o1777)
except Exception as ex:
msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex)
raise BossAirPluginException(msg)
elif not os.path.isdir(self.proxyDir):
msg = "Error: proxyDir '%s' is not a directory" % self.proxyDir
raise BossAirPluginException(msg)
if self.serverCert and self.serverKey and self.myproxySrv:
self.proxy = self.setupMyProxy()
# Build a request string
self.reqStr = "(Memory >= 1 && OpSys == \"LINUX\" ) && (Arch == \"INTEL\" || Arch == \"X86_64\") && stringListMember(GLIDEIN_CMSSite, DESIRED_Sites) && ((REQUIRED_OS==\"any\") || (GLIDEIN_REQUIRED_OS==REQUIRED_OS))"
if hasattr(config.BossAir, 'condorRequirementsString'):
self.reqStr = config.BossAir.condorRequirementsString
return
def __init__(self, code, name, detail):
WMException.__init__(self, detail, code)
self.code = code
self.name = name
self.detail = detail
class LogArchive(Executor):
"""
_LogArchive_
Execute a LogArchive Step
"""
def pre(self, emulator = None):
"""
_pre_
Pre execution checks
"""
#Are we using an emulator?
if (emulator != None):
return emulator.emulatePre( self.step )
logging.info("Steps.Executors.LogArchive.pre called")
return None
def execute(self, emulator = None, **overrides):
"""
_execute_
"""
# Are we using emulators again?
if (emulator != None):
return emulator.emulate( self.step, self.job )
overrides = {}
if hasattr(self.step, 'override'):
overrides = self.step.override.dictionary_()
# Find alternate stageout location
self.altLFN = overrides.get('altLFN', None)
logging.info("Beginning Steps.Executors.LogArchive.Execute")
logging.info("Using the following overrides: %s " % overrides)
logging.info("Step is: %s" % self.step)
# Wait timout for stageOut
waitTime = overrides.get('waitTime', 3600 + (self.step.retryDelay * self.step.retryCount))
matchFiles = [
".log$",
"FrameworkJobReport",
"Report.pkl",
"Report.pcl",
"^PSet.py$",
"^PSet.pkl$"
]
#Okay, we need a stageOut Manager
useNewStageOutCode = False
if getattr(self.step, 'newStageout', False) or \
(overrides.has_key('newStageOut') and overrides.get('newStageOut')):
useNewStageOutCode = True
if not useNewStageOutCode:
# old style
manager = StageOutMgr.StageOutMgr(**overrides)
manager.numberOfRetries = self.step.retryCount
manager.retryPauseTime = self.step.retryDelay
else:
# new style
logging.info("LOGARCHIVE IS USING NEW STAGEOUT CODE")
manager = WMCore.Storage.FileManager.StageOutMgr(
retryPauseTime = self.step.retryDelay,
numberOfRetries = self.step.retryCount,
**overrides)
#Now we need to find all the reports
logFilesForTransfer = []
#Look in the taskSpace first
logFilesForTransfer.extend(self.findFilesInDirectory(self.stepSpace.taskSpace.location, matchFiles))
#What if it's empty?
if len(logFilesForTransfer) == 0:
msg = "Could find no log files in job"
logging.error(msg)
return logFilesForTransfer
#Now that we've gone through all the steps, we have to tar it out
tarName = 'logArchive.tar.gz'
tarBallLocation = os.path.join(self.stepSpace.location, tarName)
tarBall = tarfile.open(tarBallLocation, 'w:gz')
for f in logFilesForTransfer:
tarBall.add(name = f,
arcname = f.replace(self.stepSpace.taskSpace.location, '', 1).lstrip('/'))
tarBall.close()
fileInfo = {'LFN': self.getLFN(tarName),
'PFN' : tarBallLocation,
'SEName' : None,
'GUID' : None
}
signal.signal(signal.SIGALRM, alarmHandler)
signal.alarm(waitTime)
try:
manager(fileInfo)
self.report.addOutputModule(moduleName = "logArchive")
reportFile = {"lfn": fileInfo["LFN"], "pfn": fileInfo["PFN"],
"location": fileInfo["SEName"], "module_label": "logArchive",
"events": 0, "size": 0, "merged": False,
"checksums": {'md5': BasicAlgos.getMD5(tarBallLocation),
'adler32': readAdler32(tarBallLocation),
'cksum': readCksum(tarBallLocation)}}
self.report.addOutputFile(outputModule = "logArchive", file = reportFile)
except Alarm:
msg = "Indefinite hang during stageOut of logArchive"
logging.error(msg)
self.report.addError(self.stepName, 60404, "LogArchiveTimeout", msg)
self.report.persist("Report.pkl")
raise WMExecutionFailure(60404, "LogArchiveTimeout", msg)
except WMException, ex:
self.report.addError(self.stepName, 60307, "LogArchiveFailure", str(ex))
self.report.setStepStatus(self.stepName, 0)
self.report.persist("Report.pkl")
raise ex
except Exception, ex:
self.report.addError(self.stepName, 60405, "LogArchiveFailure", str(ex))
self.report.setStepStatus(self.stepName, 0)
self.report.persist("Report.pkl")
msg = "Failure in transferring logArchive tarball\n"
msg += str(ex) + "\n"
msg += traceback.format_exc()
logging.error(msg)
raise WMException("LogArchiveFailure", message = str(ex))
#try reconnecting
self.begin()
try:
for sql, binds in self.sqlBuffer:
result = self.dbi.processData(sql, binds, \
conn = self.conn, transaction = True)
tries = self.retries
connectionProblem = False
except Exception, ex:
logging.warning("Again error. " + str(ex))
logging.warning(str(self.retries - tries) + " tries left")
reportedError = str(ex)
#if connection problem:
tries += 1
if tries > self.retries:
fatalError = True
##else:
##connectionProblem = False
##tries = self.retries
# wait a few seconds if connection failed again:
# multiply wait time.
if connectionProblem:
waitTime = waitTime * 3
logging.warning("Waiting: " + str(waitTime) +
" seconds before retry")
time.sleep(waitTime)
if connectionProblem or fatalError:
raise WMException(WMEXCEPTION['WMCORE-12']+str(reportedError), \
'WMCORE-12')
return result
def execute(self, emulator=None):
"""
_execute_
"""
# Are we using emulators again?
if emulator is not None:
return emulator.emulate(self.step, self.job)
logging.info("Steps.Executors.%s.execute called", self.__class__.__name__)
overrides = {}
if hasattr(self.step, 'override'):
overrides = self.step.override.dictionary_()
logging.info("Using the following overrides: %s ", overrides)
# Find alternate stageout location
self.altLFN = overrides.get('altLFN', None)
self.failedPreviousStep = overrides.get('previousCmsRunFailure', False)
logging.info("Step configuration is: %s", self.step)
# Wait timeout for stageOut
waitTime = overrides.get('waitTime', 3600 + (self.step.retryDelay * self.step.retryCount))
matchFiles = [
".log$", # matches the scram, wmagent and cmsRun logs
"FrameworkJobReport.xml",
"Report.pkl",
"^PSet.py$",
"^PSet.pkl$",
"_condor_std*", # condor wrapper logs at the pilot top level
]
ignoredDirs = ['Utils', 'WMCore', 'WMSandbox']
# Okay, we need a stageOut Manager
useNewStageOutCode = False
if getattr(self.step, 'newStageout', False) or \
('newStageOut' in overrides and overrides.get('newStageOut')):
useNewStageOutCode = True
if not useNewStageOutCode:
# old style
manager = StageOutMgr.StageOutMgr(**overrides)
manager.numberOfRetries = self.step.retryCount
manager.retryPauseTime = self.step.retryDelay
else:
# new style
logging.info("LOGARCHIVE IS USING NEW STAGEOUT CODE")
manager = WMCore.Storage.FileManager.StageOutMgr(retryPauseTime=self.step.retryDelay,
numberOfRetries=self.step.retryCount,
**overrides)
# Now we need to find all the reports
# The log search follows this structure: ~pilotArea/jobArea/WMTaskSpaceArea/StepsArea
# Start looking at the pilot scratch area first, such that we find the condor logs
# Then look at the job area in order to find the wmagentJob log
# Finally, at the taskspace area to find the cmsRun/FWJR/PSet files
pilotScratchDir = os.path.join(self.stepSpace.taskSpace.location, '../../')
logFilesToArchive = self.findFilesInDirectory(pilotScratchDir, matchFiles, ignoredDirs)
# What if it's empty?
if len(logFilesToArchive) == 0:
msg = "Couldn't find any log files in the job"
logging.error(msg)
return logFilesToArchive
# Now that we've gone through all the steps, we have to tar it out
tarName = 'logArchive.tar.gz'
tarBallLocation = os.path.join(self.stepSpace.location, tarName)
with tarfile.open(tarBallLocation, 'w:gz') as tarBall:
for fName in logFilesToArchive:
altName = fName.replace(pilotScratchDir, '', 1)
tarBall.add(name=fName, arcname=altName)
fileInfo = {'LFN': self.getLFN(tarName),
'PFN': tarBallLocation,
'PNN': None,
'GUID': None
}
signal.signal(signal.SIGALRM, alarmHandler)
signal.alarm(waitTime)
try:
manager(fileInfo)
self.report.addOutputModule(moduleName="logArchive")
(adler32, cksum) = calculateChecksums(tarBallLocation)
reportFile = {"lfn": fileInfo["LFN"], "pfn": fileInfo["PFN"],
"location": fileInfo["PNN"], "module_label": "logArchive",
"events": 0, "size": 0, "merged": False,
"checksums": {'adler32': adler32, 'cksum': cksum}}
self.report.addOutputFile(outputModule="logArchive", aFile=reportFile)
except Alarm:
msg = "Indefinite hang during stageOut of logArchive"
logging.error(msg)
self.report.addError(self.stepName, 60404, "LogArchiveTimeout", msg)
self.saveReport()
raise WMExecutionFailure(60404, "LogArchiveTimeout", msg)
except WMException as ex:
self.report.addError(self.stepName, 60307, "LogArchiveFailure", str(ex))
self.saveReport()
raise ex
except Exception as ex:
self.report.addError(self.stepName, 60405, "LogArchiveFailure", str(ex))
self.saveReport()
msg = "Failure in transferring logArchive tarball\n"
logging.exception(msg)
raise WMException("LogArchiveFailure", message=str(ex))
signal.alarm(0)
signal.alarm(waitTime)
self.sendLogToEOS(overrides, tarBallLocation, useNewStageOutCode)
signal.alarm(0)
return
def __init__(self, message, **data):
WMException.__init__(self, message, **data)
self.data.setdefault("ErrorCode", 60313)
self.data.setdefault("ErrorType", self.__class__.__name__)
def execute(self, conn=None, transaction=False):
"""
_execute_
Generic method to create tables and constraints by execute
sql statements in the create, and constraints dictionaries.
Before execution the keys assigned to the tables in the self.create
dictionary are sorted, to offer the possibilitiy of executing
table creation in a certain order.
"""
# create tables
for i in sorted(self.create.keys()):
try:
self.dbi.processData(self.create[i],
conn=conn,
transaction=transaction)
except Exception as e:
msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
str(self.create[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCORE-2')
# create indexes
for i in self.indexes:
try:
self.dbi.processData(self.indexes[i],
conn=conn,
transaction=transaction)
except Exception as e:
msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
str(self.indexes[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCORE-2')
# set constraints
for i in self.constraints:
try:
self.dbi.processData(self.constraints[i],
conn=conn,
transaction=transaction)
except Exception as e:
msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
str(self.constraints[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCORE-2')
# insert permanent data
for i in self.inserts:
try:
self.dbi.processData(self.inserts[i],
conn=conn,
transaction=transaction)
except Exception as e:
msg = WMEXCEPTION['WMCORE-2'] + '\n\n' +\
str(self.inserts[i]) +'\n\n' +str(e)
self.logger.debug(msg)
raise WMException(msg, 'WMCORE-2')
return True
def __init__(self, config):
self.config = config
BasePlugin.__init__(self, config)
self.locationDict = {}
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
dbinterface = myThread.dbi)
self.locationAction = daoFactory(classname = "Locations.GetSiteInfo")
self.packageDir = None
if os.path.exists(os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')):
self.unpacker = os.path.join(getWMBASE(),
'src/python/WMCore/WMRuntime/Unpacker.py')
else:
self.unpacker = os.path.join(getWMBASE(),
'WMCore/WMRuntime/Unpacker.py')
self.agent = getattr(config.Agent, 'agentName', 'WMAgent')
self.sandbox = None
self.scriptFile = None
self.submitDir = None
self.removeTime = getattr(config.BossAir, 'removeTime', 60)
self.multiTasks = getattr(config.BossAir, 'multicoreTaskTypes', [])
self.useGSite = getattr(config.BossAir, 'useGLIDEINSites', False)
self.submitWMSMode = getattr(config.BossAir, 'submitWMSMode', False)
self.errorThreshold= getattr(config.BossAir, 'submitErrorThreshold', 10)
self.errorCount = 0
self.defaultTaskPriority = getattr(config.BossAir, 'defaultTaskPriority', 0)
self.maxTaskPriority = getattr(config.BossAir, 'maxTaskPriority', 1e7)
# Build ourselves a pool
self.pool = []
self.input = None
self.result = None
self.nProcess = getattr(self.config.BossAir, 'nCondorProcesses', 4)
# Set up my proxy and glexec stuff
self.setupScript = getattr(config.BossAir, 'UISetupScript', None)
self.proxy = None
self.serverCert = getattr(config.BossAir, 'delegatedServerCert', None)
self.serverKey = getattr(config.BossAir, 'delegatedServerKey', None)
self.myproxySrv = getattr(config.BossAir, 'myproxyServer', None)
self.proxyDir = getattr(config.BossAir, 'proxyDir', '/tmp/')
self.serverHash = getattr(config.BossAir, 'delegatedServerHash', None)
self.glexecPath = getattr(config.BossAir, 'glexecPath', None)
self.glexecWrapScript = getattr(config.BossAir, 'glexecWrapScript', None)
self.glexecUnwrapScript = getattr(config.BossAir, 'glexecUnwrapScript', None)
self.jdlProxyFile = None # Proxy name to put in JDL (owned by submit user)
self.glexecProxyFile = None # Copy of same file owned by submit user
if self.glexecPath:
if not (self.myproxySrv and self.proxyDir):
raise WMException('glexec requires myproxyServer and proxyDir to be set.')
if self.myproxySrv:
if not (self.serverCert and self.serverKey):
raise WMException('MyProxy server requires serverCert and serverKey to be set.')
# Make the directory for the proxies
if self.proxyDir and not os.path.exists(self.proxyDir):
logging.debug("proxyDir not found: creating it.")
try:
os.makedirs(self.proxyDir, 01777)
except Exception, ex:
msg = "Error: problem when creating proxyDir directory - '%s'" % str(ex)
raise BossAirPluginException(msg)
