def execute(self, emulator=None):
"""
_execute_
"""
stepModule = "WMTaskSpace.%s" % self.stepName
if emulator is not None:
return emulator.emulate(self.step, self.job)
# write the wrapper script to a temporary location
# I don't pass it directly through os.system because I don't
# trust that there won't be shell-escape shenanigans with
# arbitrary input files
scramSetup = self.step.application.setup.softwareEnvironment
scramCommand = self.step.application.setup.scramCommand
scramProject = self.step.application.setup.scramProject
scramArch = self.step.application.setup.scramArch
cmsswVersion = self.step.application.setup.cmsswVersion
jobReportXML = self.step.output.jobReport
cmsswCommand = self.step.application.command.executable
cmsswConfig = self.step.application.command.configuration
cmsswArguments = self.step.application.command.arguments
userTarball = ",".join(self.step.user.inputSandboxes)
userFiles = ",".join(self.step.user.userFiles)
logging.info("User files are %s", userFiles)
logging.info("User sandboxes are %s", userTarball)
multicoreSettings = self.step.application.multicore
try:
logging.info("CMSSW configured for %s cores", multicoreSettings.numberOfCores)
except AttributeError:
logging.info("No value set for multicore.numberOfCores")
logging.info("Executing CMSSW step")
#
# set any global environment variables
#
try:
os.environ["FRONTIER_ID"] = "wmagent_%s" % (self.report.data.workload)
except Exception as ex:
logging.error("Have critical error in setting FRONTIER_ID: %s", str(ex))
logging.error("Continuing, as this is not a critical function yet.")
#
# scram bootstrap
#
scram = Scram(
command=scramCommand,
version=cmsswVersion,
initialise=self.step.application.setup.softwareEnvironment,
directory=self.step.builder.workingDir,
architecture=scramArch,
)
logging.info("Runing SCRAM")
try:
projectOutcome = scram.project()
except Exception as ex:
msg = "Exception raised while running scram.\n"
msg += str(ex)
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
if projectOutcome > 0:
msg = scram.diagnostic()
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
runtimeOutcome = scram.runtime()
if runtimeOutcome > 0:
msg = scram.diagnostic()
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
#
# pre scripts
#
logging.info("Running PRE scripts")
for script in self.step.runtime.preScripts:
# TODO: Exception handling and error handling & logging
scriptProcess = subprocess.Popen(
["/bin/bash"],
shell=True,
cwd=self.step.builder.workingDir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
# BADPYTHON
scriptProcess.stdin.write("export LD_LIBRARY_PATH=$LD_LIBRARY_PATH\n")
invokeCommand = "%s -m WMCore.WMRuntime.ScriptInvoke %s %s \n" % (sys.executable, stepModule, script)
logging.info(" Invoking command: %s", invokeCommand)
scriptProcess.stdin.write(invokeCommand)
stdout, stderr = scriptProcess.communicate()
retCode = scriptProcess.returncode
if retCode > 0:
msg = "Error running command\n%s\n" % invokeCommand
msg += "%s\n %s\n %s\n" % (retCode, stdout, stderr)
logging.critical("Error running command")
logging.critical(msg)
raise WMExecutionFailure(50513, "PreScriptFailure", msg)
#
# pre scripts with scram
#
logging.info("RUNNING SCRAM SCRIPTS")
for script in self.step.runtime.scramPreScripts:
# invoke scripts with scram()
runtimeDir = getattr(self.step.runtime, "scramPreDir", None)
invokeCommand = (
self.step.runtime.invokeCommand
if hasattr(self.step.runtime, "invokeCommand")
else "%s -m WMCore.WMRuntime.ScriptInvoke %s" % (sys.executable, stepModule)
)
invokeCommand += " %s \n" % script
retCode = scram(invokeCommand, runtimeDir=runtimeDir)
if retCode > 0:
msg = "Error running command\n%s\n" % invokeCommand
msg += "%s\n " % retCode
msg += scram.diagnostic()
logging.critical(msg)
raise WMExecutionFailure(50513, "PreScriptScramFailure", msg)
configPath = "%s/%s-main.sh" % (self.step.builder.workingDir, self.stepName)
handle = open(configPath, "w")
handle.write(CONFIG_BLOB)
handle.close()
# spawn this new process
# the script looks for:
# <SCRAM_COMMAND> <SCRAM_PROJECT> <CMSSW_VERSION> <JOB_REPORT> <EXECUTABLE> <CONFIG>
# open the output files
stdoutHandle = open(self.step.output.stdout, "w")
stderrHandle = open(self.step.output.stderr, "w")
args = [
"/bin/bash",
configPath,
scramSetup,
scramArch,
scramCommand,
scramProject,
cmsswVersion,
jobReportXML,
cmsswCommand,
cmsswConfig,
userTarball,
userFiles,
cmsswArguments,
]
logging.info("Executing CMSSW. args: %s", args)
# at CERN override the environment to work around problem with GSI authentication plugin and EOS
if socket.getfqdn().endswith("cern.ch"):
returncode = subprocess.call(
args, stdout=stdoutHandle, stderr=stderrHandle, env=dict(os.environ, XRD_LOADBALANCERTTL="86400")
)
else:
returncode = subprocess.call(args, stdout=stdoutHandle, stderr=stderrHandle)
self.setCondorChirpAttrDelayed("Chirp_WMCore_cmsRun_ExitCode", returncode)
self.setCondorChirpAttrDelayed("Chirp_WMCore_%s_ExitCode" % self.stepName, returncode)
stdoutHandle.close()
stderrHandle.close()
self.step.execution.exitStatus = returncode
argsDump = {"arguments": args}
if returncode != 0:
msg = "Error running cmsRun\n%s\n" % argsDump
msg += "Return code: %s\n" % returncode
logging.critical(msg)
raise WMExecutionFailure(returncode, "CmsRunFailure", msg)
try:
self.report.parse(jobReportXML, stepName=self.stepName)
except Exception as ex:
# Catch it if something goes wrong
raise WMExecutionFailure(50115, "BadJobReportXML", str(ex))
stepHelper = WMStepHelper(self.step)
typeHelper = stepHelper.getTypeHelper()
acquisitionEra = self.task.getAcquisitionEra()
processingVer = self.task.getProcessingVersion()
processingStr = self.task.getProcessingString()
validStatus = self.workload.getValidStatus()
inputPath = self.task.getInputDatasetPath()
globalTag = typeHelper.getGlobalTag()
prepID = self.task.getPrepID()
cacheUrl, cacheDB, configID = stepHelper.getConfigInfo()
self.report.setValidStatus(validStatus=validStatus)
self.report.setGlobalTag(globalTag=globalTag)
self.report.setPrepID(prepID)
self.report.setInputDataset(inputPath=inputPath)
self.report.setAcquisitionProcessing(
acquisitionEra=acquisitionEra, processingVer=processingVer, processingStr=processingStr
)
self.report.setConfigURL(configURL="%s;;%s;;%s" % (cacheUrl, cacheDB, configID))
# Attach info to files
self.report.addInfoToOutputFilesForStep(stepName=self.stepName, step=self.step)
self.report.checkForOutputFiles(stepName=self.stepName)
self.report.checkForAdlerChecksum(stepName=self.stepName)
self.report.checkForRunLumiInformation(stepName=self.stepName)
if self.step.output.keep != True:
self.report.killOutput()
else:
# Check that we only keep the desired output
for module in stepHelper.getIgnoredOutputModules():
self.report.deleteOutputModuleForStep(stepName=self.stepName, moduleName=module)
# Add stageout LFN to existing TFileService files
reportAnalysisFiles = self.report.getAnalysisFilesFromStep(self.stepName)
for reportAnalysisFile in reportAnalysisFiles:
newLFN = analysisFileLFN(reportAnalysisFile.fileName, self.step.user.lfnBase, self.job)
addAttributesToFile(reportAnalysisFile, pfn=reportAnalysisFile.fileName, lfn=newLFN, validate=False)
# Add analysis file entries for additional files listed in workflow
for fileName in stepHelper.listAnalysisFiles():
analysisFile = stepHelper.getAnalysisFile(fileName)
if os.path.isfile(analysisFile.fileName):
newLFN = analysisFileLFN(analysisFile.fileName, analysisFile.lfnBase, self.job)
self.report.addAnalysisFile(
analysisFile.fileName,
lfn=newLFN,
Source="UserDefined",
pfn=os.path.join(os.getcwd(), analysisFile.fileName),
validate=False,
)
return
def execute(self, emulator=None):
"""
_execute_
"""
stepModule = "WMTaskSpace.%s" % self.stepName
if emulator is not None:
return emulator.emulate(self.step, self.job)
# write the wrapper script to a temporary location
# I don't pass it directly through os.system because I don't
# trust that there won't be shell-escape shenanigans with
# arbitrary input files
scramSetup = self.step.application.setup.softwareEnvironment
scramCommand = self.step.application.setup.scramCommand
scramProject = self.step.application.setup.scramProject
scramArch = self.step.application.setup.scramArch
cmsswVersion = self.step.application.setup.cmsswVersion
jobReportXML = self.step.output.jobReport
cmsswCommand = self.step.application.command.executable
cmsswConfig = self.step.application.command.configuration
cmsswArguments = self.step.application.command.arguments
userTarball = ','.join(self.step.user.inputSandboxes)
userFiles = ','.join(self.step.user.userFiles)
logging.info('User files are %s', userFiles)
logging.info('User sandboxes are %s', userTarball)
scramArch = getSingleScramArch(scramArch)
multicoreSettings = self.step.application.multicore
try:
logging.info("CMSSW configured for %s cores and %s event streams",
multicoreSettings.numberOfCores,
multicoreSettings.eventStreams)
except AttributeError:
logging.info(
"No value set for multicore numberOfCores or eventStreams")
logging.info("Executing CMSSW step")
#
# set any global environment variables
#
try:
os.environ['FRONTIER_ID'] = 'wmagent_%s' % (
self.report.data.workload)
except Exception as ex:
logging.error('Have critical error in setting FRONTIER_ID: %s',
str(ex))
logging.error(
'Continuing, as this is not a critical function yet.')
#
# scram bootstrap
#
scram = Scram(
command=scramCommand,
version=cmsswVersion,
initialise=self.step.application.setup.softwareEnvironment,
directory=self.step.builder.workingDir,
architecture=scramArch,
)
logging.info("Runing SCRAM")
try:
projectOutcome = scram.project()
except Exception as ex:
msg = "Exception raised while running scram.\n"
msg += str(ex)
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
if projectOutcome > 0:
msg = scram.diagnostic()
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
runtimeOutcome = scram.runtime()
if runtimeOutcome > 0:
msg = scram.diagnostic()
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
#
# pre scripts
#
logging.info("Running PRE scripts")
for script in self.step.runtime.preScripts:
# TODO: Exception handling and error handling & logging
scriptProcess = subprocess.Popen(
["/bin/bash"],
shell=True,
cwd=self.step.builder.workingDir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
# BADPYTHON
scriptProcess.stdin.write(
"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH\n")
invokeCommand = "%s -m WMCore.WMRuntime.ScriptInvoke %s %s \n" % (
sys.executable, stepModule, script)
logging.info(" Invoking command: %s", invokeCommand)
scriptProcess.stdin.write(invokeCommand)
stdout, stderr = scriptProcess.communicate()
retCode = scriptProcess.returncode
if retCode > 0:
msg = "Error running command\n%s\n" % invokeCommand
msg += "%s\n %s\n %s\n" % (retCode, stdout, stderr)
logging.critical("Error running command")
logging.critical(msg)
raise WMExecutionFailure(50513, "PreScriptFailure", msg)
#
# pre scripts with scram
#
logging.info("RUNNING SCRAM SCRIPTS")
for script in self.step.runtime.scramPreScripts:
# invoke scripts with scram()
runtimeDir = getattr(self.step.runtime, 'scramPreDir', None)
invokeCommand = self.step.runtime.invokeCommand if hasattr(self.step.runtime, 'invokeCommand') else \
"%s -m WMCore.WMRuntime.ScriptInvoke %s" % (sys.executable, stepModule)
invokeCommand += " %s \n" % script
retCode = scram(invokeCommand, runtimeDir=runtimeDir)
if retCode > 0:
msg = "Error running command\n%s\n" % invokeCommand
msg += "%s\n " % retCode
msg += scram.diagnostic()
logging.critical(msg)
raise WMExecutionFailure(50513, "PreScriptScramFailure", msg)
configPath = "%s/%s-main.sh" % (self.step.builder.workingDir,
self.stepName)
handle = open(configPath, 'w')
handle.write(CONFIG_BLOB)
handle.close()
# spawn this new process
# the script looks for:
# <SCRAM_COMMAND> <SCRAM_PROJECT> <CMSSW_VERSION> <JOB_REPORT> <EXECUTABLE> <CONFIG>
# open the output files
stdoutHandle = open(self.step.output.stdout, 'w')
stderrHandle = open(self.step.output.stderr, 'w')
args = [
'/bin/bash', configPath, scramSetup, scramArch, scramCommand,
scramProject, cmsswVersion, jobReportXML, cmsswCommand,
cmsswConfig, userTarball, userFiles, cmsswArguments
]
logging.info("Executing CMSSW. args: %s", args)
# possibly needed environment overrides for CMSSW call go here
envOverride = {}
# work around problem with GSI authentication plugin and EOS at CERN
if socket.getfqdn().endswith("cern.ch"):
envOverride['XRD_LOADBALANCERTTL'] = "86400"
# some libraries linked with CMSSW need HOME in the environment
if 'HOME' not in os.environ:
envOverride['HOME'] = os.environ.get('PWD', "/")
os.environ.update(envOverride)
returncode = subprocess.call(args,
stdout=stdoutHandle,
stderr=stderrHandle)
self.setCondorChirpAttrDelayed('Chirp_WMCore_cmsRun_ExitCode',
returncode)
self.setCondorChirpAttrDelayed(
'Chirp_WMCore_%s_ExitCode' % self.stepName, returncode)
stdoutHandle.close()
stderrHandle.close()
self.step.execution.exitStatus = returncode
argsDump = {'arguments': args}
if returncode != 0:
msg = "Error running cmsRun\n%s\n" % argsDump
msg += "Return code: %s\n" % returncode
logging.critical(msg)
raise WMExecutionFailure(returncode, "CmsRunFailure", msg)
try:
self.report.parse(jobReportXML, stepName=self.stepName)
except Exception as ex:
# Catch it if something goes wrong
raise WMExecutionFailure(50115, "BadJobReportXML", str(ex))
stepHelper = WMStepHelper(self.step)
typeHelper = stepHelper.getTypeHelper()
acquisitionEra = self.task.getAcquisitionEra()
processingVer = self.task.getProcessingVersion()
processingStr = self.task.getProcessingString()
validStatus = self.workload.getValidStatus()
inputPath = self.task.getInputDatasetPath()
globalTag = typeHelper.getGlobalTag()
prepID = self.task.getPrepID()
campaign = self.workload.getCampaign()
cacheUrl, cacheDB, configID = stepHelper.getConfigInfo()
self.report.setValidStatus(validStatus=validStatus)
self.report.setGlobalTag(globalTag=globalTag)
self.report.setCampaign(campaign)
self.report.setPrepID(prepID)
self.report.setInputDataset(inputPath=inputPath)
self.report.setAcquisitionProcessing(acquisitionEra=acquisitionEra,
processingVer=processingVer,
processingStr=processingStr)
self.report.setConfigURL(configURL="%s;;%s;;%s" %
(cacheUrl, cacheDB, configID))
# Attach info to files
self.report.addInfoToOutputFilesForStep(stepName=self.stepName,
step=self.step)
self.report.checkForOutputFiles(stepName=self.stepName)
self.report.checkForAdlerChecksum(stepName=self.stepName)
self.report.checkForRunLumiInformation(stepName=self.stepName)
if self.step.output.keep != True:
self.report.killOutput()
else:
# Check that we only keep the desired output
for module in stepHelper.getIgnoredOutputModules():
self.report.deleteOutputModuleForStep(stepName=self.stepName,
moduleName=module)
# Add stageout LFN to existing TFileService files
reportAnalysisFiles = self.report.getAnalysisFilesFromStep(
self.stepName)
for reportAnalysisFile in reportAnalysisFiles:
newLFN = analysisFileLFN(reportAnalysisFile.fileName,
self.step.user.lfnBase, self.job)
addAttributesToFile(reportAnalysisFile,
pfn=reportAnalysisFile.fileName,
lfn=newLFN,
validate=False)
# Add analysis file entries for additional files listed in workflow
for fileName in stepHelper.listAnalysisFiles():
analysisFile = stepHelper.getAnalysisFile(fileName)
if os.path.isfile(analysisFile.fileName):
newLFN = analysisFileLFN(analysisFile.fileName,
analysisFile.lfnBase, self.job)
self.report.addAnalysisFile(analysisFile.fileName,
lfn=newLFN,
Source='UserDefined',
pfn=os.path.join(
os.getcwd(),
analysisFile.fileName),
validate=False)
return
self.report.setConfigURL(configURL = "%s;;%s;;%s" % (cacheUrl,
cacheDB,
configID))
# Attach info to files
self.report.addInfoToOutputFilesForStep(stepName = self.stepName, step = self.step)
self.report.checkForOutputFiles(stepName = self.stepName)
self.report.checkForAdlerChecksum(stepName = self.stepName)
self.report.checkForRunLumiInformation(stepName = self.stepName)
if self.step.output.keep != True:
self.report.killOutput()
else:
#Check that we only keep the desired output
for module in stepHelper.getIgnoredOutputModules():
self.report.deleteOutputModuleForStep(stepName = self.stepName, moduleName = module)
# Add stageout LFN to existing TFileService files
reportAnalysisFiles = self.report.getAnalysisFilesFromStep(self.stepName)
for reportAnalysisFile in reportAnalysisFiles:
newLFN = analysisFileLFN(reportAnalysisFile.fileName, self.step.user.lfnBase, self.job)
addAttributesToFile(reportAnalysisFile, pfn=reportAnalysisFile.fileName, lfn=newLFN, validate=False)
# Add analysis file entries for additional files listed in workflow
for fileName in stepHelper.listAnalysisFiles():
analysisFile = stepHelper.getAnalysisFile(fileName)
if os.path.isfile(analysisFile.fileName):
newLFN = analysisFileLFN(analysisFile.fileName, analysisFile.lfnBase, self.job)
self.report.addAnalysisFile(analysisFile.fileName, lfn=newLFN, Source='UserDefined',
pfn=os.path.join(os.getcwd(), analysisFile.fileName), validate=False)
self.report.setConfigURL(configURL="%s;;%s;;%s" %
(cacheUrl, cacheDB, configID))
# Attach info to files
self.report.addInfoToOutputFilesForStep(stepName=self.stepName,
step=self.step)
self.report.checkForOutputFiles(stepName=self.stepName)
self.report.checkForAdlerChecksum(stepName=self.stepName)
self.report.checkForRunLumiInformation(stepName=self.stepName)
if self.step.output.keep != True:
self.report.killOutput()
else:
#Check that we only keep the desired output
for module in stepHelper.getIgnoredOutputModules():
self.report.deleteOutputModuleForStep(stepName=self.stepName,
moduleName=module)
# Add stageout LFN to existing TFileService files
reportAnalysisFiles = self.report.getAnalysisFilesFromStep(
self.stepName)
for reportAnalysisFile in reportAnalysisFiles:
newLFN = analysisFileLFN(reportAnalysisFile.fileName,
self.step.user.lfnBase, self.job)
addAttributesToFile(reportAnalysisFile,
pfn=reportAnalysisFile.fileName,
lfn=newLFN,
validate=False)
# Add analysis file entries for additional files listed in workflow