def processFile(self, filename, fileReport, step, outputModule):
"""
This attaches the info to the fileReport instance
"""
# Get checksum
(adler32, cksum) = calculateChecksums(filename)
# Get info from spec
output = getattr(step.output.modules, outputModule)
disableGUID = getattr(output, 'disableGUID', False)
fixedLFN = getattr(output, 'fixedLFN', False)
primaryDataset = output.primaryDataset
processedDataset = output.processedDataset
dataTier = output.dataTier
# Get other file information
size = os.stat(filename)[6]
#Get info from file
mergedLFNBase = getattr(fileReport, 'MergedLFNBase', None)
mergedBySize = getattr(fileReport, 'MergedBySize', False)
lfn = getattr(fileReport, 'lfn')
# Do LFN manipulation
# First in the standard case
if not fixedLFN and not disableGUID:
guid = getattr(fileReport, 'guid', None)
if not guid:
msg = "No GUID for file %s" %(lfn)
logging.error(msg)
raise Exception(msg)
# Then we have to change the LFN to match the GUID
dirname = os.path.dirname(lfn)
filelfn = '%s.root' %(str(guid))
setattr(fileReport, 'lfn', os.path.join(dirname, filelfn))
elif not fixedLFN and mergedBySize and mergedLFNBase:
# Then we better do the merge stuff
# Not tested for now
mergedLFNBase.rstrip('/')
newLFN = os.path.join(mergedLFNBase, os.path.basename(lfn))
setattr(fileReport, 'lfn', newLFN)
# Attach values
setattr(fileReport, 'checksums', {'adler32': adler32, 'cksum': cksum})
setattr(fileReport, 'size', size)
setattr(fileReport, "dataset", {"applicationName": "cmsRun",
"applicationVersion": step.application.setup.cmsswVersion,
"primaryDataset": primaryDataset,
"processedDataset": processedDataset,
"dataTier": dataTier})
return fileReport
def AddChecksums(report):
if 'steps' not in report:
return
if 'cmsRun' not in report['steps']:
return
if 'output' not in report['steps']['cmsRun']:
return
for outputMod in report['steps']['cmsRun']['output'].values():
for fileInfo in outputMod:
if 'checksums' in fileInfo:
continue
if 'pfn' not in fileInfo:
if 'fileName' in fileInfo:
fileInfo['pfn'] = fileInfo['fileName']
else:
continue
print("==== Checksum STARTING at %s ====" % time.asctime(time.gmtime()))
print("== Filename: %s" % fileInfo['pfn'])
(adler32, cksum) = calculateChecksums(fileInfo['pfn'])
print("==== Checksum FINISHING at %s ====" % time.asctime(time.gmtime()))
fileInfo['checksums'] = {'adler32': adler32, 'cksum': cksum}
fileInfo['size'] = os.stat(fileInfo['pfn']).st_size
def execute(self, emulator = None, **overrides):
"""
_execute_
"""
# Are we using emulators again?
if (emulator != None):
return emulator.emulate( self.step, self.job )
overrides = {}
if hasattr(self.step, 'override'):
overrides = self.step.override.dictionary_()
# Find alternate stageout location
self.altLFN = overrides.get('altLFN', None)
logging.info("Beginning Steps.Executors.LogArchive.Execute")
logging.info("Using the following overrides: %s " % overrides)
logging.info("Step is: %s" % self.step)
# Wait timout for stageOut
waitTime = overrides.get('waitTime', 3600 + (self.step.retryDelay * self.step.retryCount))
matchFiles = [
".log$",
"FrameworkJobReport",
"Report.pkl",
"Report.pcl",
"^PSet.py$",
"^PSet.pkl$"
]
#Okay, we need a stageOut Manager
useNewStageOutCode = False
if getattr(self.step, 'newStageout', False) or \
('newStageOut' in overrides and overrides.get('newStageOut')):
useNewStageOutCode = True
if not useNewStageOutCode:
# old style
manager = StageOutMgr.StageOutMgr(**overrides)
manager.numberOfRetries = self.step.retryCount
manager.retryPauseTime = self.step.retryDelay
else:
# new style
logging.info("LOGARCHIVE IS USING NEW STAGEOUT CODE")
manager = WMCore.Storage.FileManager.StageOutMgr(
retryPauseTime = self.step.retryDelay,
numberOfRetries = self.step.retryCount,
**overrides)
#Now we need to find all the reports
logFilesForTransfer = []
#Look in the taskSpace first
logFilesForTransfer.extend(self.findFilesInDirectory(self.stepSpace.taskSpace.location, matchFiles))
#What if it's empty?
if len(logFilesForTransfer) == 0:
msg = "Could find no log files in job"
logging.error(msg)
return logFilesForTransfer
#Now that we've gone through all the steps, we have to tar it out
tarName = 'logArchive.tar.gz'
tarBallLocation = os.path.join(self.stepSpace.location, tarName)
tarBall = tarfile.open(tarBallLocation, 'w:gz')
for f in logFilesForTransfer:
tarBall.add(name = f,
arcname = f.replace(self.stepSpace.taskSpace.location, '', 1).lstrip('/'))
tarBall.close()
fileInfo = {'LFN': self.getLFN(tarName),
'PFN' : tarBallLocation,
'PNN' : None,
'GUID' : None
}
signal.signal(signal.SIGALRM, alarmHandler)
signal.alarm(waitTime)
try:
manager(fileInfo)
self.report.addOutputModule(moduleName = "logArchive")
(adler32, cksum) = calculateChecksums(tarBallLocation)
reportFile = {"lfn": fileInfo["LFN"], "pfn": fileInfo["PFN"],
"location": fileInfo["PNN"], "module_label": "logArchive",
"events": 0, "size": 0, "merged": False,
"checksums": {'adler32': adler32, 'cksum' : cksum}}
self.report.addOutputFile(outputModule = "logArchive", aFile= reportFile)
except Alarm:
msg = "Indefinite hang during stageOut of logArchive"
logging.error(msg)
self.report.addError(self.stepName, 60404, "LogArchiveTimeout", msg)
self.report.persist("Report.pkl")
raise WMExecutionFailure(60404, "LogArchiveTimeout", msg)
except WMException as ex:
self.report.addError(self.stepName, 60307, "LogArchiveFailure", str(ex))
self.report.setStepStatus(self.stepName, 0)
self.report.persist("Report.pkl")
raise ex
except Exception as ex:
self.report.addError(self.stepName, 60405, "LogArchiveFailure", str(ex))
self.report.setStepStatus(self.stepName, 0)
self.report.persist("Report.pkl")
msg = "Failure in transferring logArchive tarball\n"
msg += str(ex) + "\n"
msg += traceback.format_exc()
logging.error(msg)
raise WMException("LogArchiveFailure", message = str(ex))
signal.alarm(0)
return
