def athenaMPoutputsLinkAndUpdate(newFullFilenames, fileArg): # Any files we link are numbered from 1, because we always set # the filename given to athena has _000 as a suffix so that the # mother process' file can be used without linking fileIndex = 1 linkedNameList = [] newFilenameValue = [] for fname in newFullFilenames: if path.dirname(fname) == "": linkedNameList.append(None) newFilenameValue.append(fname) else: linkName = "{0}{1:03d}".format(path.basename(fname).rstrip('0'), fileIndex) linkedNameList.append(linkName) newFilenameValue.append(linkName) fileIndex += 1 for linkname, fname in zip(linkedNameList, newFullFilenames): if linkname: if len(newFullFilenames) == 1: try: os.rename(fname,fileArg.originalName) newFilenameValue[0]=fileArg.originalName except OSError, e: raise trfExceptions.TransformExecutionException(trfExit.nameToCode("TRF_OUTPUT_FILE_ERROR"), "Failed to move {0} to {1}: {2}".format(fname, linkname, e)) else: try: if path.lexists(linkname): os.unlink(linkname) os.symlink(fname, linkname) except OSError, e: raise trfExceptions.TransformExecutionException(trfExit.nameToCode("TRF_OUTPUT_FILE_ERROR"), "Failed to link {0} to {1}: {2}".format(fname, linkname, e))
def detectAthenaMPProcs(argdict = {}): athenaMPProcs = 0 # Try and detect if any AthenaMP has been enabled try: if 'ATHENA_PROC_NUMBER' in os.environ: athenaMPProcs = int(os.environ['ATHENA_PROC_NUMBER']) if athenaMPProcs < 0: raise ValueError("ATHENA_PROC_NUMBER value was less than zero") msg.info('AthenaMP detected from ATHENA_PROC_NUMBER with {0} workers'.format(athenaMPProcs)) elif 'athenaopts' in argdict: for substep in argdict['athenaopts'].value: procArg = [opt.replace("--nprocs=", "") for opt in argdict['athenaopts'].value[substep] if '--nprocs' in opt] if len(procArg) == 0: athenaMPProcs = 0 elif len(procArg) == 1: athenaMPProcs = int(procArg[0]) if athenaMPProcs < 0: raise ValueError("--nprocs was set to a value less than zero") else: raise ValueError("--nprocs was set more than once in 'athenaopts'") msg.info('AthenaMP detected from "nprocs" setting with {0} workers for substep {1}'.format(athenaMPProcs,substep)) except ValueError, errMsg: myError = 'Problem discovering AthenaMP setup: {0}'.format(errMsg) raise trfExceptions.TransformExecutionException(trfExit.nameToCode('TRF_EXEC_SETUP_FAIL'), myError)
def postExecute(self): msg.info("Check for trig_cost.root file") #costmon generates the file trig_cost.root #to save on panda it needs to be renamed via the outputNTUP_TRIGCOSTFile argument expectedFileName = 'trig_cost.root' #first check argument is in dict if 'outputNTUP_TRIGCOSTFile' in self.conf.argdict: #check file is created if (os.path.isfile(expectedFileName)): msg.info( 'Renaming %s to %s' % (expectedFileName, self.conf.argdict['outputNTUP_TRIGCOSTFile'].value[0])) try: os.rename( expectedFileName, self.conf.argdict['outputNTUP_TRIGCOSTFile'].value[0]) except OSError, e: raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_OUTPUT_FILE_ERROR'), 'Exception raised when renaming {0} to {1}: {2}'. format( expectedFileName, self.conf. argdict['outputNTUP_TRIGCOSTFile'].value[0], e)) else: msg.error( 'NTUP_TRIGCOST argument defined %s but %s not created' % (self.conf.argdict['outputNTUP_TRIGCOSTFile'].value[0], expectedFileName))
def preExecute(self, input = set(), output = set()): # First we need to strip the filter file down to events that are present # in the RAW file we are going to skim. This is because the HI workflow # will provide millions of events in their filter file, more than acmd.py # can cope with. listEvtCommand = ['AtlListBSEvents.exe', '-l'] listEvtCommand.extend(self.conf.argdict['inputBSFile'].value) # For best lookup speed, we store the runnumber/eventnumber in a dictionary (set would also # be fast) rawEventList = {} try: for line in subprocess.check_output(listEvtCommand).split("\n"): if line.startswith("Index="): try: splitStrings = line.split(" ") runprefix, runstr = splitStrings[1].split("=") evtprefix, evtstr = splitStrings[2].split("=") # Check sanity if runprefix != "Run" or evtprefix != "Event": msg.warning("Failed to understand this line from AtlListBSEvents: {0}".format(line)) else: runnumber = int(runstr) evtnumber = int(evtstr) # We build up a string key as "RUN-EVENT", so that we can take advantage of # the fast hash search against a dictionary rawEventList[runstr + "-" + evtstr] = True msg.debug("Identified run {0}, event {1} in input RAW files".format(runstr, evtstr)) except ValueError, e: msg.warning("Failed to understand this line from AtlListBSEvents: {0}".format(line)) except subprocess.CalledProcessError, e: errMsg = "Call to AtlListBSEvents.exe failed: {0}".format(e) msg.error(erMsg) raise trfExceptions.TransformExecutionException(trfExit.nameToCode("TRF_EXEC_SETUP_FAIL"), errMsg)
def writeTranslate(runTranslate, runArgs, name, substep, first, output): msg.info('Writing options to file \"%s\"' % runTranslate) option = getOption(runArgs, name, substep, first, output) msg.info('Options set to: \"%s\":' % option) with open(runTranslate, 'w') as runTranslateFile: try: print >> runTranslateFile, os.linesep, "option = ", option except (IOError, OSError) as e: errMsg = 'Got an error when writing JO template {0}: {1}'.format( runTranslateFile, e) msg.error(errMsg) raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_EXEC_RUNARGS_ERROR'), errMsg)
def writeRunArgs(self, input=dict(), output=dict()): msg.info('Writing runArgs to file \"%s\"', self._runArgsFile) ## Check consistency btw --CA flag and provided skeletons: if 'CA' in self._exe.conf.argdict: if self._exe._skeletonCA is None: errMsg = "Got the --CA option but this transform doesn't supply a ComponentAccumulator-based skeleton file" msg.error(errMsg) raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_EXEC_RUNARGS_ERROR'), errMsg) else: # 'CA' not in self._exe.conf.argdict if self._exe._skeleton is None: errMsg = "No --CA option given, but this transform doesn't supply old-style skeleton file" msg.error(errMsg) raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_EXEC_RUNARGS_ERROR'), errMsg) with open(self._runArgsFile, 'w') as runargsFile: try: # First write a little header print(os.linesep.join( ("# Run arguments file auto-generated on {0} by:".format( time.asctime()), "# JobTransform: {0}".format(self._exe.name), "# Version: {0}".format(self._version))), file=runargsFile) # Now make sure we import the runArgs class for out job options print(os.linesep.join( ("# Import runArgs class", "from PyJobTransforms.trfJobOptions import RunArguments", "{0} = RunArguments()".format(self._runArgsName))), file=runargsFile) # Handy to write the substep name here as it can be used as (part of) a random seed # in some cases print('{0}.trfSubstepName = {1!r}'.format( self._runArgsName, self._exe.name), os.linesep, file=runargsFile) # Now loop over the core argdict and see what needs to be given as a runArg declaredRunargs = [] for k, v in iteritems(self._exe.conf.argdict): # Check if this arg is supposed to be in runArgs if isinstance(v, trfArgClasses.argument) and v.isRunarg: # Files handled later if isinstance(v, trfArgClasses.argFile): continue msg.debug( 'Argument {0} is a runarg, will be added to JO file (value {1})' .format(k, v.value)) ## @note Substep type arguments are rather special, they apply to only named # executors or substeps. We use the returnMyValue() method to sort out what # specific value applies to us if isinstance(v, trfArgClasses.argSubstep): myValue = v.returnMyValue(exe=self._exe) if myValue is not None: print("{0}.{1!s} = {2!r}".format( self._runArgsName, k, myValue), file=runargsFile) msg.debug( 'Added substep type argument {0} as: {1}'. format(k, myValue)) declaredRunargs.append(k) else: print("{0}.{1!s} = {2!r}".format( self._runArgsName, k, v.value), file=runargsFile) declaredRunargs.append(k) else: msg.debug( 'Argument {0} is not a runarg - ignored'.format(k)) # Now make sure that if we did not add maxEvents then we set this to -1, which # avoids some strange defaults that only allow 5 events to be processed if 'maxEvents' not in declaredRunargs: print(os.linesep.join(( "", "# Explicitly added to process all events in this step", "{0}.maxEvents = -1".format(self._runArgsName), )), file=runargsFile) # Now deal with our input and output files print(os.linesep, "# Input data", file=runargsFile) for dataType, dataArg in iteritems(input): print('{0}.input{1}File = {2!r}'.format( self._runArgsName, dataType, dataArg.value), file=runargsFile) print('{0}.input{1}FileType = {2!r}'.format( self._runArgsName, dataType, dataArg.type), file=runargsFile) # Add the input event count, if we know it if dataArg.isCached(metadataKeys=['nentries']): print('{0}.input{1}FileNentries = {2!r}'.format( self._runArgsName, dataType, dataArg.nentries), file=runargsFile) print("{0}.{1}FileIO = {2!r}".format( self._runArgsName, dataType, self._exe.conf.dataDictionary[dataType].io), file=runargsFile) print(os.linesep, "# Output data", file=runargsFile) for dataType, dataArg in iteritems(output): # Need to be careful to convert _output_ filename as a strings, not a list print('{0}.output{1}File = {2!r}'.format( self._runArgsName, dataType, dataArg.value[0]), file=runargsFile) print('{0}.output{1}FileType = {2!r}'.format( self._runArgsName, dataType, dataArg.type), file=runargsFile) # Process all of the tweaky special runtime arguments print(os.linesep, "# Extra runargs", file=runargsFile) ## @note extraRunargs are passed using repr, i.e., they should be constants for k, v in iteritems(self._exe._extraRunargs): ## @note: What to do if this is a CLI argument as well, in particular # for arguments like preExec we want to add to the list, not replace it if k in declaredRunargs: if isinstance(self._exe.conf.argdict[k].value, list): msg.debug('Extending runarg {0!s}={1!r}'.format( k, v)) print('{0}.{1!s}.extend({2!r})'.format( self._runArgsName, k, v), file=runargsFile) else: msg.debug('Adding runarg {0!s}={1!r}'.format(k, v)) print('{0}.{1!s} = {2!r}'.format( self._runArgsName, k, v), file=runargsFile) ## @note runtime runargs are passed as strings, i.e., they can be evaluated print(os.linesep, '# Extra runtime runargs', file=runargsFile) for k, v in iteritems(self._exe._runtimeRunargs): # These options are string converted, not repred, so they can write an option # which is evaluated at runtime # Protect this with try: except: for the Embedding use case msg.debug('Adding runarg {0!s}={1!r}'.format(k, v)) print(os.linesep.join( ('try:', ' {0}.{1!s} = {2!s}'.format( self._runArgsName, k, v), 'except AttributeError:', ' printfunc ("WARNING - AttributeError for {0}")'. format(k))), file=runargsFile) ## @note Now write the literals into the runargs file if self._exe._literalRunargs is not None: print(os.linesep, '# Literal runargs snippets', file=runargsFile) for line in self._exe._literalRunargs: print(line, file=runargsFile) ## Another special option - dataArgs are always written to the runargs file for dataType in self._exe._dataArgs: print(os.linesep, '# Forced data value arguments', file=runargsFile) if dataType in self._exe.conf.dataDictionary: print('{0}.data{1}arg = {2!r}'.format( self._runArgsName, dataType, self._exe.conf.dataDictionary[dataType].value), file=runargsFile) else: print( '# Warning: data type "{0}" is not part of this transform' .format(dataType), file=runargsFile) # This adds the correct JO fragment for AthenaMP job, where we need to ask # the FileMgr to produce the requested log and report files # Also, aggregating the workers' logfiles into the mother's makes life # easier for debugging if self._exe._athenaMP: print(os.linesep, '# AthenaMP Options. nprocs = %d' % self._exe._athenaMP, file=runargsFile) # Proxy for both options print(os.linesep.join(( os.linesep, 'from AthenaMP.AthenaMPFlags import jobproperties as AthenaMPJobProps', 'AthenaMPJobProps.AthenaMPFlags.WorkerTopDir="{0}"'. format(self._exe._athenaMPWorkerTopDir), 'AthenaMPJobProps.AthenaMPFlags.OutputReportFile="{0}"' .format(self._exe._athenaMPFileReport), 'AthenaMPJobProps.AthenaMPFlags.EventOrdersFile="{0}"'. format(self._exe._athenaMPEventOrdersFile), 'AthenaMPJobProps.AthenaMPFlags.CollectSubprocessLogs=True' )), file=runargsFile) if self._exe._athenaMPStrategy: # Beware of clobbering a non default value (a feature used by EventService) print( 'if AthenaMPJobProps.AthenaMPFlags.Strategy.isDefault():', file=runargsFile) print( '\tAthenaMPJobProps.AthenaMPFlags.Strategy="{0}"'. format(self._exe._athenaMPStrategy), file=runargsFile) if self._exe._athenaMPReadEventOrders: if os.path.isfile(self._exe._athenaMPEventOrdersFile): print( 'AthenaMPJobProps.AthenaMPFlags.ReadEventOrders=True', file=runargsFile) else: raise trfExceptions.TransformExecutionException( trfExit.nameToCode("TRF_EXEC_RUNARGS_ERROR"), "Failed to find file: {0} required by athenaMP option: --athenaMPUseEventOrders true" .format(self._exe._athenaMPEventOrdersFile)) if 'athenaMPEventsBeforeFork' in self._exe.conf.argdict: print( 'AthenaMPJobProps.AthenaMPFlags.EventsBeforeFork={0}' .format(self._exe.conf. argdict['athenaMPEventsBeforeFork'].value), file=runargsFile) if 'CA' in self._exe.conf.argdict: print(os.linesep, '# Threading flags', file=runargsFile) #Pass the number of threads threads = self._exe._athenaMT concurrentEvents = self._exe._athenaConcurrentEvents msg.debug('Adding runarg {0!s}={1!r}'.format( 'threads', threads)) print('{0}.{1!s} = {2!r}'.format(self._runArgsName, 'threads', threads), file=runargsFile) msg.debug('Adding runarg {0!s}={1!r}'.format( 'concurrentEvents', concurrentEvents)) print('{0}.{1!s} = {2!r}'.format(self._runArgsName, 'concurrentEvents', concurrentEvents), file=runargsFile) #ComponentAccumulator based config, import skeleton here: print(os.linesep, '# Import skeleton and execute it', file=runargsFile) print('from {0} import fromRunArgs'.format( self._exe._skeletonCA), file=runargsFile) print('fromRunArgs({0})'.format(self._runArgsName), file=runargsFile) msg.info('Successfully wrote runargs file {0}'.format( self._runArgsFile)) except (IOError, OSError) as e: errMsg = 'Got an error when writing JO template {0}: {1}'.format( self._runArgsFile, e) msg.error(errMsg) raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_EXEC_RUNARGS_ERROR'), errMsg)
def athenaMPOutputHandler(athenaMPFileReport, athenaMPWorkerTopDir, dataDictionary, athenaMPworkers, skipFileChecks = False, argdict = {}): msg.debug("MP output handler called for report {0} and workers in {1}, data types {2}".format(athenaMPFileReport, athenaMPWorkerTopDir, dataDictionary.keys())) outputHasBeenHandled = dict([ (dataType, False) for dataType in dataDictionary.keys() if dataDictionary[dataType] ]) # if sharedWriter mode is active ignore athenaMPFileReport sharedWriter=False if 'sharedWriter' in argdict and argdict['sharedWriter'].value: sharedWriter=True skipFileChecks=True if not sharedWriter: # First, see what AthenaMP told us mpOutputs = ElementTree.ElementTree() try: mpOutputs.parse(athenaMPFileReport) except IOError: raise trfExceptions.TransformExecutionException(trfExit.nameToCode("TRF_OUTPUT_FILE_ERROR"), "Missing AthenaMP outputs file {0} (probably athena crashed)".format(athenaMPFileReport)) for filesElement in mpOutputs.getroot().getiterator(tag='Files'): msg.debug('Examining element {0} with attributes {1}'.format(filesElement, filesElement.attrib)) originalArg = None startName = filesElement.attrib['OriginalName'] for dataType, fileArg in dataDictionary.iteritems(): if fileArg.value[0] == startName: originalArg = fileArg outputHasBeenHandled[dataType] = True break if originalArg is None: msg.warning('Found AthenaMP output with name {0}, but no matching transform argument'.format(startName)) continue msg.debug('Found matching argument {0}'.format(originalArg)) fileNameList = [] for fileElement in filesElement.getiterator(tag='File'): msg.debug('Examining element {0} with attributes {1}'.format(fileElement, fileElement.attrib)) fileNameList.append(path.relpath(fileElement.attrib['name'])) athenaMPoutputsLinkAndUpdate(fileNameList, fileArg) # Now look for additional outputs that have not yet been handled if len([ dataType for dataType in outputHasBeenHandled if outputHasBeenHandled[dataType] is False]): # OK, we have something we need to search for; cache the dirwalk here MPdirWalk = [ dirEntry for dirEntry in os.walk(athenaMPWorkerTopDir) ] for dataType, fileArg in dataDictionary.iteritems(): if outputHasBeenHandled[dataType]: continue if fileArg.io is "input": continue msg.info("Searching MP worker directories for {0}".format(dataType)) startName = fileArg.value[0] fileNameList = [] for entry in MPdirWalk: if "evt_count" in entry[0]: continue if "range_scatterer" in entry[0]: continue # N.B. AthenaMP may have made the output name unique for us, so # we need to treat the original name as a prefix possibleOutputs = [ fname for fname in entry[2] if fname.startswith(startName) ] if len(possibleOutputs) == 0: continue elif len(possibleOutputs) == 1: fileNameList.append(path.join(entry[0], possibleOutputs[0])) elif skipFileChecks: pass else: raise trfExceptions.TransformExecutionException(trfExit.nameToCode("TRF_OUTPUT_FILE_ERROR"), "Found multiple matching outputs for datatype {0} in {1}: {2}".format(dataType, entry[0], possibleOutputs)) if skipFileChecks: pass elif len(fileNameList) != athenaMPworkers: raise trfExceptions.TransformExecutionException(trfExit.nameToCode("TRF_OUTPUT_FILE_ERROR"), "Found {0} output files for {1}, expected {2} (found: {3})".format(len(fileNameList), dataType, athenaMPworkers, fileNameList)) # Found expected number of files - good! athenaMPoutputsLinkAndUpdate(fileNameList, fileArg)
def detectAthenaMTThreads(argdict={}): athenaMTThreads = 0 athenaConcurrentEvents = 0 # Try and detect if any AthenaMT has been enabled try: if 'athenaopts' in argdict: for substep in argdict['athenaopts'].value: threadArg = [ opt.replace("--threads=", "") for opt in argdict['athenaopts'].value[substep] if '--threads' in opt ] if len(threadArg) == 0: athenaMTThreads = 0 elif len(threadArg) == 1: if 'multithreaded' in argdict: raise ValueError( "Detected conflicting methods to configure AthenaMT: --multithreaded and --threads=N (via athenaopts). Only one method must be used" ) athenaMTThreads = int(threadArg[0]) if athenaMTThreads < -1: raise ValueError( "--threads was set to a value less than -1") else: raise ValueError( "--threads was set more than once in 'athenaopts'") msg.info( 'AthenaMT detected from "threads" setting with {0} threads for substep {1}' .format(athenaMTThreads, substep)) concurrentEventsArg = [ opt.replace("--concurrent-events=", "") for opt in argdict['athenaopts'].value[substep] if '--concurrent-events' in opt ] if len(concurrentEventsArg) == 1: athenaConcurrentEvents = int(concurrentEventsArg[0]) if athenaConcurrentEvents < -1: raise ValueError( "--concurrent-events was set to a value less than -1" ) msg.info( 'Custom concurrent event setting read from "concurrent-events" with {0} events for substep {1}' .format(athenaConcurrentEvents, substep)) else: athenaConcurrentEvents = athenaMTThreads if (athenaMTThreads == 0 and 'ATHENA_CORE_NUMBER' in os.environ and 'multithreaded' in argdict): athenaMTThreads = int(os.environ['ATHENA_CORE_NUMBER']) if athenaMTThreads < -1: raise ValueError("ATHENA_CORE_NUMBER value was less than -1") msg.info( 'AthenaMT detected from ATHENA_CORE_NUMBER with {0} threads'. format(athenaMTThreads)) athenaConcurrentEvents = athenaMTThreads except ValueError as errMsg: myError = 'Problem discovering AthenaMT setup: {0}'.format(errMsg) raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_EXEC_SETUP_FAIL'), myError) return athenaMTThreads, athenaConcurrentEvents
#first check argument is in dict if 'outputNTUP_TRIGRATEFile' in self.conf.argdict: #check file is created if (os.path.isfile(expectedFileName)): msg.info( 'Renaming %s to %s' % (expectedFileName, self.conf.argdict['outputNTUP_TRIGRATEFile'].value[0])) try: os.rename( expectedFileName, self.conf.argdict['outputNTUP_TRIGRATEFile'].value[0]) except OSError, e: raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_OUTPUT_FILE_ERROR'), 'Exception raised when renaming {0} to {1}: {2}'. format( expectedFileName, self.conf. argdict['outputNTUP_TRIGRATEFile'].value[0], e)) else: msg.error( 'NTUP_TRIGRATE argument defined %s but %s not created' % (self.conf.argdict['outputNTUP_TRIGRATEFile'].value[0], expectedFileName)) else: msg.info('NTUP_TRIGRATE argument not defined so skip %s check' % expectedFileName) msg.info("Check for trig_ebweight.root file") #costmon generates the file trig_ebweight.root #to save on panda it needs to be renamed via the outputNTUP_TRIGEBWGHTFile argument expectedFileName = 'trig_ebweight.root'
def preExecute(self, input=set(), output=set()): msg.debug( 'Preparing for execution of {0} with inputs {1} and outputs {2}'. format(self.name, input, output)) ## Try to detect AthenaMP mode #self._athenaMP = self._detectAthenaMP() # ## And if this is athenaMP, then set some options for workers and output file report #if self._athenaMP: # self._athenaMPWorkerTopDir = 'athenaMP-workers-{0}-{1}'.format(self._name, self._substep) # self._athenaMPFileReport = 'athenaMP-outputs-{0}-{1}'.format(self._name, self._substep) #else: # self._athenaMPWorkerTopDir = self._athenaMPFileReport = None # Check we actually have events to process! if (self._inputEventTest and 'skipEvents' in self.conf.argdict and self.conf.argdict['skipEvents'].returnMyValue( name=self._name, substep=self._substep, first=self.conf.firstExecutor) is not None): msg.debug('Will test for events to process') for dataType in input: inputEvents = self.conf.dataDictionary[dataType].nentries msg.debug('Got {0} events for {1}'.format( inputEvents, dataType)) if not isinstance(inputEvents, (int, long)): msg.warning( 'Are input events countable? Got nevents={0} so disabling event count check for this input' .format(inputEvents)) elif self.conf.argdict['skipEvents'].returnMyValue( name=self._name, substep=self._substep, first=self.conf.firstExecutor) >= inputEvents: raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_NOEVENTS'), 'No events to process: {0} (skipEvents) >= {1} (inputEvents of {2}' .format( self.conf.argdict['skipEvents'].returnMyValue( name=self._name, substep=self._substep, first=self.conf.firstExecutor), inputEvents, dataType)) ## Write the skeleton file and prep athena if self._skeleton is not None: inputFiles = dict() for dataType in input: inputFiles[dataType] = self.conf.dataDictionary[dataType] outputFiles = dict() for dataType in output: outputFiles[dataType] = self.conf.dataDictionary[dataType] # See if we have any 'extra' file arguments for dataType, dataArg in self.conf.dataDictionary.iteritems(): if dataArg.io == 'input' and self._name in dataArg.executor: inputFiles[dataArg.subtype] = dataArg msg.info('Input Files: {0}; Output Files: {1}'.format( inputFiles, outputFiles)) # Get the list of top options files that will be passed to athena (=runargs file + all skeletons) self._topOptionsFiles = self._jobOptionsTemplate.getTopOptions( input=inputFiles, output=outputFiles) ## Add input/output file information - this can't be done in __init__ as we don't know what our # inputs and outputs will be then if len(input) > 0: self._extraMetadata['inputs'] = list(input) if len(output) > 0: self._extraMetadata['outputs'] = list(output) ## Do we need to run asetup first? asetupString = None if 'asetup' in self.conf.argdict: asetupString = self.conf.argdict['asetup'].returnMyValue( name=self._name, substep=self._substep, first=self.conf.firstExecutor) else: msg.info('Asetup report: {0}'.format(asetupReport())) ## DBRelease configuration dbrelease = dbsetup = None if 'DBRelease' in self.conf.argdict: dbrelease = self.conf.argdict['DBRelease'].returnMyValue( name=self._name, substep=self._substep, first=self.conf.firstExecutor) if dbrelease: # Classic tarball - filename format is DBRelease-X.Y.Z.tar.gz dbdMatch = re.match(r'DBRelease-([\d\.]+)\.tar\.gz', os.path.basename(dbrelease)) if dbdMatch: msg.debug( 'DBRelease setting {0} matches classic tarball file'. format(dbrelease)) if not os.access(dbrelease, os.R_OK): msg.warning( 'Transform was given tarball DBRelease file {0}, but this is not there' .format(dbrelease)) msg.warning( 'I will now try to find DBRelease {0} in cvmfs'. format(dbdMatch.group(1))) dbrelease = dbdMatch.group(1) dbsetup = cvmfsDBReleaseCheck(dbrelease) else: # Check if the DBRelease is setup unpacked, dbsetup = unpackDBRelease( tarball=dbrelease, dbversion=dbdMatch.group(1)) if unpacked: # Now run the setup.py script to customise the paths to the current location... setupDBRelease(dbsetup) # For cvmfs we want just the X.Y.Z release string (and also support 'current') else: dbsetup = cvmfsDBReleaseCheck(dbrelease) # Look for environment updates and perpare the athena command line self._envUpdate = trfEnv.environmentUpdate() #above is needed by _prepAthenaCommandLine, but remove the setStandardEnvironment so doesn't include imf or tcmalloc #self._envUpdate.setStandardEnvironment(self.conf.argdict) self._prepAthenaCommandLine() #to get athenaHLT to read in the relevant parts from the runargs file we have to add the -F option if 'athenaHLT' in self._exe: self._cmd = [ '-F runtranslate.BSRDOtoRAW.py' if x == 'runargs.BSRDOtoRAW.py' else x for x in self._cmd ] # write runTranslate file to be used by athenaHLT writeTranslate('runtranslate.BSRDOtoRAW.py', self.conf.argdict, name=self._name, substep=self._substep, first=self.conf.firstExecutor, output=outputFiles) #instead of running athenaHLT we can dump the options it has loaded #note the -D needs to go after the -F in the command if 'dumpOptions' in self.conf.argdict: self._cmd = [ '-F runtranslate.BSRDOtoRAW.py -D' if x == '-F runtranslate.BSRDOtoRAW.py' else x for x in self._cmd ] #Run preRun step debug_stream analysis if debug_stream=True if 'debug_stream' in self.conf.argdict: inputFiles = dict() for dataType in input: inputFiles[dataType] = self.conf.dataDictionary[dataType] outputFiles = dict() for dataType in output: outputFiles[dataType] = self.conf.dataDictionary[dataType] #set default file name for debug_stream analysis output fileNameDbg = ['debug-stream-monitoring.root'] if 'HIST_DEBUGSTREAMMON' in output: fileNameDbg = outputFiles['HIST_DEBUGSTREAMMON'].value #if file exist then rename file to -old.root to keep as backup if (os.path.isfile(fileNameDbg[0])): oldOutputFileNameDbg = fileNameDbg[0].replace( ".root", "_old.root") msg.info('Renaming %s to %s' % (fileNameDbg[0], oldOutputFileNameDbg)) os.rename(fileNameDbg[0], oldOutputFileNameDbg) #do debug_stream preRun step and get asetup string from debug_stream input files dbgAsetupString = dbgStream.dbgPreRun(inputFiles['BS_RDO'], fileNameDbg) # setup asetup from debug_stream if no --asetup r2b:string was given and is not running with tzero/software/patches as TestArea if asetupString == None and dbgAsetupString != None: asetupString = dbgAsetupString msg.info( 'Will use asetup string for debug_stream analsys %s' % dbgAsetupString) #call athenaExecutor parent as the above overrides what athenaExecutor would have done super(athenaExecutor, self).preExecute(input, output) # Now we always write a wrapper, because it's very convenient for re-running individual substeps # This will have asetup and/or DB release setups in it # Do this last in this preExecute as the _cmd needs to be finalised msg.info('Now writing wrapper for substep executor {0}'.format( self._name)) self._writeAthenaWrapper(asetup=asetupString, dbsetup=dbsetup) msg.info('Athena will be executed in a subshell via {0}'.format( self._cmd))
(argInDict.value[0], matchedOutputFileNames)) argInDict.multipleOK = True argInDict.value = matchedOutputFileNames argInDict._dataset = dataset_argInDict elif (len(matchedOutputFileNames)): msg.info('Single BS file found: will rename file') msg.info('Renaming BS file from %s to %s' % (matchedOutputFileNames[0], argInDict.value[0])) try: os.rename(matchedOutputFileNames[0], argInDict.value[0]) except OSError, e: msg.error( 'Exception raised when renaming {0} #to {1}: {2}'. format(expectedInput, inputFile, e)) raise trfExceptions.TransformExecutionException( trfExit.nameToCode('TRF_OUTPUT_FILE_ERROR'), 'Exception raised when renaming {0} #to {1}: {2}'. format(expectedInput, inputFile, e)) else: msg.error('no BS files created with expected name: %s' % expectedOutputFileName) else: msg.info( 'BS output filetype not defined so skip BS filename check') #Run PostRun step debug_stream analysis if debug_stream=True if 'debug_stream' in self.conf.argdict: msg.info("debug_stream analysis in postExecute") #set default file name for debug_stream analysis output fileNameDbg = ['debug-stream-monitoring.root'] if "outputHIST_DEBUGSTREAMMONFile" in self.conf.argdict:
def preExecute(self, input=set(), output=set()): # First we need to strip the filter file down to events that are present # in the RAW file we are going to skim. This is because the HI workflow # will provide millions of events in their filter file, more than acmd.py # can cope with. listEvtCommand = ['AtlListBSEvents', '-l'] listEvtCommand.extend(self.conf.argdict['inputBSFile'].value) # For best lookup speed, we store the runnumber/eventnumber in a dictionary (set would also # be fast) rawEventList = {} try: for line in subprocess.check_output(listEvtCommand).split("\n"): if line.startswith("Index="): try: splitStrings = line.split(" ") runprefix, runstr = splitStrings[1].split("=") evtprefix, evtstr = splitStrings[2].split("=") # Check sanity if runprefix != "Run" or evtprefix != "Event": msg.warning( "Failed to understand this line from AtlListBSEvents: {0}" .format(line)) else: runnumber = int(runstr) evtnumber = int(evtstr) # We build up a string key as "RUN-EVENT", so that we can take advantage of # the fast hash search against a dictionary rawEventList[runstr + "-" + evtstr] = True msg.debug( "Identified run {0}, event {1} in input RAW files" .format(runstr, evtstr)) except ValueError as e: msg.warning( "Failed to understand this line from AtlListBSEvents: {0}" .format(line)) except subprocess.CalledProcessError as e: errMsg = "Call to AtlListBSEvents failed: {0}".format(e) msg.error(erMsg) raise trfExceptions.TransformExecutionException( trfExit.nameToCode("TRF_EXEC_SETUP_FAIL"), errMsg) msg.info("Found {0} events as skim candidates in RAW inputs".format( len(rawEventList))) # Now open the list of filter events, and check through them slimmedFilterFile = "slimmedFilterFile.{0}".format(os.getpid()) with open(slimmedFilterFile, "w") as slimFF, open( self.conf.argdict['filterFile'].value) as masterFF: count = 0 for line in masterFF: try: runstr, evtstr = line.split() if runstr + "-" + evtstr in rawEventList: msg.debug( "Found run {0}, event {1} in master filter list". format(runstr, evtstr)) os.write(slimFF.fileno(), line) count += 1 except ValueError as e: msg.warning( "Failed to understand this line from master filter file: {0} {1}" .format(line, e)) if count == 0: # If there are no matched events, create a bogus request for run and event 0 to keep # AtlCopyBSEvent.exe CLI msg.info( "No events matched in this input file - empty RAW file output will be made" ) os.write(slimFF.fileno(), "0 0\n") msg.info( "Matched {0} lines from the master filter file against input events; wrote these to {1}" .format(count, slimmedFilterFile)) # Build up the right command line for acmd.py self._cmd = ['acmd.py', 'filter-files'] self._cmd.extend( ('-o', self.conf.argdict['outputBS_SKIMFile'].value[0])) self._cmd.extend(('-s', slimmedFilterFile)) self._cmd.extend(self.conf.argdict['inputBSFile'].value) super(skimRawExecutor, self).preExecute()