class RootJob(Job): def __init__(self, script, parameters=None, softwarePackage='HESS/v0.1/root', compiled=False, cpuTime=3600): Job.__init__(self) self.workflow = Workflow() self.executable = '$DIRACROOT/scripts/cta-root-macro' self.setName(os.path.basename(script)) self.script = os.path.basename(script) self.setInputSandbox([script]) self.setCPUTime(cpuTime) arguments = [] toCompile = '' if compiled: toCompile = '+' if parameters: arguments = [ repr(k).replace('"', "\\\\'").replace("'", "\\\\'") for k in parameters ] argumentStr = "%s%s %s" % (os.path.basename(script), toCompile, ' '.join(arguments)) self.setConfigArgs(argumentStr) self.__addSoftwarePackage(softwarePackage) def __addSoftwarePackage(self, package): swPackages = 'SoftwarePackages' description = 'List of Software Packages to be installed' if not self.workflow.findParameter(swPackages): self._addParameter(self.workflow, swPackages, 'JDL', package, description) else: apps = self.workflow.findParameter(swPackages).getValue() if not package in string.split(apps, ';'): apps += ';' + package self._addParameter(self.workflow, swPackages, 'JDL', apps, description)
def __getProductionParameters(self, prodXMLFile, prodID, groupDescription='', bkPassInfo={}, derivedProd=0, reqID=0): """ This method will publish production parameters. """ prodWorkflow = Workflow(prodXMLFile) parameters = {} info = [] for parameterName in ('Priority', 'CondDBTag', 'DDDBTag', 'DQTag', 'eventType', 'processingPass', 'FractionToProcess', 'MinFilesToProcess', 'configName', 'configVersion', 'outputDataFileMask', 'JobType', 'MaxNumberOfTasks'): try: parameters[parameterName] = prodWorkflow.findParameter( parameterName).getValue() info.append( "%s: %s" % (parameterName, prodWorkflow.findParameter(parameterName).getValue())) except AttributeError: continue parameters['SizeGroup'] = self.jobFileGroupSize if prodWorkflow.findParameter( 'InputData'): # now only comes from BK query prodWorkflow.findParameter('InputData').setValue('') gLogger.verbose( 'Resetting input data for production to null, this comes from a BK query...' ) prodXMLFile = self.__createWorkflow(prodXMLFile) # prodWorkflow.toXMLFile(prodXMLFile) if self.transformationFamily: parameters['TransformationFamily'] = self.transformationFamily if not bkPassInfo: bkPassInfo = prodWorkflow.findParameter( 'BKProcessingPass').getValue() if not groupDescription: groupDescription = prodWorkflow.findParameter( 'groupDescription').getValue() parameters['BKCondition'] = prodWorkflow.findParameter( 'conditions').getValue() parameters['BKProcessingPass'] = bkPassInfo parameters['groupDescription'] = groupDescription parameters['RequestID'] = reqID parameters['DerivedProduction'] = derivedProd result = self.__getOutputLFNs(prodID, '99999999', prodXMLFile) if not result['OK']: gLogger.error('Could not create production LFNs', result) outputLFNs = result['Value'] parameters['OutputLFNs'] = outputLFNs # the list of output directories is later used for consistency check and for removing output data outputDirectories = [] del outputLFNs[ 'BookkeepingLFNs'] # since ProductionOutputData uses the file mask for i in outputLFNs.values(): for j in i: outputDir = '%s%s' % (j.split(str(prodID))[0], prodID) if outputDir not in outputDirectories: outputDirectories.append(outputDir) parameters['OutputDirectories'] = outputDirectories # Now for the steps of the workflow stepKeys = sorted(bkPassInfo.keys()) for step in stepKeys: info.append('====> %s %s %s' % (bkPassInfo[step]['ApplicationName'], bkPassInfo[step]['ApplicationVersion'], step)) info.append('%s Option Files:' % (bkPassInfo[step]['ApplicationName'])) if bkPassInfo[step]['OptionFiles']: for opts in bkPassInfo[step]['OptionFiles'].split(';'): info.append('%s' % opts) if bkPassInfo[step]['ExtraPackages']: info.append('ExtraPackages: %s' % (bkPassInfo[step]['ExtraPackages'])) try: if parameters['BkQuery']: info.append('\nBK Input Data Query:') for bkn, bkv in parameters['BkQuery'].iteritems(): info.append('%s= %s' % (bkn, bkv)) except KeyError: pass # BK output directories (very useful) bkPaths = [] bkOutputPath = '%s/%s/%s/%s/%s/%s' % ( parameters['configName'], parameters['configVersion'], parameters['BKCondition'], parameters.get('processingPass', ''), parameters['groupDescription'], parameters['eventType']) fileTypes = parameters['outputDataFileMask'] fileTypes = [a.upper() for a in fileTypes.split(';')] # Annoying that histograms are extension root if 'ROOT' in fileTypes: fileTypes.remove('ROOT') fileTypes.append('HIST') for ft in fileTypes: bkPaths.append('%s/%s' % (bkOutputPath, ft)) parameters['BKPaths'] = bkPaths info.append('\nBK Browsing Paths:\n%s' % ('\n'.join(bkPaths))) infoString = '\n'.join(info) parameters['DetailedInfo'] = infoString gLogger.verbose('Parameters that will be added: %s' % parameters) return parameters