def GetNewLDLibs(systemConfig, application, applicationVersion): """ Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included @param systemConfig: System config used for the job @param application: name of the application considered @param applicationVersion: version of the application considered @return: new LD_LIBRARY_PATH """ new_ld_lib_path = "" deps = resolveDepsTar(systemConfig, application, applicationVersion) for dep in deps: depfolder = dep.replace(".tgz","").replace(".tar.gz","") res = getSoftwareFolder(depfolder) if not res['OK']: continue basedepfolder = res['Value'] if os.path.exists(os.path.join(basedepfolder, "lib")): gLogger.verbose("Found lib folder in %s" % (basedepfolder)) newlibdir = os.path.join(basedepfolder, "lib") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.path.exists(os.path.join(basedepfolder, "LDLibs")): gLogger.verbose("Found lib folder in %s" % (depfolder)) newlibdir = os.path.join(basedepfolder, "LDLibs") new_ld_lib_path = newlibdir ####Remove the libc removeLibc(new_ld_lib_path) if os.environ.has_key("LD_LIBRARY_PATH"): if new_ld_lib_path: new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"] else: new_ld_lib_path = os.environ["LD_LIBRARY_PATH"] return new_ld_lib_path
def execute(self): """ Run the module """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationName: self.result = S_ERROR("Pythia version name not given") elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) appDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'% (self.systemConfig, self.applicationName, self.applicationVersion), '') appDir = appDir.replace(".tgz","").replace(".tar.gz","") res = getSoftwareFolder(appDir) if not res['OK']: self.setApplicationStatus('Pythia: Could not find neither local area not shared area install') return res myappDir = res['Value'] deptar = resolveDepsTar(self.systemConfig, self.applicationName, self.applicationVersion)[0] depdir = deptar.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(depdir) if not res['OK']: return res path = res['Value'] if not os.path.exists(path + "/%s.ep" % depdir): return S_ERROR("Lumi files not found") originpath = os.path.join(path, "/%s.ep" % depdir) randomName = '/tmp/LumiFile-' + GenRandString(8) try: os.mkdir(randomName) except Exception, x: return S_ERROR("Could not create temp dir: %s %s" % (Exception, x))
def GetNewPATH(systemConfig, application, applicationVersion): """ Same as L{GetNewLDLibs},but for the PATH """ new_path = "" deps = resolveDepsTar(systemConfig, application, applicationVersion) for dep in deps: depfolder = dep.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(depfolder) if not res['OK']: continue depfolder = res['Value'] if os.path.exists(os.path.join(depfolder, "bin")): gLogger.verbose("Found bin folder in %s" % (depfolder)) newpathdir = os.path.join(depfolder, "bin") new_path = newpathdir if os.environ.has_key("PATH"): if new_path: new_path = new_path + ":%s" % os.environ["PATH"] else: new_path = os.environ["PATH"] return new_path
def execute(self): """ Run the module """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) tomatoDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "tomato", self.applicationVersion), '') if not tomatoDir: self.log.error('Could not get Tomato tar ball name, cannot proceed') return S_ERROR('Problem accessing CS') tomatoDir = tomatoDir.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(tomatoDir) if not res['Value']: self.setApplicationStatus('Tomato: Could not find neither local area not shared area install') return res myTomatoDir = res['Value'] res = self.prepareMARLIN_DLL(myTomatoDir) if not res['OK']: self.log.error('Failed building MARLIN_DLL: %s' % res['Message']) self.setApplicationStatus('Failed to setup MARLIN_DLL') return S_ERROR('Something wrong with software installation') self.envdict['MARLIN_DLL'] = res['Value'] deps = resolveDepsTar(self.systemConfig, "tomato", self.applicationVersion) for dep in deps: if dep.lower().count('marlin'): marlindir = dep.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(marlindir) if not res['OK']: self.log.error('Marlin was not found in software directory') return res else: self.envdict['MarlinDIR'] = res['Value'] break new_ldlibs = '' if os.environ.has_key('LD_LIBRARY_PATH'): new_ldlibs = os.path.join(myTomatoDir, 'LDLibs') + ":%s" % os.environ['LD_LIBRARY_PATH'] else: new_ldlibs = os.path.join(myTomatoDir, 'LDLibs') self.envdict['LD_LIB_PATH'] = new_ldlibs res = self.GetInputFiles() if not res['OK']: self.log.error(res['Message']) return res listofslcio = res['Value'] finalXML = 'tomato.xml' res = PrepareTomatoSalad(None, finalXML, self.OutputFile, listofslcio, self.collection) if not res['OK']: self.log.error('Could not prepare the Tomato XML: %s' % res['Message']) self.setApplicationStatus('Failed to setup Tomato') return S_ERROR('Failed to setup Tomato') self.result = self.runMarlin(finalXML, self.envdict) if not self.result['OK']: self.log.error('Something wrong during running: %s' % self.result['Message']) self.setApplicationStatus('Error during running %s' % self.applicationName) return S_ERROR('Failed to run %s' % self.applicationName) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info( "Status after the application execution is %s" % str( status ) ) return self.finalStatusReport(status)
def execute(self): """ Called by Agent Executes the following - resolve input variables - resolve installation location - resolve dependencies location (beam_spectra) - get processlist if needed - define output file name - prepare whizard.in - make magic @return: S_OK(), S_ERROR() """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Whizard should not proceed as previous step did not end properly') #if self.debug: # self.excludeAllButEventString = False whizardDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'%(self.systemConfig, "whizard", self.applicationVersion), '') if not whizardDir: self.log.error('Could not get info from CS') self.setApplicationStatus('Failed finding info from CS') return S_ERROR('Failed finding info from CS') whizardDir = whizardDir.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(whizardDir) if not res['OK']: self.log.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res mySoftDir = res['Value'] ###Remove libc removeLibc(mySoftDir + "/lib") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "whizard", self.applicationVersion) #Don't forget to prepend the application's libs new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path ### Resolve dependencies (look for beam_spectra) deps = resolveDepsTar(self.systemConfig, "whizard", self.applicationVersion) path_to_beam_spectra = "" path_to_gridfiles = "" for dep in deps: depfolder = dep.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(depfolder) if not res['OK']: self.log.error("Failed getting software folder", res['Message']) self.setApplicationStatus('Failed finding software') return res depfolder = res['Value'] if depfolder.count("beam_spectra"): path_to_beam_spectra = depfolder elif depfolder.count("gridfiles"): path_to_gridfiles = depfolder ##Env variables needed to run whizard: avoids hard coded locations os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000" os.environ['PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000" os.environ['PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000" os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000" os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000" os.environ['LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000" os.environ['LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000" os.environ['LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000" os.environ['LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000" list_of_gridfiles = [] if path_to_gridfiles and self.useGridFiles: tmp_list_of_gridfiles = [os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles)] gridfilesfound = False for path in tmp_list_of_gridfiles: if os.path.isdir(path) and path.count(str(self.energy)): #Here look for a sub directory for the energy related grid files list_of_gridfiles = [os.path.join(path, item) for item in os.listdir(path)] gridfilesfound = True self.log.info('Found grid files specific for energy %s' % self.energy) break if not gridfilesfound: self.log.info("Will use generic grid files found, hope the energy is set right") list_of_gridfiles = [item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc"))] template = False if self.SteeringFile.count("template"): template = True ## Get from process file the proper whizard.in file if self.getProcessInFile: whizardin = "" res = self.obtainProcessList() if not res['OK']: self.log.error("Could not obtain process list") self.setApplicationStatus('Failed getting processlist') return res whizardin = self.processlist.getInFile(self.evttype) if not whizardin: self.log.error("Whizard input file was not found in process list, cannot proceed") self.setApplicationStatus('Whizard input file was not found') return S_ERROR("Error while resolving whizard input file") if whizardin.count("template"): template = True try: shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in") self.SteeringFile = "whizardnew.in" except: self.log.error("Could not copy %s from %s" % (whizardin, mySoftDir)) self.setApplicationStatus('Failed getting whizard.in file') return S_ERROR("Failed to obtain %s" % whizardin) ##Check existence of Les Houches input file leshouchesfiles = '' if not os.path.exists("LesHouches.msugra_1.in"): if self.susymodel: if self.susymodel == 1: if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir) if self.susymodel == 2: if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)): leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (mySoftDir) if self.Model: if self.genmodel.hasModel(self.Model)['OK']: if self.genmodel.getFile(self.Model)['OK']: if os.path.exists("%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])): leshouchesfiles = "%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value']) else: self.log.error("Request LesHouches file is missing, cannot proceed") self.setApplicationStatus("LesHouches file missing") return S_ERROR("The LesHouches file was not found. Probably you are using a wrong version of whizard.") else: self.log.warn("No file found attached to model %s" % self.Model) else: self.log.error("Model undefined:",self.Model) self.setApplicationStatus("Model undefined") return S_ERROR("No Model %s defined" % self.Model) else: leshouchesfiles = "LesHouches.msugra_1.in" outputfilename = self.evttype if self.jobindex: outputfilename = "%s_%s" % (outputfilename, self.jobindex) if self.optionsdict: self.log.info("Using: %s" % self.optionsdict) self.options = WhizardOptions(self.Model) res = self.options.changeAndReturn(self.optionsdict) if not res['OK']: return res res = self.options.toWhizardDotIn("whizard.in") elif not template: res = PrepareWhizardFile(self.SteeringFile, outputfilename, self.energy, self.RandomSeed, self.NumberOfEvents, self.Lumi, "whizard.in") else: res = PrepareWhizardFileTemplate(self.SteeringFile, outputfilename, self.parameters, "whizard.in") if not res['OK']: self.log.error('Something went wrong with input file generation') self.setApplicationStatus('Whizard: something went wrong with input file generation') return S_ERROR('Something went wrong with whizard.in file generation') foundproceesinwhizardin = res['Value'] scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/sh \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') script.write('declare -x PATH=%s:$PATH\n' % mySoftDir) script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path) script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write('echo Printing content of whizard.in \n') script.write('cat whizard.in\n') script.write('echo =============================\n') script.write('cp %s/whizard.mdl ./\n' % mySoftDir) if leshouchesfiles: if not leshouchesfiles == 'LesHouches.msugra_1.in': script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles)) script.write('ln -s LesHouches.msugra_1.in fort.71\n') if len(list_of_gridfiles): for gridfile in list_of_gridfiles: script.write('cp %s ./\n' % (gridfile)) script.write('cp %s/whizard.prc ./\n' % mySoftDir) if self.genlevelcuts: res = self.makeWhizardDotCut1(self.genlevelcuts) if not res['OK']: script.close() self.log.error("Could not create the cut1 file") return S_ERROR("Could not create the cut1 file") script.write('echo =============================\n') script.write('echo Printing content of whizard.prc \n') script.write('cat whizard.prc\n') script.write('echo =============================\n') extracmd = "" if not self.debug: extracmd = "2>/dev/null" comm = "" if foundproceesinwhizardin: comm = 'whizard --simulation_input \'write_events_file = \"%s\"\' %s\n' % (outputfilename, extracmd) else: comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' %s\n' % (self.evttype, outputfilename, extracmd) self.log.info("Will run %s" % comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % (scriptName) self.setApplicationStatus('Whizard %s step %s' %(self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit=209715200) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} if not self.result['OK']: self.log.error("Failed with error %s" % self.result['Message']) if not os.path.exists(self.applicationLog): self.log.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) lumi = '' message = "" success = False ###Analyse log file logfile = file(self.applicationLog) for line in logfile: if line.count('! Event sample corresponds to luminosity'): elems = line.split() lumi = elems[-1] if line.count("*** Fatal error:"): status = 1 message = line break elif line.count("PYSTOP"): status = 1 message = line break elif line.count("No matrix element available"): status = 1 message = line break elif line.count("Floating point exception"): status = 1 message = line break elif line.count("Event generation finished."): success = True else: status = 0 if success: status = 0 else: status = 1 self.log.info('The sample generated has an equivalent luminosity of %s' % lumi) if lumi: self.workflow_commons['Luminosity'] = float(lumi) else: status = 1 ##Now care for the cross sections info = {} res = self.options.getAsDict() if os.path.exists("whizard.out") and res['OK']: full_opts_dict = res['Value'] processes = full_opts_dict['process_input']['process_id'].split() info = {} info['xsection'] = {} processes.append('sum') inf = open("whizard.out", "r") for line in inf: line = line.rstrip() for process in processes: if not process: continue if line.count(" %s " % process): info['xsection'][process] = {} line = line.lstrip() crosssection = line.split()[1] err_crosssection = line.split()[2] frac = line.split()[4] info['xsection'][process]['xsection'] = float(crosssection) info['xsection'][process]['err_xsection'] = float(err_crosssection) info['xsection'][process]['fraction'] = float(frac) inf.close() if info: if 'Info' not in self.workflow_commons: self.workflow_commons['Info'] = info else: self.workflow_commons['Info'].update(info) self.log.info( "Status after the application execution is %s" % str( status ) ) messageout = 'Whizard %s Successful' % (self.applicationVersion) failed = False if status != 0: self.log.error( "Whizard execution completed with errors:" ) failed = True else: self.log.info( "Whizard execution completed successfully") ###Deal with output file if len(self.OutputFile): if os.path.exists(outputfilename + ".001.stdhep"): ofnames = glob.glob('*.stdhep') if len(ofnames) > 1: basename = self.OutputFile.split(".stdhep")[0] i = 0 for f in ofnames: i += 1 name = basename + "_" + str(i) + ".stdhep" os.rename(f, name) else: os.rename(outputfilename + ".001.stdhep", self.OutputFile) else: self.log.error( "Whizard execution did not produce a stdhep file" ) self.setApplicationStatus('Whizard %s Failed to produce STDHEP file' % (self.applicationVersion)) messageout = 'Whizard Failed to produce STDHEP file' if not self.ignoreapperrors: return S_ERROR(messageout) if failed == True: self.log.error( "==================================\n StdError:\n" ) self.log.error( message ) self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status)) self.log.error('Whizard Exited With Status %s' % (status)) messageout = 'Whizard Exited With Status %s' % (status) if not self.ignoreapperrors: return S_ERROR(messageout) else: self.setApplicationStatus(messageout) return S_OK( { "OutputFile": self.OutputFile } )